ModuleBuilder/src/index.js

import ModuleRepos from './modules/index.js';

import AutoTag from './autoTag.js';
import WebhookSend from './webhook.js';
import ImageCDN from './imageCdn.js';
import AuthorGen from './authorGen.js';
import SiteGen from './siteGen/index.js';

import Parcel from 'parcel-bundler';
import axios from 'axios';
import glob from 'glob';

import { rmSync, mkdirSync, readFileSync, writeFileSync, existsSync } from 'fs';
import { createHash } from 'crypto';

import { dirname, sep } from 'path';
import { fileURLToPath } from 'url';

let file;
let githubPAT;
try {
  file = JSON.parse(readFileSync('./env.json'));
  githubPAT = file.token;
} catch (error) {
  if (error.code !== 'ENOENT') throw error;
  githubPAT = process.env.GHTOKEN;
}

const __dirname = dirname(fileURLToPath(import.meta.url));

const clonesDir = `${__dirname.replace(`${sep}src`, '')}/clones`;

const distDir = `${__dirname.replace(`${sep}src`, '')}/dist`;
global.distDir = distDir;

const modulesDir = `${distDir}/module`;

const resetDir = (dir) => {
  rmSync(dir, { recursive: true, force: true });
  mkdirSync(dir, { recursive: true });
};

if (process.argv[2] === '-f') {
  resetDir(clonesDir);

  resetDir(distDir);
  resetDir(modulesDir);
}

let previous = [];
if (existsSync(clonesDir)) {
  for (const cloneDir of glob.sync(`${clonesDir}/*/*`)) {
    process.chdir(cloneDir);

    const currentHash = await new Promise((res) =>
      exec(`git rev-parse HEAD`, (err, stdout) => res(stdout.trim())),
    );

    const moduleInRepos = ModuleRepos.map((x) =>
      x.modules.filter(
        (y) => y[0] === cloneDir.replace(`${clonesDir}/`, '') && (y[1] === currentHash || !y[1]),
      ),
    ).find((x) => x.length > 0);

    if (moduleInRepos) {
      previous = previous.concat(moduleInRepos);
    }
  }
}

import { exec } from 'child_process';

const parcelOptions = {
  minify: true,
  watch: false,
  sourceMaps: false,
  outDir: modulesDir,
  logLevel: 0,
};

const githubCache = {};

const getGithubInfo = async (repo) => {
  if (githubCache[repo]) return githubCache[repo];

  const info = (
    await axios.get(`https://api.github.com/repos/${repo}`, {
      headers: {
        Authorization: `token ${githubPAT}`,
      },
    })
  ).data;

  githubCache[repo] = info;
  return info;
};

for (const parentRepo of ModuleRepos) {
  let moduleJson = {
    modules: [],
    meta: parentRepo.meta,
  };

  const repoJsonPath = `${distDir}/${parentRepo.filename}.json`;

  const currentRepoJson = existsSync(repoJsonPath)
    ? JSON.parse(readFileSync(repoJsonPath, 'utf8'))
    : undefined;

  for (const repo of parentRepo.modules) {
    console.time(repo.slice(0, 2).join(' @ ') + `${repo[2] ? ` ${repo[2]}` : ''}`);

    const name = repo[0];
    const cloneDir = `${clonesDir}/${name}`;
    let moduleDir = repo[2] || '';

    try {
      if (previous.includes(repo)) {
        let currentModule = currentRepoJson.modules.filter((x) => x?.github?.repo === repo[0]);
        if (currentModule.length > 1) {
          const manifest = JSON.parse(readFileSync(`${cloneDir}${moduleDir}/goosemodModule.json`));

          currentModule = currentModule.find((x) => x.name === manifest.name);
        } else {
          currentModule = currentModule[0];
        }

        moduleJson.modules.push(currentModule);

        process.stdout.write('[SKIP] ');

        console.timeEnd(repo.slice(0, 2).join(' @ ') + `${repo[2] ? ` ${repo[2]}` : ''}`);

        continue;
      }
    } catch (e) {
      console.log('Cache fail', repo[0], e);
    }

    let githubInfo = getGithubInfo(repo[0]);

    // console.log(repo);

    const url = `https://github.com/${repo[0]}.git`;
    const commitHash = repo[1];

    const preprocessor = repo[3];

    //  resetDir(cloneDir);
    //  rmSync(cloneDir, { recursive: true, force: true });

    if (existsSync(cloneDir)) {
      process.chdir(cloneDir);

      const currentHash = await new Promise((res) =>
        exec(`git rev-parse HEAD`, (err, stdout) => res(stdout.trim())),
      );

      if (currentHash !== repo[1] && repo[1] !== '')
        rmSync(cloneDir, { recursive: true, force: true });
    }

    process.chdir(distDir); // Incase current wd is broken, in which case exec / git crashes

    await new Promise((res) => exec(`git clone ${url} ${cloneDir}`, res));

    process.chdir(cloneDir);

    const lastHash = await new Promise((res) =>
      exec(`git rev-parse HEAD`, (err, stdout) => res(stdout.trim())),
    );

    await new Promise((res) => exec(`git checkout ${commitHash}`, res));

    const commitTimestamp = await new Promise((res) =>
      exec(`git log -1 --format="%at" | xargs -I{} date -d @{} +%s`, (err, stdout) =>
        res(stdout.trim()),
      ),
    );

    if (preprocessor) {
      const preOut = await (
        await import(`./preprocessors/${preprocessor}.js`)
      ).default(`${cloneDir}${moduleDir}`, repo);

      if (preOut !== undefined) {
        moduleDir = preOut;
      }
    }

    const manifest = JSON.parse(readFileSync(`${cloneDir}${moduleDir}/goosemodModule.json`));

    // console.log(manifest);

    const outFile = `${manifest.name}.js`;

    const bundler = new Parcel(
      `${cloneDir}${moduleDir}/${manifest.main}`,
      Object.assign(parcelOptions, {
        outFile,
      }),
    );

    const bundle = await bundler.bundle();

    const outPath = `${modulesDir}/${outFile}`;
    let jsCode = readFileSync(outPath, 'utf8');

    jsCode = `${jsCode};parcelRequire('${bundle.entryAsset.basename}').default`; // Make eval return the index module's default export

    // console.log(jsCode);

    writeFileSync(outPath, jsCode);

    const jsHash = createHash('sha512').update(jsCode).digest('hex');

    githubInfo = await githubInfo; // GitHub info is gotten async during other stuff to reduce time

    const manifestJson = {
      name: manifest.name,
      description: manifest.description,

      version: manifest.version,

      tags: AutoTag(jsCode, manifest.tags),

      authors: manifest.authors,

      hash: jsHash,

      github: {
        stars: githubInfo.stargazers_count,
        repo: repo[0],
      },

      lastUpdated: parseInt(commitTimestamp),

      ...repo[4],
    };

    if (manifest.images) manifestJson.images = manifest.images;
    if (manifest.dependencies) manifestJson.dependencies = manifest.dependencies;

    manifestJson.images = await ImageCDN(manifestJson);

    if (Array.isArray(manifestJson.authors))
      manifestJson.authors = await Promise.all(
        manifestJson.authors.map(async (x) => {
          if (x.match(/^[0-9]{17,18}$/)) {
            return await AuthorGen(x);
          }

          return x;
        }),
      );

    moduleJson.modules.push(manifestJson);

    console.timeEnd(repo.slice(0, 2).join(' @ ') + `${repo[2] ? ` ${repo[2]}` : ''}`);

    // console.log(lastHash);

    if (commitHash !== '' && lastHash !== commitHash) {
      console.log('[Warning] Commit hash in modules does not match latest commit in repo');
    }
  }

  moduleJson.modules = moduleJson.modules.filter((x) => x !== null);

  writeFileSync(repoJsonPath, JSON.stringify(moduleJson));
}

WebhookSend();
await SiteGen();