2021-01-27 22:05:15 +00:00
|
|
|
import ModuleRepos from './modules/index.js';
|
2021-02-14 18:30:46 +00:00
|
|
|
import AutoTag from './autoTag.js';
|
2021-03-02 15:57:58 +00:00
|
|
|
import WebhookSend from './webhook.js';
|
2021-01-22 21:26:27 +00:00
|
|
|
|
|
|
|
import Parcel from 'parcel-bundler';
|
2021-01-24 10:49:41 +00:00
|
|
|
import axios from 'axios';
|
2021-01-28 15:24:07 +00:00
|
|
|
import glob from 'glob';
|
2021-01-22 21:26:27 +00:00
|
|
|
|
2021-03-07 20:13:00 +00:00
|
|
|
import { rmSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, existsSync, rmdirSync } from 'fs';
|
2021-01-23 19:51:49 +00:00
|
|
|
import { createHash } from 'crypto';
|
2021-01-22 21:26:27 +00:00
|
|
|
|
|
|
|
import { dirname } from 'path';
|
|
|
|
import { fileURLToPath } from 'url';
|
|
|
|
|
2021-03-01 19:32:30 +00:00
|
|
|
let file;
|
|
|
|
let githubPAT;
|
|
|
|
try {
|
|
|
|
file = JSON.parse(readFileSync('./gh_pat.json'));
|
|
|
|
githubPAT = file.token;
|
|
|
|
} catch (error) {
|
|
|
|
if (error.code !== 'ENOENT') throw error;
|
|
|
|
githubPAT = process.env.GHTOKEN;
|
|
|
|
}
|
2021-01-24 10:49:41 +00:00
|
|
|
|
2021-01-22 21:26:27 +00:00
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
|
|
|
|
|
|
const clonesDir = `${__dirname.replace('/src', '')}/clones`;
|
2021-01-23 10:38:44 +00:00
|
|
|
|
|
|
|
const distDir = `${__dirname.replace('/src', '')}/dist`;
|
|
|
|
|
2021-01-23 19:51:49 +00:00
|
|
|
const modulesDir = `${distDir}/module`;
|
2021-01-22 21:26:27 +00:00
|
|
|
|
|
|
|
const resetDir = (dir) => {
|
|
|
|
rmSync(dir, { recursive: true, force: true });
|
2021-01-30 13:35:08 +00:00
|
|
|
mkdirSync(dir, { recursive: true });
|
2021-01-22 21:26:27 +00:00
|
|
|
};
|
|
|
|
|
2021-01-30 13:35:08 +00:00
|
|
|
if (process.argv[2] === '-f') {
|
|
|
|
resetDir(clonesDir);
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-30 13:35:08 +00:00
|
|
|
resetDir(distDir);
|
|
|
|
resetDir(modulesDir);
|
|
|
|
}
|
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
let previous = [];
|
|
|
|
if (existsSync(clonesDir)) {
|
|
|
|
for (const cloneDir of glob.sync(`${clonesDir}/*/*`)) {
|
|
|
|
process.chdir(cloneDir);
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
const currentHash = await new Promise((res) => exec(`git rev-parse HEAD`, (err, stdout) => res(stdout.trim())));
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-03-07 20:06:04 +00:00
|
|
|
const moduleInRepos = ModuleRepos.map(
|
|
|
|
(x) => x.modules.filter(
|
|
|
|
(y) => y[0] === cloneDir.replace(`${clonesDir}/`, '') && (y[1] === currentHash || !y[1])
|
|
|
|
)
|
|
|
|
).find((x) => x.length > 0);
|
|
|
|
|
|
|
|
if (moduleInRepos) {
|
|
|
|
previous = previous.concat(moduleInRepos);
|
|
|
|
}
|
2021-01-28 15:24:07 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-01-22 21:26:27 +00:00
|
|
|
import { exec } from 'child_process';
|
|
|
|
|
|
|
|
const parcelOptions = {
|
|
|
|
minify: true,
|
|
|
|
watch: false,
|
|
|
|
sourceMaps: false,
|
2021-01-23 10:22:37 +00:00
|
|
|
outDir: modulesDir,
|
|
|
|
logLevel: 0
|
2021-01-22 21:26:27 +00:00
|
|
|
};
|
|
|
|
|
2021-01-24 10:49:41 +00:00
|
|
|
const githubCache = {};
|
|
|
|
|
|
|
|
const getGithubInfo = async (repo) => {
|
|
|
|
if (githubCache[repo]) return githubCache[repo];
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-24 10:49:41 +00:00
|
|
|
const info = (await axios.get(`https://api.github.com/repos/${repo}`, {
|
|
|
|
headers: {
|
|
|
|
'Authorization': `token ${githubPAT}`
|
|
|
|
}
|
|
|
|
})).data;
|
|
|
|
|
|
|
|
githubCache[repo] = info;
|
|
|
|
return info;
|
|
|
|
};
|
|
|
|
|
2021-02-16 21:18:05 +00:00
|
|
|
let oldTotalModulesJson = [];
|
|
|
|
|
2021-02-13 17:36:08 +00:00
|
|
|
for (const parentRepo of ModuleRepos) {
|
|
|
|
let moduleJson = {
|
|
|
|
modules: [],
|
|
|
|
meta: parentRepo.meta
|
|
|
|
};
|
2021-01-22 21:26:27 +00:00
|
|
|
|
2021-03-07 20:06:04 +00:00
|
|
|
const repoJsonPath = `${distDir}/${parentRepo.filename}.json`;
|
|
|
|
|
|
|
|
const currentRepoJson = existsSync(repoJsonPath) ? JSON.parse(readFileSync(repoJsonPath, 'utf8')) : undefined;
|
|
|
|
|
2021-02-13 17:36:08 +00:00
|
|
|
for (const repo of parentRepo.modules) {
|
|
|
|
console.time(repo.slice(0, 2).join(' @ ')+`${repo[2] ? ` ${repo[2]}` : ''}`);
|
|
|
|
|
2021-03-07 20:48:26 +00:00
|
|
|
const name = repo[0];
|
|
|
|
const cloneDir = `${clonesDir}/${name}`;
|
|
|
|
const moduleDir = repo[2] || '';
|
|
|
|
|
2021-02-13 17:36:08 +00:00
|
|
|
if (previous.includes(repo)) {
|
2021-03-07 20:48:26 +00:00
|
|
|
let currentModule = currentRepoJson.modules.filter((x) => x.github.repo === repo[0]);
|
|
|
|
if (currentModule.length > 1) {
|
|
|
|
const manifest = JSON.parse(readFileSync(`${cloneDir}${moduleDir}/goosemodModule.json`));
|
|
|
|
|
|
|
|
currentModule = currentModule.find((x) => x.name === manifest.name);
|
|
|
|
} else {
|
|
|
|
currentModule = currentModule[0];
|
|
|
|
}
|
2021-03-07 20:06:04 +00:00
|
|
|
|
|
|
|
moduleJson.modules.push(currentModule);
|
2021-02-13 17:36:08 +00:00
|
|
|
|
|
|
|
process.stdout.write('[SKIP] ');
|
|
|
|
|
|
|
|
console.timeEnd(repo.slice(0, 2).join(' @ ')+`${repo[2] ? ` ${repo[2]}` : ''}`);
|
|
|
|
|
|
|
|
continue;
|
|
|
|
}
|
2021-03-07 20:06:04 +00:00
|
|
|
|
2021-03-19 20:38:27 +00:00
|
|
|
let githubInfo = getGithubInfo(repo[0]);
|
|
|
|
|
2021-02-13 17:36:08 +00:00
|
|
|
// console.log(repo);
|
|
|
|
|
|
|
|
const url = `https://github.com/${repo[0]}.git`;
|
|
|
|
const commitHash = repo[1];
|
|
|
|
|
|
|
|
const preprocessor = repo[3];
|
|
|
|
|
|
|
|
// resetDir(cloneDir);
|
|
|
|
// rmSync(cloneDir, { recursive: true, force: true });
|
2021-03-07 20:13:00 +00:00
|
|
|
|
2021-03-08 19:12:41 +00:00
|
|
|
if (existsSync(cloneDir)) {
|
|
|
|
process.chdir(cloneDir);
|
|
|
|
|
|
|
|
const currentHash = await new Promise((res) => exec(`git rev-parse HEAD`, (err, stdout) => res(stdout.trim())));
|
|
|
|
|
|
|
|
if (currentHash !== repo[1]) rmSync(cloneDir, { recursive: true, force: true });
|
|
|
|
}
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-03-08 19:12:41 +00:00
|
|
|
process.chdir(distDir); // Incase current wd is broken, in which case exec / git crashes
|
|
|
|
|
2021-02-13 17:36:08 +00:00
|
|
|
await new Promise((res) => exec(`git clone ${url} ${cloneDir}`, res));
|
|
|
|
|
|
|
|
process.chdir(cloneDir);
|
|
|
|
|
|
|
|
const lastHash = await new Promise((res) => exec(`git rev-parse HEAD`, (err, stdout) => res(stdout.trim())));
|
|
|
|
|
|
|
|
await new Promise((res) => exec(`git checkout ${commitHash}`, res));
|
|
|
|
|
|
|
|
if (preprocessor) {
|
|
|
|
(await import(`./preprocessors/${preprocessor}.js`)).default(`${cloneDir}${moduleDir}`, repo);
|
|
|
|
}
|
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
const manifest = JSON.parse(readFileSync(`${cloneDir}${moduleDir}/goosemodModule.json`));
|
2021-02-13 17:36:08 +00:00
|
|
|
|
|
|
|
// console.log(manifest);
|
|
|
|
|
|
|
|
const outFile = `${manifest.name}.js`;
|
|
|
|
|
|
|
|
const bundler = new Parcel(`${cloneDir}${moduleDir}/${manifest.main}`, Object.assign(parcelOptions, {
|
|
|
|
outFile
|
|
|
|
}));
|
|
|
|
|
|
|
|
const bundle = await bundler.bundle();
|
|
|
|
|
|
|
|
const outPath = `${modulesDir}/${outFile}`;
|
|
|
|
let jsCode = readFileSync(outPath, 'utf8');
|
|
|
|
|
|
|
|
jsCode = `${jsCode};parcelRequire('${bundle.entryAsset.basename}').default`; // Make eval return the index module's default export
|
|
|
|
|
|
|
|
// console.log(jsCode);
|
|
|
|
|
|
|
|
writeFileSync(outPath, jsCode);
|
|
|
|
|
|
|
|
const jsHash = createHash('sha512').update(jsCode).digest('hex');
|
|
|
|
|
2021-03-19 20:38:27 +00:00
|
|
|
githubInfo = await githubInfo; // GitHub info is gotten async during other stuff to reduce time
|
|
|
|
|
2021-02-13 17:36:08 +00:00
|
|
|
const manifestJson = {
|
2021-01-28 15:24:07 +00:00
|
|
|
name: manifest.name,
|
|
|
|
description: manifest.description,
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
version: manifest.version,
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-03-07 15:38:12 +00:00
|
|
|
tags: AutoTag(jsCode, manifest.tags),
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
authors: manifest.authors,
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
hash: jsHash,
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
github: {
|
|
|
|
stars: githubInfo.stargazers_count,
|
|
|
|
repo: repo[0]
|
2021-03-07 15:38:12 +00:00
|
|
|
},
|
|
|
|
|
|
|
|
...repo[4]
|
2021-02-13 17:36:08 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
if (manifest.images) manifestJson.images = manifest.images;
|
|
|
|
if (manifest.dependencies) manifestJson.dependencies = manifest.dependencies;
|
|
|
|
|
|
|
|
moduleJson.modules.push(manifestJson);
|
|
|
|
|
2021-01-28 15:24:07 +00:00
|
|
|
console.timeEnd(repo.slice(0, 2).join(' @ ')+`${repo[2] ? ` ${repo[2]}` : ''}`);
|
2021-02-13 17:36:08 +00:00
|
|
|
|
|
|
|
// console.log(lastHash);
|
|
|
|
|
2021-02-14 18:30:46 +00:00
|
|
|
if (commitHash !== '' && lastHash !== commitHash) {
|
2021-02-13 17:36:08 +00:00
|
|
|
console.log('[Warning] Commit hash in modules does not match latest commit in repo');
|
2021-01-24 10:49:41 +00:00
|
|
|
}
|
2021-01-23 10:34:45 +00:00
|
|
|
}
|
2021-02-13 17:36:08 +00:00
|
|
|
|
2021-03-07 20:06:04 +00:00
|
|
|
writeFileSync(repoJsonPath, JSON.stringify(moduleJson));
|
2021-02-16 21:18:05 +00:00
|
|
|
|
|
|
|
oldTotalModulesJson = oldTotalModulesJson.concat(moduleJson.modules);
|
2021-01-23 10:47:15 +00:00
|
|
|
}
|
|
|
|
|
2021-02-16 21:18:05 +00:00
|
|
|
writeFileSync(`${distDir}/modules.json`, JSON.stringify(oldTotalModulesJson));
|
|
|
|
|
2021-03-01 19:32:30 +00:00
|
|
|
copyFileSync(`${__dirname.replace('/src', '')}/_headers`, `${distDir}/_headers`);
|
2021-03-02 15:57:58 +00:00
|
|
|
|
|
|
|
WebhookSend();
|