import fs from 'node:fs/promises'; import nodePath from "node:path"; import { DatabaseSync } from "node:sqlite"; import "./data-export/facebook.ts"; import { google } from "./data-export/google.ts"; import { TaskTargetPipelineHelper } from "./data-export/task.ts"; declare module "./data-export/task.ts" { interface TaskTargetPipelineHelper { google: typeof google; } } Object.assign(TaskTargetPipelineHelper.prototype, { google }); function loadIntoSqlite( paths: string[], sqlitePath: string ) { // Open an in-memory db for speed const db = new DatabaseSync(":memory:", { allowExtension: true }); db.loadExtension("/home/cobertos/sqlite-files/csv.so") db.enableLoadExtension(false); for (const path of paths) { const table = nodePath.basename(path, ".csv"); console.log(`Loading ${path} → table ${table}`); // const headers = lines[0].split(","); // const columnsSql = headers.map(h => `"${h}" TEXT`).join(", "); db.exec(`CREATE VIRTUAL TABLE temp.intermediate USING csv(filename='${path}');`); db.exec(`CREATE TABLE "${table}" AS SELECT * FROM intermediate;`); db.exec(`DROP TABLE IF EXISTS intermediate;`); } // Dump it all to the path specified db.exec(`VACUUM main INTO '${sqlitePath}'`); db.close(); } async function main() { const t = TaskTargetPipelineHelper; // TODO: // t.fork().cd("/home/cobertos/Seafile/archive/ExportedServiceData/facebook/formapcast_facebook-DEADNAME-May2021-json") // .facebook() // (await t.fork().cd("/home/cobertos/Seafile/archive/ExportedServiceData/facebook/facebook-x-2025-11-29-x.zip").zip()).facebook_v2(); // t.fork().cd("/home/cobertos/Seafile/archive/ExportedServiceData/google/2023-NAMEwork-001") // .google() // let zipTask = t.fork().zip("/home/cobertos/Seafile/archive/ExportedServiceData/facebook/facebook-DEADNAME-May2021-json.zip"); // await (zipTask.fsImpl as any).init(); // zipTask.facebook(); // Now take the output and load it all into a single SQLITE file // const entries = await fs.readdir('OUTTEST', { withFileTypes: true }); // const csvFiles = entries // .filter(e => e.isFile() && e.name.endsWith(".csv")) // .map(e => nodePath.join('OUTTEST', e.name)); // await fs.unlink('your.db'); // loadIntoSqlite(csvFiles, 'your.db'); } main();