67 lines
No EOL
2.3 KiB
TypeScript
67 lines
No EOL
2.3 KiB
TypeScript
import fs from 'node:fs/promises';
|
|
import nodePath from "node:path";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import "./data-export/facebook.ts";
|
|
import { google } from "./data-export/google.ts";
|
|
import { TaskTargetPipelineHelper } from "./data-export/task.ts";
|
|
|
|
declare module "./data-export/task.ts" {
|
|
interface TaskTargetPipelineHelper {
|
|
google: typeof google;
|
|
}
|
|
}
|
|
|
|
Object.assign(TaskTargetPipelineHelper.prototype, {
|
|
google
|
|
});
|
|
|
|
function loadIntoSqlite(
|
|
paths: string[],
|
|
sqlitePath: string
|
|
) {
|
|
// Open an in-memory db for speed
|
|
const db = new DatabaseSync(":memory:", { allowExtension: true });
|
|
db.loadExtension("/home/cobertos/sqlite-files/csv.so")
|
|
db.enableLoadExtension(false);
|
|
for (const path of paths) {
|
|
const table = nodePath.basename(path, ".csv");
|
|
console.log(`Loading ${path} → table ${table}`);
|
|
|
|
// const headers = lines[0].split(",");
|
|
// const columnsSql = headers.map(h => `"${h}" TEXT`).join(", ");
|
|
db.exec(`CREATE VIRTUAL TABLE temp.intermediate USING csv(filename='${path}');`);
|
|
db.exec(`CREATE TABLE "${table}" AS SELECT * FROM intermediate;`);
|
|
db.exec(`DROP TABLE IF EXISTS intermediate;`);
|
|
}
|
|
|
|
// Dump it all to the path specified
|
|
db.exec(`VACUUM main INTO '${sqlitePath}'`);
|
|
db.close();
|
|
}
|
|
|
|
async function main() {
|
|
const t = TaskTargetPipelineHelper;
|
|
// TODO:
|
|
// t.fork().cd("/home/cobertos/Seafile/archive/ExportedServiceData/facebook/formapcast_facebook-DEADNAME-May2021-json")
|
|
// .facebook()
|
|
|
|
// (await t.fork().cd("/home/cobertos/Seafile/archive/ExportedServiceData/facebook/facebook-x-2025-11-29-x.zip").zip()).facebook_v2();
|
|
|
|
// t.fork().cd("/home/cobertos/Seafile/archive/ExportedServiceData/google/2023-NAMEwork-001")
|
|
// .google()
|
|
|
|
|
|
// let zipTask = t.fork().zip("/home/cobertos/Seafile/archive/ExportedServiceData/facebook/facebook-DEADNAME-May2021-json.zip");
|
|
// await (zipTask.fsImpl as any).init();
|
|
|
|
// zipTask.facebook();
|
|
// Now take the output and load it all into a single SQLITE file
|
|
// const entries = await fs.readdir('OUTTEST', { withFileTypes: true });
|
|
// const csvFiles = entries
|
|
// .filter(e => e.isFile() && e.name.endsWith(".csv"))
|
|
// .map(e => nodePath.join('OUTTEST', e.name));
|
|
// await fs.unlink('your.db');
|
|
// loadIntoSqlite(csvFiles, 'your.db');
|
|
}
|
|
|
|
main(); |