import test from "node:test"; import nodePath from "node:path"; import fs from "node:fs/promises"; import { strict as assert } from "node:assert"; import { unzip, pipe, execPaths, type PipelineOp } from "../data-export/task.ts"; import * as DataIO from "../data-export/io.ts"; import { assertCSVWellFormed } from "./utils/csvUtils.ts"; import { assertStringEq, ptry } from "./utils/general.ts"; import { facebook, facebook_v2 } from "../data-export/facebook.ts"; import { discord } from "../data-export/discord.ts"; import { snapchat } from "../data-export/snapchat.ts"; import { discord_chat_exporter } from "../data-export/discord-chat-exporter.ts"; import { fitbit } from "../data-export/fitbit.ts"; const THIS_FILE = import.meta.dirname; const SNAPSHOT_DIR = nodePath.join(THIS_FILE, 'snapshots'); const updateSnapshots = process.execArgv.includes("--test-update-snapshots"); /**Custom version of t.snapshot * * We save each csv id to it's own file (regardless of where it came from) * * Properly handles \r\n which nodejs's t.snapshot gets rid of because of * how it encodes it into backticks/template literals */ async function snapshotCSV(id: string, csv: string) { const snapshotFilePath = nodePath.join(SNAPSHOT_DIR, id) + "snapshot.csv"; if (updateSnapshots) { // Update the snapshots, do no checking, our internal csv is the source of truth await fs.writeFile(snapshotFilePath, csv, { encoding: "utf8" }); } else { const [err, prevCSV] = await ptry(fs.readFile(snapshotFilePath, { encoding: "utf8" })); assert(!err, `Snapshot file '${snapshotFilePath}' did not exist. Perhaps you need to update snapshots, "--test-update-snapshots"?`); assertStringEq(csv, prevCSV, "csv and snapshot csv should be the same"); } } async function testPipelineOp(path: string, op: PipelineOp, overwriteIdPrefix?: string) { const targets = await execPaths([{ path, op }]); const out = await DataIO.runPipeline(targets); const idAndCSVs: [string, string][] = []; // Verify and collect all the id + csv tuples for (const {target, result} of out) { const id = target.id; // Check the result for success assert.ok(!result.stderr, `Task ${id} should have no stderr output`); assert.ok(result.ok, `Task ${id} should be okay`); // Check the CSV itself for correctness const csv = result.stdout; assertCSVWellFormed(csv, `${csv}\nTask ${id} should have well-formed csv.`); idAndCSVs.push([target.id, csv]); } // Everything is verified for cleanliness coming out of the current run, verify // against the snapshots + save if we're updating snapshots const idPrefix = overwriteIdPrefix ?? path.split("/").pop(); // Make unique with the last name of the path await Promise.all(idAndCSVs.map(([id, csv])=>snapshotCSV(`${idPrefix}_${id}`, csv))); } test("facebook: Can load the 2021-01 export", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/facebook-json-2021-05-01'); await testPipelineOp(path, facebook()); }); test("facebook: Can load the 2021-01 export zipped", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/facebook-json-2021-05-01.zip'); await testPipelineOp(path, pipe(unzip(), facebook())); }); test("facebook: Can load the 2025-11 export", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/facebook-json-2025-11-29'); await testPipelineOp(path, facebook_v2()); }); test("discord: Can load the 2021-05 export", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/discord-json-2021-01'); await testPipelineOp(path, discord()); }); test("snapchat: Can load the 2023-11 export", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/snapchat-2023-11'); await testPipelineOp(path, snapchat()); }); test("discord-chat-exporter: Can load the 2026-02 export", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/discord-chat-exporter-2026-02'); await testPipelineOp(path, discord_chat_exporter()); }); test("fitbit: Can load the 2026-02 export", async () => { const path = nodePath.join(THIS_FILE, 'fixtures/fitbit-2026-02/FullHumanName'); await testPipelineOp(path, fitbit(), "fitbit-2026-02"); });