93 lines
4.1 KiB
TypeScript
93 lines
4.1 KiB
TypeScript
import test from "node:test";
|
|
import nodePath from "node:path";
|
|
import fs from "node:fs/promises";
|
|
import { strict as assert } from "node:assert";
|
|
|
|
import { unzip, pipe, execPaths, type PipelineOp } from "../data-export/task.ts";
|
|
import * as DataIO from "../data-export/io.ts";
|
|
import { assertCSVWellFormed } from "./utils/csvUtils.ts";
|
|
import { assertStringEq, ptry } from "./utils/general.ts";
|
|
|
|
import { facebook, facebook_v2 } from "../data-export/facebook.ts";
|
|
import { discord } from "../data-export/discord.ts";
|
|
import { snapchat } from "../data-export/snapchat.ts";
|
|
import { discord_chat_exporter } from "../data-export/discord-chat-exporter.ts";
|
|
import { fitbit } from "../data-export/fitbit.ts";
|
|
|
|
const THIS_FILE = import.meta.dirname;
|
|
const SNAPSHOT_DIR = nodePath.join(THIS_FILE, 'snapshots');
|
|
const updateSnapshots = process.execArgv.includes("--test-update-snapshots");
|
|
|
|
/**Custom version of t.snapshot
|
|
* * We save each csv id to it's own file (regardless of where it came from)
|
|
* * Properly handles \r\n which nodejs's t.snapshot gets rid of because of
|
|
* how it encodes it into backticks/template literals
|
|
*/
|
|
async function snapshotCSV(id: string, csv: string) {
|
|
const snapshotFilePath = nodePath.join(SNAPSHOT_DIR, id) + "snapshot.csv";
|
|
if (updateSnapshots) {
|
|
// Update the snapshots, do no checking, our internal csv is the source of truth
|
|
await fs.writeFile(snapshotFilePath, csv, { encoding: "utf8" });
|
|
}
|
|
else {
|
|
const [err, prevCSV] = await ptry(fs.readFile(snapshotFilePath, { encoding: "utf8" }));
|
|
assert(!err, `Snapshot file '${snapshotFilePath}' did not exist. Perhaps you need to update snapshots, "--test-update-snapshots"?`);
|
|
assertStringEq(csv, prevCSV, "csv and snapshot csv should be the same");
|
|
}
|
|
}
|
|
|
|
async function testPipelineOp(path: string, op: PipelineOp, overwriteIdPrefix?: string) {
|
|
const targets = await execPaths([{ path, op }]);
|
|
const out = await DataIO.runPipeline(targets);
|
|
const idAndCSVs: [string, string][] = [];
|
|
// Verify and collect all the id + csv tuples
|
|
for (const {target, result} of out) {
|
|
const id = target.id;
|
|
// Check the result for success
|
|
assert.ok(!result.stderr, `Task ${id} should have no stderr output`);
|
|
assert.ok(result.ok, `Task ${id} should be okay`);
|
|
// Check the CSV itself for correctness
|
|
const csv = result.stdout;
|
|
assertCSVWellFormed(csv, `${csv}\nTask ${id} should have well-formed csv.`);
|
|
|
|
idAndCSVs.push([target.id, csv]);
|
|
}
|
|
|
|
// Everything is verified for cleanliness coming out of the current run, verify
|
|
// against the snapshots + save if we're updating snapshots
|
|
const idPrefix = overwriteIdPrefix ?? path.split("/").pop(); // Make unique with the last name of the path
|
|
await Promise.all(idAndCSVs.map(([id, csv])=>snapshotCSV(`${idPrefix}_${id}`, csv)));
|
|
}
|
|
|
|
test("facebook: Can load the 2021-01 export", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/facebook-json-2021-05-01');
|
|
await testPipelineOp(path, facebook());
|
|
});
|
|
test("facebook: Can load the 2021-01 export zipped", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/facebook-json-2021-05-01.zip');
|
|
await testPipelineOp(path, pipe(unzip(), facebook()));
|
|
});
|
|
test("facebook: Can load the 2025-11 export", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/facebook-json-2025-11-29');
|
|
await testPipelineOp(path, facebook_v2());
|
|
});
|
|
|
|
test("discord: Can load the 2021-05 export", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/discord-json-2021-01');
|
|
await testPipelineOp(path, discord());
|
|
});
|
|
|
|
test("snapchat: Can load the 2023-11 export", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/snapchat-2023-11');
|
|
await testPipelineOp(path, snapchat());
|
|
});
|
|
|
|
test("discord-chat-exporter: Can load the 2026-02 export", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/discord-chat-exporter-2026-02');
|
|
await testPipelineOp(path, discord_chat_exporter());
|
|
});
|
|
|
|
test("fitbit: Can load the 2026-02 export", async () => {
|
|
const path = nodePath.join(THIS_FILE, 'fixtures/fitbit-2026-02/FullHumanName');
|
|
await testPipelineOp(path, fitbit(), "fitbit-2026-02");
|
|
});
|