base-data-manager/data-export/snapchat.ts

import { pipe, cmd, assignMeta, cd, read, branchGen, type PipelineOp } from "./task.ts";

/**
 * jq helper to normalize Snapchat's "YYYY-MM-DD HH:MM:SS UTC" timestamps
 * to ISO 8601 "YYYY-MM-DDTHH:MM:SS+00:00". Passes through empty strings/nulls.
 */
const SNAPISO = `def snapiso: if . == null or . == "" then "" elif endswith(" UTC") then ((.[:-4] | gsub(" "; "T")) + "+00:00") else . end;`;

/** Login events from account.json */
function snapchat_login_history(): PipelineOp {
  return pipe(
    cd(`json/account.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["ip", "country", "created", "status", "device"],
      (
        .["Login History"][]
        | [.IP, .Country, (.Created | snapiso), .Status, .Device]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Login History",
      columnMeta: ["text", "text", "isodatetime", "text", "text"],
      perRowDescription: 'Login from {0} ({1}) on {2}',
      perRowTags: "snapchat,security",
    })
  );
}

/**
 * Account changes over time from account_history.json.
 * Flattens display name changes, email changes, password changes,
 * bitmoji links, and data download requests into one table.
 */
function snapchat_account_history(): PipelineOp {
  return pipe(
    cd(`json/account_history.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["change_type", "date", "detail"],
      (
        (
          (.["Display Name Change"][]?  | {t: "display_name_change", d: .Date, v: ."Display Name"}),
          (.["Email Change"][]?         | {t: "email_change", d: .Date, v: ."Email Address"}),
          (.["Password Change"][]?      | {t: "password_change", d: .Date, v: ""}),
          (.["Snapchat Linked to Bitmoji"][]? | {t: "linked_to_bitmoji", d: .Date, v: ""}),
          (.["Download My Data Reports"][]? | {t: "data_download", d: .Date, v: (.Status + " / " + ."Email Address")})
        )
        | [.t, (.d | snapiso), .v]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Account History",
      columnMeta: ["text", "isodatetime", "text"],
      perRowDescription: '{0} on {1}: {2}',
      perRowTags: "snapchat,security",
    })
  );
}

/**
 * All friend relationship types from friends.json combined into one table.
 * relationship_type column distinguishes Friends, Blocked Users, etc.
 */
function snapchat_friends(): PipelineOp {
  return pipe(
    cd(`json/friends.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["relationship_type", "username", "display_name", "created_at", "modified_at", "source"],
      (
        ["Friends", "Friend Requests Sent", "Blocked Users", "Deleted Friends", "Ignored Snapchatters", "Pending Requests"][]
        as $key
        | .[$key][]?
        | [$key, .Username, ."Display Name", (."Creation Timestamp" | snapiso), (."Last Modified Timestamp" | snapiso), .Source]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Friends",
      columnMeta: ["text", "text", "text", "isodatetime", "isodatetime", "text"],
      perRowDescription: '{0}: {2} (@{1}) since {3}',
      perRowTags: "snapchat",
    })
  );
}

/**
 * All chat messages from chat_history.json.
 * Keys in that file are friend usernames; they become the conversation_with column.
 */
function snapchat_chat_history(): PipelineOp {
  return pipe(
    cd(`json/chat_history.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["conversation_with", "from", "media_type", "created", "content", "is_sender"],
      (
        to_entries[]
        | .key as $conv
        | .value[]
        | [$conv, .From, ."Media Type", (.Created | snapiso), (.Content // ""), (.IsSender | tostring)]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Chat History",
      columnMeta: ["text", "sender", "text", "isodatetime", "text", "any"],
      perRowDescription: '"{4}" from {1} in {0} at {3}',
      perRowTags: "snapchat,message",
    })
  );
}

/** Approximate visited areas from location_history.json */
function snapchat_location_visits(): PipelineOp {
  return pipe(
    cd(`json/location_history.json`), read(),
    cmd(["jq", "-r", `
      ["time", "city", "region", "postal_code"],
      (
        .["Areas you may have visited in the last two years"][]
        | [.Time, .City, .Region, ."Postal Code"]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Location Visits",
      columnMeta: ["any", "text", "text", "any"],
      perRowDescription: 'Visited {1}, {2} ({3}) around {0}',
      perRowTags: "snapchat,location",
    })
  );
}

/** Spotlight/story activity from shared_story.json */
function snapchat_spotlight(): PipelineOp {
  return pipe(
    cd(`json/shared_story.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["story_date", "story_url", "action_type", "view_time"],
      (
        .["Spotlight History"][]
        | [(.["Story Date"] | snapiso), .["Story URL"], .["Action Type"], .["View Time"]]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Spotlight",
      columnMeta: ["isodatetime", "url", "text", "any"],
      perRowDescription: '{2} on spotlight at {0}',
      perRowTags: "snapchat",
    })
  );
}

/** Terms of service acceptance history from terms_history.json */
function snapchat_terms_history(): PipelineOp {
  return pipe(
    cd(`json/terms_history.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["version", "acceptance_date"],
      (
        .["Snap Inc. Terms of Service"][]
        | [.Version, (.["Acceptance Date"] | snapiso)]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Terms History",
      columnMeta: ["text", "isodatetime"],
      perRowDescription: 'Accepted terms {0} on {1}',
      perRowTags: "snapchat",
    })
  );
}

/** Third-party app permissions from connected_apps.json */
function snapchat_connected_apps(): PipelineOp {
  return pipe(
    cd(`json/connected_apps.json`), read(),
    cmd(["jq", "-r", `${SNAPISO}
      ["app", "time", "type"],
      (
        .Permissions[]
        | [.App, (.Time | snapiso), .Type]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Connected App Permissions",
      columnMeta: ["text", "isodatetime", "text"],
      perRowDescription: '{2} permission for {0} on {1}',
      perRowTags: "snapchat",
    })
  );
}

/** Email campaign subscription preferences from email_campaign_history.json */
function snapchat_email_campaigns(): PipelineOp {
  return pipe(
    cd(`json/email_campaign_history.json`), read(),
    cmd(["jq", "-r", `
      ["campaign", "opt_out_status"],
      (
        .["Email Campaign Subscriptions"][]
        | [.["Email Campaign"], .["Opt Out Status"]]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - Email Campaigns",
      columnMeta: ["text", "text"],
      perRowDescription: 'Email campaign "{0}": {1}',
      perRowTags: "snapchat",
    })
  );
}

/**
 * In-app survey responses from in_app_surveys.json.
 * Keys are dynamic dates ("Survey YYYY/MM/DD"), flattened via to_entries.
 */
function snapchat_surveys(): PipelineOp {
  return pipe(
    cd(`json/in_app_surveys.json`), read(),
    cmd(["jq", "-r", `
      ["survey", "time", "question", "response"],
      (
        to_entries[]
        | .key as $survey
        | .value[]
        | [$survey, .Time, .["Survey Question"], .["Survey Response"]]
      )
      | @csv
    `]),
    assignMeta({
      idValue: "Snapchat - In-App Surveys",
      columnMeta: ["text", "any", "text", "text"],
      perRowDescription: 'Survey "{2}": {3}',
      perRowTags: "snapchat",
    })
  );
}

export function snapchat(): PipelineOp {
  return pipe(
    assignMeta({ idValue: t => `Snapchat - ${t.basename}` }),
    branchGen(function* () {
      yield snapchat_login_history();
      yield snapchat_account_history();
      yield snapchat_friends();
      yield snapchat_chat_history();
      yield snapchat_location_visits();
      yield snapchat_spotlight();
      yield snapchat_terms_history();
      yield snapchat_connected_apps();
      yield snapchat_email_campaigns();
      yield snapchat_surveys();
    })
  );
}