Multiple accounts support, load tags online

db/migrations/0001-schema.sql

@@ -1,6 +1,14 @@
 BEGIN TRANSACTION;
 
+CREATE TABLE account (
+   account TEXT NOT NULL,
+   fan_id INTEGER NOT NULL,
+   currency TEXT,
+   PRIMARY KEY (account)
+) WITHOUT ROWID;
+
 CREATE TABLE item (
+   account TEXT NOT NULL,
    item_id INTEGER NOT NULL,
    item_type TEXT NOT NULL,
    band_id INTEGER NOT NULL,
@@ -24,10 +32,11 @@ CREATE TABLE item (
    currency STRING NOT NULL,
    label TEXT,
    label_id INTEGER,
-   PRIMARY KEY (item_id)
+   PRIMARY KEY (account, item_id)
 ) WITHOUT ROWID;
 
 CREATE TABLE track (
+   account TEXT NOT NULL,
    item_id INTEGER NOT NULL,
    track_id INTEGER NOT NULL,
    title TEXT NOT NULL,
@@ -35,15 +44,16 @@ CREATE TABLE track (
    track_number INTEGER,
    duration NUMERIC NOT NULL,
    mp3 TEXT,
-   PRIMARY KEY (item_id, track_id),
+   PRIMARY KEY (account, item_id, track_id),
-   FOREIGN KEY (item_id) REFERENCES item (item_id) ON DELETE CASCADE
+   FOREIGN KEY (account, item_id) REFERENCES item (account, item_id) ON DELETE CASCADE
 ) WITHOUT ROWID;
 
 CREATE TABLE item_tag (
+   account TEXT NOT NULL,
    item_id INTEGER NOT NULL,
    tag TEXT NOT NULL,
-   PRIMARY KEY (item_id, tag),
+   PRIMARY KEY (account, item_id, tag),
-   FOREIGN KEY (item_id) REFERENCES item (item_id) ON DELETE CASCADE
+   FOREIGN KEY (account, item_id) REFERENCES item (account, item_id) ON DELETE CASCADE
 ) WITHOUT ROWID;
 
 COMMIT;

db/orm-defs.d.ts

@@ -1,5 +1,12 @@
 export type Models = {
+	account: {
+		account: string
+		fan_id: number
+		currency: string | null
+	}
+
 	item: {
+		account: string
 		item_id: number
 		item_type: string
 		band_id: number
@@ -26,6 +33,7 @@ export type Models = {
 	}
 
 	track: {
+		account: string
 		item_id: number
 		track_id: number
 		title: string
@@ -36,6 +44,7 @@ export type Models = {
 	}
 
 	item_tag: {
+		account: string
 		item_id: number
 		tag: string
 	}

pug/album_grid.pug

@@ -20,5 +20,5 @@ block view
               = item.band_name
             - let label = item.band_url.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1")
             a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label}))
-              span.s-tag--sponsor!= icons.get("component", 16)
+              span.s-tag--sponsor!= icons.get("flower", 16)
               = label

pug/artist_grid.pug

@@ -20,7 +20,7 @@ block view
               - let labels = item.labels.split("|").map(x => x.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1"))
               each label in labels
                 a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label}))
-                  span.s-tag--sponsor!= icons.get("component", 16)
+                  span.s-tag--sponsor!= icons.get("flower", 16)
                   = label
           each preview in item.previews
             a.d-flex(href=preview.item_url)

pug/home.pug

@@ -0,0 +1,20 @@
+doctype html
+html
+  head
+    meta(charset="utf-8")
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    title BC Explorer
+    link(rel="stylesheet" type="text/css" href="/static/stacks.min.css")
+  body.themed.theme-system.overflow-y-scroll
+    header.s-topbar.ps-sticky.t0
+      .s-topbar--container.wmx9
+          .s-topbar--logo
+            != icons.get("compass-solid", 24)
+            .ml4 BC Explorer
+
+    .mx-auto.wmx9.py24.px16.g24.s-prose
+      h1 Select profile
+      - const names = select("account", "account").pluck().all()
+      ul
+        each name in names
+          li: a(href=`/${name}/`)= name

pug/includes/layout.pug

@@ -63,14 +63,15 @@ html
           ul.s-navigation
             li: +navi("arrange", "album", "album", "Album")
             li: +navi("arrange", "artist", "people-tag", "Artist")
-            li: +navi("arrange", "label", "component", "Label")
+            li: +navi("arrange", "label", "flower", "Label")
+            //- asana, flower, component, circle-spark, rhombus, sphere, union-alt, color-wheel, community, combine
             li: +navi("arrange", "tag", "label", "Tag")
             li: +navi("arrange", "track", "music-note", "Track")
         .px16
         nav
           ul.s-navigation.s-navigation__toggle.g0
             li: +navi("shape", "grid").brr0!= icons.get("view-grid")
-            li: +navi("shape", "list").blr0!= icons.get("table-rows")
+            li: +navi("shape", "list").blr0!= icons.get("align-justify")
         .fl-grow1
         #player(hx-preserve)
           button.s-btn.s-btn__outlined.s-btn__xs!= icons.get("play")
@@ -80,12 +81,17 @@ html
         block view
 
       div
-        .ps-sticky.d-flex.fd-column.g12(style="top: 80px")
+        .ps-sticky.d-flex.fd-column.g12.wmx4(style="top: 80px")
+          if arrange === "tag"
+            include tag-status.pug
           if filter
             .s-sidebarwidget.s-sidebarwidget__blue.d-flex.ai-center.gx16.jc-space-between.p8.pl16
-              != icons.get("search")
+              .fl-shrink0!= icons.get("search")
-              div Searching for #[strong= filter]
+              .fl-grow1= `Searching for `
-              .fl-grow1
+                if filter_field === "item_id"
+                  strong= items[0].item_title
+                else
+                  strong= filter
               a.s-btn.s-notice--btn(href=and({filter: null, filter_field: null})) Clear
           .s-sidebarwidget.wmn3
             .s-sidebarwidget--header Collection

pug/includes/tag-status.pug

@@ -0,0 +1,32 @@
+if downloader.total > 0
+  .d-flex.jc-center#tag-download
+    if !downloader.running && !downloader.outcome
+      form.s-notice.s-notice__info.d-flex.ai-center.p8.gx16.pl16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-post="/api/tag-download")
+        != icons.get("info-circle")
+        div Tag data needs to be downloaded. This will take a while.
+        input(type="hidden" name="account" value=account)
+        button.s-btn.s-btn__outlined Download now
+
+    else if !downloader.outcome
+      .s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s")
+        .d-flex.gx16.ai-center
+          != icons.get("cloud-download")
+          div Downloading tags...
+        p.mt12 You can keep using BC Explorer while this continues in the background.
+        .s-progress.mt16
+          .s-progress--bar(style=`width: ${downloader.processed/downloader.total*100}%`)
+        .d-flex.jc-space-between.fs-fine
+          span= downloader.processed
+          span= downloader.total
+
+    else if downloader.outcome === "Success"
+      .s-notice.s-notice__success.p8.gx16.pl16.d-flex.ai-center.wmn3
+        != icons.get("cloud-check")
+        .fl-grow1 Tags downloaded.
+        - downloadManager.resolve(account)
+        a.s-btn.s-btn__outlined(href="") Refresh
+
+    else
+      .s-notice.s-notice__danger.p8.gx16.pl16.d-flex.ai-center.wmn3
+        != icons.get("cloud-xmark")
+        .fl-grow1= downloader.outcome

pug/tag_grid.pug

@@ -4,18 +4,13 @@ block view
   script
     | var items =
     != JSON.stringify(items)
+    | ; var filter_field =
+    != JSON.stringify(filter_field || null)
   .mx-auto.w100.wmx11.fs-body1
-    if !items.length
+    .word-cloud(style="cursor: default")
-      .d-flex.jc-center
-        .s-notice.d-flex.ai-center.p8.gx16.pl16(role="status")
-          != icons.get("info-circle")
-          div Tag data needs to be downloaded. This will take a while.
-          button.s-btn.s-btn__outlined Download now
-
-    #content(style="cursor: default")
     script.
       setTimeout(() => {
-      const content = document.getElementById("content")
+        const content = document.querySelector(".word-cloud")
         content.style.height = `${Math.round(Math.min(content.clientWidth, window.innerHeight)*0.8)}px`
         const dark = window.matchMedia?.("(prefers-color-scheme: dark)").matches
         WordCloud(content, {
@@ -32,10 +27,11 @@ block view
             const newURL = new URL(location)
             newURL.searchParams.set("filter", highlightedItem)
             newURL.searchParams.set("filter_field", "tag")
-          newURL.searchParams.set("arrange", "label")
+            newURL.searchParams.set("arrange", filter_field ? "album" : "label")
             location = newURL
           }
         })
+        content.style.background = "none"
         content.addEventListener("wordcloudstop", () => {
           for (const child of content.children) {
             child.addEventListener("mouseenter", () => {

readme.md

@@ -16,8 +16,7 @@ but the idea is you can more easily search your whole collection and play it str
 4. go to network tab and save all as har, save as scripts/account.har
 5. npm install
 6. node scripts/populate-albums-tracks.js
-7. node scripts/load-tags.js (if you want to be able to search through album tags. takes a while to run)
+8. node start.js
-8. node server.js
 9. http://localhost:2239
 
 ## license

routes/app.js

@@ -1,10 +1,13 @@
 // @ts-check
 
 const {z} = require("zod")
-const {defineEventHandler, getQuery, getValidatedQuery, sendRedirect, createError} = require("h3")
+const {defineEventHandler, getQuery, getValidatedQuery, sendRedirect, createError, getValidatedRouterParams} = require("h3")
 const {router, db, sync, select, from} = require("../passthrough")
 const pugSync = sync.require("../pug-sync")
 
+/** @type {import("./load-tags")} */
+const loadTags = sync.require("./load-tags")
+
 const displayCurrency = "NZD"
 const displayCurrencySymbol = "$"
 const currencyExchange = new Map([
@@ -14,32 +17,38 @@ const currencyExchange = new Map([
 	["EUR", 1.08],
 	["GBP", 1.3],
 	["CAD", 0.7],
-	["NOK", 0.1]
+	["NOK", 0.1],
+	["CHF", 1.13]
 ])
 
 const sqls = {
-	album_grid:  "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY item_id {ORDER}",
+	album_grid:  "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY item_id {ORDER}",
-	album_list:  "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY item_id ORDER BY band_url, band_name COLLATE NOCASE, item_title COLLATE NOCASE",
+	album_list:  "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY item_id ORDER BY band_url, band_name COLLATE NOCASE, item_title COLLATE NOCASE",
-	artist_grid: "SELECT band_name, count(DISTINCT item_id) AS album_count, group_concat(DISTINCT band_url) AS labels, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
+	artist_grid: "SELECT band_name, count(DISTINCT item_id) AS album_count, group_concat(DISTINCT band_url) AS labels, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
-	artist_list: "SELECT band_name, count(DISTINCT item_id) AS album_count, band_url, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
+	artist_list: "SELECT band_name, count(DISTINCT item_id) AS album_count, band_url, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
-	label_grid:  "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
+	label_grid:  "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
-	label_list:  "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
+	label_list:  "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
-	tag_grid:    "SELECT tag, count(*) AS count FROM (SELECT tag, band_url, band_name, item_id, count(*) AS count FROM item_tag INNER JOIN item USING (item_id) GROUP BY tag, band_url) {WHERE} GROUP BY tag ORDER BY count DESC",
+	tag_grid:    "SELECT tag, count(*) AS count FROM item_tag INNER JOIN item USING (account, item_id) WHERE account = ? {WHERE} GROUP BY tag, band_url ORDER BY count DESC",
-	track_list:  "SELECT * FROM track INNER JOIN item USING (item_id) {JOIN TAG} {WHERE} ORDER BY band_url, item_title COLLATE NOCASE, track_number"
+	track_list:  "SELECT * FROM track INNER JOIN item USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} ORDER BY band_url, item_title COLLATE NOCASE, track_number"
 }
 
-function loadPreviews(locals, field, number, whereClause, filter_field, filter) {
+function loadPreviews(locals, field, number, whereClause, account, filter_field, filter) {
-	const params = [number]
+	const params = [account, number]
-	let sql = `SELECT ${field}, item_url, item_art_url FROM (SELECT ${field}, item_url, item_art_url, row_number() OVER (PARTITION BY ${field} ORDER BY purchased DESC) AS row_number FROM item {JOIN TAG} {WHERE}) WHERE row_number <= ?`
+	let sql = `SELECT ${field}, item_url, item_art_url FROM (SELECT ${field}, item_url, item_art_url, row_number() OVER (PARTITION BY ${field} ORDER BY purchased DESC) AS row_number FROM item {JOIN TAG} WHERE account = ? {WHERE}) WHERE row_number <= ?`
 	sql = sql.replace("{WHERE}", whereClause)
 	if (whereClause) {
-		params.unshift(filter)
+		if (filter_field === "band_url") {
+			params.splice(1, 0, `%${filter}%`)
+		} else {
+			params.splice(1, 0, filter)
+		}
 	}
 	if (filter_field === "tag" && filter) {
-		sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (item_id)")
+		sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (account, item_id)")
 	} else {
 		sql = sql.replace("{JOIN TAG}", "")
 	}
+
 	const previews = db.prepare(sql).all(params)
 	// TODO: performance?
 	for (const item of locals.items) {
@@ -52,20 +61,26 @@ function loadPreviews(locals, field, number, whereClause, filter_field, filter)
 	}
 }
 
-const schema = z.object({
+const schema = {
+	query: z.object({
 		arrange: z.enum(["album", "artist", "label", "tag", "track"]),
 		shape: z.enum(["grid", "list"]),
 		filter_field: z.enum(["band_name", "band_url", "item_id", "tag"]).optional(),
 		filter: z.string().optional()
-})
+	}),
+	params: z.object({
+		account: z.string()
+	})
+}
 
-router.get("/", defineEventHandler({
+router.get("/:account/", defineEventHandler({
 	onBeforeResponse: pugSync.compressResponse,
 	handler: async event => {
 		try {
-			var {arrange, shape, filter, filter_field} = await getValidatedQuery(event, schema.parse)
+			var {account} = await getValidatedRouterParams(event, schema.params.parse)
+			var {arrange, shape, filter, filter_field} = await getValidatedQuery(event, schema.query.parse)
 		} catch (e) {
-			return sendRedirect(event, "/?arrange=album&shape=grid", 302)
+			return sendRedirect(event, "?arrange=album&shape=grid", 302)
 		}
 		const query = getQuery(event)
 		if (arrange === "track") {
@@ -73,7 +88,7 @@ router.get("/", defineEventHandler({
 			query.shape = "list"
 		}
 		const mode = `${arrange}_${shape}`
-		const params = []
+		const params = [account]
 		let sql = sqls[mode]
 		let whereClause = ""
 		if (filter_field && filter) {
@@ -84,23 +99,33 @@ router.get("/", defineEventHandler({
 			} else {
 				params.push(filter)
 			}
-			whereClause = `WHERE ${filter_field} ${operator} ?`
+			whereClause = `AND ${filter_field} ${operator} ?`
 			sql = sql.replace("{ORDER}", "ORDER BY item_title COLLATE NOCASE")
 		} else {
 			sql = sql.replace("{ORDER}", "ORDER BY purchased DESC")
 		}
 		sql = sql.replace("{WHERE}", whereClause)
 		if (filter_field === "tag" && filter) {
-			sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (item_id)")
+			sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (account, item_id)")
 		} else {
 			sql = sql.replace("{JOIN TAG}", "")
 		}
+		if (mode === "tag_grid" && ((!filter_field || !filter) || filter_field === "tag")) {
+			sql = `SELECT tag, count(*) AS count FROM (${sql}) GROUP BY tag ORDER BY count DESC`
+		}
+		try {
 			const prepared = db.prepare(sql)
 			if (arrange === "tag") {
 				prepared.raw()
 			}
+			var items = prepared.all(params)
+		} catch (e) {
+			console.error(sql, params)
+			throw e
+		}
 		const locals = {
-			items: prepared.all(params),
+			items,
+			account,
 			query,
 			count: {
 				total: db.prepare("SELECT count(*) FROM item").pluck().get(),
@@ -123,9 +148,12 @@ router.get("/", defineEventHandler({
 			}
 		}
 		if (mode === "artist_grid") {
-			loadPreviews(locals, "band_name", 4, whereClause, filter_field, filter)
+			loadPreviews(locals, "band_name", 4, whereClause, account, filter_field, filter)
 		} else if (mode === "label_grid") {
-			loadPreviews(locals, "band_url", 6, whereClause, filter_field, filter)
+			loadPreviews(locals, "band_url", 6, whereClause, account, filter_field, filter)
+		} else if (arrange === "tag") {
+			locals.downloadManager = loadTags.downloadManager
+			locals.downloader = loadTags.downloadManager.check(account)
 		}
 		return pugSync.render(event, `${arrange}_${shape}.pug`, locals)
 	}

routes/load-tags.js

@@ -0,0 +1,99 @@
+// @ts-check
+
+const domino = require("domino")
+const {z} = require("zod")
+const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3")
+
+const {sync, db, from, router} = require("../passthrough")
+const pugSync = sync.require("../pug-sync")
+
+const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
+
+class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
+	constructor(account) {
+		super()
+		this.account = account
+		this.processed = 0
+		this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
+		this.total = this.untaggedItems.length
+		this.running = false
+		this.outcome = null
+	}
+
+	async _start() {
+		if (this.running) return
+		this.running = true
+		try {
+			for (const {account, item_id, item_title, item_url} of this.untaggedItems) {
+				const html = await fetch(item_url).then(res => res.text())
+				const doc = domino.createDocument(html)
+				// @ts-ignore
+				const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
+				db.transaction(() => {
+					for (const tag of tags) {
+						i.run(account, item_id, tag)
+					}
+				})()
+				this.processed++
+			}
+			this.outcome = "Success"
+		} catch (e) {
+			console.error(e)
+			this.outcome = e.toString()
+		} finally {
+			this.running = false
+		}
+	}
+}
+
+const downloadManager = new class {
+	/** @type {Map<string, TagDownloader>} */
+	inProgressTagDownloads = sync.remember(() => new Map())
+
+	/** @param {string} account */
+	check(account) {
+		return this.inProgressTagDownloads.get(account) || (() => {
+			const downloader = new TagDownloader(account)
+			this.inProgressTagDownloads.set(account, downloader)
+			console.log(`created downloader ${account}`)
+			setTimeout(() => {
+				this.resolve(account)
+			})
+			return downloader
+		})()
+	}
+
+	/** @param {string} account */
+	start(account) {
+		const downloader = this.check(account)
+		downloader._start()
+		return downloader
+	}
+
+	/** @param {string} account */
+	resolve(account) {
+		const downloader = this.check(account)
+		if (!downloader.running) {
+			console.log(`disposed downloader ${account}`)
+			this.inProgressTagDownloads.delete(account)
+		}
+	}
+}
+
+const schema = z.object({
+	account: z.string()
+})
+
+router.get("/api/tag-download", defineEventHandler(async event => {
+	const {account} = await getValidatedQuery(event, schema.parse)
+	const downloader = downloadManager.check(account)
+	return pugSync.render(event, "includes/tag-status.pug", {downloadManager, downloader, account})
+}))
+
+router.post("/api/tag-download", defineEventHandler(async event => {
+	const {account} = await readValidatedBody(event, schema.parse)
+	const downloader = downloadManager.start(account)
+	return pugSync.render(event, "includes/tag-status.pug", {downloadManager, downloader, account})
+}))
+
+module.exports.downloadManager = downloadManager

scripts/load-tags.js

@@ -1,30 +0,0 @@
-// @ts-check
-
-const domino = require("domino")
-const sqlite = require("better-sqlite3")
-
-const db = new sqlite("bc-explorer.db", {fileMustExist: true})
-require("../db/migrate").migrate(db)
-Object.assign(require("../passthrough"), {db})
-const {from} = require("../db/orm")
-
-const i = db.prepare("INSERT OR IGNORE INTO item_tag (item_id, tag) VALUES (?, ?)")
-
-;(async () => {
-	const untaggedItems = from("item").select("item_id", "item_title", "item_url").join("item_tag", "item_id", "left").and("WHERE tag IS NULL").all()
-	console.log(`Downloading tags for ${untaggedItems.length} purchased items`)
-	let processed = 1
-	for (const {item_id, item_title, item_url} of untaggedItems) {
-		const html = await fetch(item_url).then(res => res.text())
-		const doc = domino.createDocument(html)
-		// @ts-ignore
-		const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
-		console.log(`[${processed}/${untaggedItems.length}] tagging ${item_title} with ${tags.join(", ")}`)
-		db.transaction(() => {
-			for (const tag of tags) {
-				i.run(item_id, tag)
-			}
-		})()
-		processed++
-	}
-})()

scripts/populate-albums-tracks.js

@@ -11,7 +11,7 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
 	assert(collection_summary)
 
 	const body = JSON.parse(collection_summary.response.content.text)
-	const {fan_id} = body
+	const {fan_id, username: account} = body.collection_summary
 	const count = Object.keys(body.collection_summary.tralbum_lookup).length
 
 	const newestPurchase = Object.values(body.collection_summary.tralbum_lookup).sort((a, b) => new Date(b.purchased).getTime() - new Date(a.purchased).getTime())[0]
@@ -36,12 +36,15 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
 	const db = new sqlite("bc-explorer.db")
 	require("../db/migrate").migrate(db)
 
-	const columns = ["item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
+	db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id)
+
+	const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
 	const upsert_columns = ["added", "updated", "purchased"]
 	const preparedItem = db.prepare(`INSERT OR IGNORE INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`)
 	db.transaction(() => {
 		for (const item of items.items) {
 			preparedItem.run({
+				account,
 				...item,
 				purchased: new Date(item.purchased).getTime(),
 				added: new Date(item.added).getTime(),
@@ -52,12 +55,13 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
 	const storedItemCount = db.prepare("SELECT count(*) AS count FROM item").pluck().get()
 	console.log(`Stored ${storedItemCount} purchased items`)
 
-	const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (item_id, track_id, title, artist, track_number, duration, mp3) VALUES (@item_id, @track_id, @title, @artist, @track_number, @duration, @mp3)")
+	const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration, mp3) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration, @mp3)")
 	db.transaction(() => {
 		for (const [key, tracklist] of Object.entries(items.tracklists)) {
 			assert.match(key[0], /[at]/)
 			for (const track of tracklist) {
 				preparedTrack.run({
+					account,
 					item_id: key.slice(1),
 					track_id: track.id,
 					mp3: track.file?.["mp3-v0"],
@@ -68,5 +72,4 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
 	})()
 	const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track").pluck().get()
 	console.log(`Stored ${storedTrackCount} tracks`)
-	console.log("To load tag data, please now run node ./scripts/load-tags.js")
 })()

start.js

@@ -31,7 +31,6 @@ const icons = sync.require("./icons")
 
 pugSync.addGlobals({h3, select, icons})
 pugSync.createRoute(router, "/", "home.pug")
-pugSync.createRoute(router, "/ok", "ok.pug")
 
 // Routes