From 26ea8692852a3e0c5dcd0a37b75143908a829bb4 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 2 Apr 2025 17:03:03 +1300 Subject: [PATCH] Multiple accounts support, load tags online --- db/migrations/0001-schema.sql | 20 +++++-- db/orm-defs.d.ts | 9 +++ pug/album_grid.pug | 2 +- pug/artist_grid.pug | 2 +- pug/home.pug | 20 +++++++ pug/includes/layout.pug | 18 ++++-- pug/includes/tag-status.pug | 32 ++++++++++ pug/tag_grid.pug | 78 ++++++++++++------------ readme.md | 3 +- routes/app.js | 94 ++++++++++++++++++----------- routes/load-tags.js | 99 +++++++++++++++++++++++++++++++ scripts/load-tags.js | 30 ---------- scripts/populate-albums-tracks.js | 11 ++-- server.js => start.js | 1 - 14 files changed, 295 insertions(+), 124 deletions(-) create mode 100644 pug/home.pug create mode 100644 pug/includes/tag-status.pug create mode 100644 routes/load-tags.js delete mode 100644 scripts/load-tags.js rename server.js => start.js (97%) diff --git a/db/migrations/0001-schema.sql b/db/migrations/0001-schema.sql index 87c3321..f124651 100644 --- a/db/migrations/0001-schema.sql +++ b/db/migrations/0001-schema.sql @@ -1,6 +1,14 @@ BEGIN TRANSACTION; +CREATE TABLE account ( + account TEXT NOT NULL, + fan_id INTEGER NOT NULL, + currency TEXT, + PRIMARY KEY (account) +) WITHOUT ROWID; + CREATE TABLE item ( + account TEXT NOT NULL, item_id INTEGER NOT NULL, item_type TEXT NOT NULL, band_id INTEGER NOT NULL, @@ -24,10 +32,11 @@ CREATE TABLE item ( currency STRING NOT NULL, label TEXT, label_id INTEGER, - PRIMARY KEY (item_id) + PRIMARY KEY (account, item_id) ) WITHOUT ROWID; CREATE TABLE track ( + account TEXT NOT NULL, item_id INTEGER NOT NULL, track_id INTEGER NOT NULL, title TEXT NOT NULL, @@ -35,15 +44,16 @@ CREATE TABLE track ( track_number INTEGER, duration NUMERIC NOT NULL, mp3 TEXT, - PRIMARY KEY (item_id, track_id), - FOREIGN KEY (item_id) REFERENCES item (item_id) ON DELETE CASCADE + PRIMARY KEY (account, item_id, track_id), + FOREIGN KEY (account, item_id) REFERENCES item (account, item_id) ON DELETE CASCADE ) WITHOUT ROWID; CREATE TABLE item_tag ( + account TEXT NOT NULL, item_id INTEGER NOT NULL, tag TEXT NOT NULL, - PRIMARY KEY (item_id, tag), - FOREIGN KEY (item_id) REFERENCES item (item_id) ON DELETE CASCADE + PRIMARY KEY (account, item_id, tag), + FOREIGN KEY (account, item_id) REFERENCES item (account, item_id) ON DELETE CASCADE ) WITHOUT ROWID; COMMIT; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index dccd6df..6943f26 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -1,5 +1,12 @@ export type Models = { + account: { + account: string + fan_id: number + currency: string | null + } + item: { + account: string item_id: number item_type: string band_id: number @@ -26,6 +33,7 @@ export type Models = { } track: { + account: string item_id: number track_id: number title: string @@ -36,6 +44,7 @@ export type Models = { } item_tag: { + account: string item_id: number tag: string } diff --git a/pug/album_grid.pug b/pug/album_grid.pug index 14cedb2..cc6c64e 100644 --- a/pug/album_grid.pug +++ b/pug/album_grid.pug @@ -20,5 +20,5 @@ block view = item.band_name - let label = item.band_url.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1") a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label})) - span.s-tag--sponsor!= icons.get("component", 16) + span.s-tag--sponsor!= icons.get("flower", 16) = label diff --git a/pug/artist_grid.pug b/pug/artist_grid.pug index d768636..816f065 100644 --- a/pug/artist_grid.pug +++ b/pug/artist_grid.pug @@ -20,7 +20,7 @@ block view - let labels = item.labels.split("|").map(x => x.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1")) each label in labels a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label})) - span.s-tag--sponsor!= icons.get("component", 16) + span.s-tag--sponsor!= icons.get("flower", 16) = label each preview in item.previews a.d-flex(href=preview.item_url) diff --git a/pug/home.pug b/pug/home.pug new file mode 100644 index 0000000..41c78b6 --- /dev/null +++ b/pug/home.pug @@ -0,0 +1,20 @@ +doctype html +html + head + meta(charset="utf-8") + + title BC Explorer + link(rel="stylesheet" type="text/css" href="/static/stacks.min.css") + body.themed.theme-system.overflow-y-scroll + header.s-topbar.ps-sticky.t0 + .s-topbar--container.wmx9 + .s-topbar--logo + != icons.get("compass-solid", 24) + .ml4 BC Explorer + + .mx-auto.wmx9.py24.px16.g24.s-prose + h1 Select profile + - const names = select("account", "account").pluck().all() + ul + each name in names + li: a(href=`/${name}/`)= name diff --git a/pug/includes/layout.pug b/pug/includes/layout.pug index 6285c08..013f458 100644 --- a/pug/includes/layout.pug +++ b/pug/includes/layout.pug @@ -63,14 +63,15 @@ html ul.s-navigation li: +navi("arrange", "album", "album", "Album") li: +navi("arrange", "artist", "people-tag", "Artist") - li: +navi("arrange", "label", "component", "Label") + li: +navi("arrange", "label", "flower", "Label") + //- asana, flower, component, circle-spark, rhombus, sphere, union-alt, color-wheel, community, combine li: +navi("arrange", "tag", "label", "Tag") li: +navi("arrange", "track", "music-note", "Track") .px16 nav ul.s-navigation.s-navigation__toggle.g0 li: +navi("shape", "grid").brr0!= icons.get("view-grid") - li: +navi("shape", "list").blr0!= icons.get("table-rows") + li: +navi("shape", "list").blr0!= icons.get("align-justify") .fl-grow1 #player(hx-preserve) button.s-btn.s-btn__outlined.s-btn__xs!= icons.get("play") @@ -80,12 +81,17 @@ html block view div - .ps-sticky.d-flex.fd-column.g12(style="top: 80px") + .ps-sticky.d-flex.fd-column.g12.wmx4(style="top: 80px") + if arrange === "tag" + include tag-status.pug if filter .s-sidebarwidget.s-sidebarwidget__blue.d-flex.ai-center.gx16.jc-space-between.p8.pl16 - != icons.get("search") - div Searching for #[strong= filter] - .fl-grow1 + .fl-shrink0!= icons.get("search") + .fl-grow1= `Searching for ` + if filter_field === "item_id" + strong= items[0].item_title + else + strong= filter a.s-btn.s-notice--btn(href=and({filter: null, filter_field: null})) Clear .s-sidebarwidget.wmn3 .s-sidebarwidget--header Collection diff --git a/pug/includes/tag-status.pug b/pug/includes/tag-status.pug new file mode 100644 index 0000000..2d2f8a6 --- /dev/null +++ b/pug/includes/tag-status.pug @@ -0,0 +1,32 @@ +if downloader.total > 0 + .d-flex.jc-center#tag-download + if !downloader.running && !downloader.outcome + form.s-notice.s-notice__info.d-flex.ai-center.p8.gx16.pl16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-post="/api/tag-download") + != icons.get("info-circle") + div Tag data needs to be downloaded. This will take a while. + input(type="hidden" name="account" value=account) + button.s-btn.s-btn__outlined Download now + + else if !downloader.outcome + .s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s") + .d-flex.gx16.ai-center + != icons.get("cloud-download") + div Downloading tags... + p.mt12 You can keep using BC Explorer while this continues in the background. + .s-progress.mt16 + .s-progress--bar(style=`width: ${downloader.processed/downloader.total*100}%`) + .d-flex.jc-space-between.fs-fine + span= downloader.processed + span= downloader.total + + else if downloader.outcome === "Success" + .s-notice.s-notice__success.p8.gx16.pl16.d-flex.ai-center.wmn3 + != icons.get("cloud-check") + .fl-grow1 Tags downloaded. + - downloadManager.resolve(account) + a.s-btn.s-btn__outlined(href="") Refresh + + else + .s-notice.s-notice__danger.p8.gx16.pl16.d-flex.ai-center.wmn3 + != icons.get("cloud-xmark") + .fl-grow1= downloader.outcome diff --git a/pug/tag_grid.pug b/pug/tag_grid.pug index 5583f21..d21e482 100644 --- a/pug/tag_grid.pug +++ b/pug/tag_grid.pug @@ -4,46 +4,42 @@ block view script | var items = != JSON.stringify(items) + | ; var filter_field = + != JSON.stringify(filter_field || null) .mx-auto.w100.wmx11.fs-body1 - if !items.length - .d-flex.jc-center - .s-notice.d-flex.ai-center.p8.gx16.pl16(role="status") - != icons.get("info-circle") - div Tag data needs to be downloaded. This will take a while. - button.s-btn.s-btn__outlined Download now - - #content(style="cursor: default") - script. - setTimeout(() => { - const content = document.getElementById("content") - content.style.height = `${Math.round(Math.min(content.clientWidth, window.innerHeight)*0.8)}px` - const dark = window.matchMedia?.("(prefers-color-scheme: dark)").matches - WordCloud(content, { - list: items, - fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI Adjusted", "Segoe UI", "Liberation Sans", sans-serif', - fontWeight: "bold", - color: dark ? "random-light": "random-dark", - minSize: 4, - gridSize: Math.round(content.clientWidth / 200), - weightFactor: size => size * content.clientWidth / 180, - rotateRatio: 0, - click: item => { - const highlightedItem = document.querySelector(".wc-hl")?.textContent || item[0] - const newURL = new URL(location) - newURL.searchParams.set("filter", highlightedItem) - newURL.searchParams.set("filter_field", "tag") - newURL.searchParams.set("arrange", "label") - location = newURL - } + .word-cloud(style="cursor: default") + script. + setTimeout(() => { + const content = document.querySelector(".word-cloud") + content.style.height = `${Math.round(Math.min(content.clientWidth, window.innerHeight)*0.8)}px` + const dark = window.matchMedia?.("(prefers-color-scheme: dark)").matches + WordCloud(content, { + list: items, + fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI Adjusted", "Segoe UI", "Liberation Sans", sans-serif', + fontWeight: "bold", + color: dark ? "random-light": "random-dark", + minSize: 4, + gridSize: Math.round(content.clientWidth / 200), + weightFactor: size => size * content.clientWidth / 180, + rotateRatio: 0, + click: item => { + const highlightedItem = document.querySelector(".wc-hl")?.textContent || item[0] + const newURL = new URL(location) + newURL.searchParams.set("filter", highlightedItem) + newURL.searchParams.set("filter_field", "tag") + newURL.searchParams.set("arrange", filter_field ? "album" : "label") + location = newURL + } + }) + content.style.background = "none" + content.addEventListener("wordcloudstop", () => { + for (const child of content.children) { + child.addEventListener("mouseenter", () => { + child.classList.add("wc-hl") + }) + child.addEventListener("mouseleave", () => { + child.classList.remove("wc-hl") + }) + } + }) }) - content.addEventListener("wordcloudstop", () => { - for (const child of content.children) { - child.addEventListener("mouseenter", () => { - child.classList.add("wc-hl") - }) - child.addEventListener("mouseleave", () => { - child.classList.remove("wc-hl") - }) - } - }) - }) diff --git a/readme.md b/readme.md index b4a2720..e6c4e27 100644 --- a/readme.md +++ b/readme.md @@ -16,8 +16,7 @@ but the idea is you can more easily search your whole collection and play it str 4. go to network tab and save all as har, save as scripts/account.har 5. npm install 6. node scripts/populate-albums-tracks.js -7. node scripts/load-tags.js (if you want to be able to search through album tags. takes a while to run) -8. node server.js +8. node start.js 9. http://localhost:2239 ## license diff --git a/routes/app.js b/routes/app.js index f05bc2a..95dcdd5 100644 --- a/routes/app.js +++ b/routes/app.js @@ -1,10 +1,13 @@ // @ts-check const {z} = require("zod") -const {defineEventHandler, getQuery, getValidatedQuery, sendRedirect, createError} = require("h3") +const {defineEventHandler, getQuery, getValidatedQuery, sendRedirect, createError, getValidatedRouterParams} = require("h3") const {router, db, sync, select, from} = require("../passthrough") const pugSync = sync.require("../pug-sync") +/** @type {import("./load-tags")} */ +const loadTags = sync.require("./load-tags") + const displayCurrency = "NZD" const displayCurrencySymbol = "$" const currencyExchange = new Map([ @@ -14,32 +17,38 @@ const currencyExchange = new Map([ ["EUR", 1.08], ["GBP", 1.3], ["CAD", 0.7], - ["NOK", 0.1] + ["NOK", 0.1], + ["CHF", 1.13] ]) const sqls = { - album_grid: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY item_id {ORDER}", - album_list: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY item_id ORDER BY band_url, band_name COLLATE NOCASE, item_title COLLATE NOCASE", - artist_grid: "SELECT band_name, count(DISTINCT item_id) AS album_count, group_concat(DISTINCT band_url) AS labels, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE", - artist_list: "SELECT band_name, count(DISTINCT item_id) AS album_count, band_url, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE", - label_grid: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE", - label_list: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE", - tag_grid: "SELECT tag, count(*) AS count FROM (SELECT tag, band_url, band_name, item_id, count(*) AS count FROM item_tag INNER JOIN item USING (item_id) GROUP BY tag, band_url) {WHERE} GROUP BY tag ORDER BY count DESC", - track_list: "SELECT * FROM track INNER JOIN item USING (item_id) {JOIN TAG} {WHERE} ORDER BY band_url, item_title COLLATE NOCASE, track_number" + album_grid: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY item_id {ORDER}", + album_list: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY item_id ORDER BY band_url, band_name COLLATE NOCASE, item_title COLLATE NOCASE", + artist_grid: "SELECT band_name, count(DISTINCT item_id) AS album_count, group_concat(DISTINCT band_url) AS labels, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE", + artist_list: "SELECT band_name, count(DISTINCT item_id) AS album_count, band_url, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE", + label_grid: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE", + label_list: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE", + tag_grid: "SELECT tag, count(*) AS count FROM item_tag INNER JOIN item USING (account, item_id) WHERE account = ? {WHERE} GROUP BY tag, band_url ORDER BY count DESC", + track_list: "SELECT * FROM track INNER JOIN item USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} ORDER BY band_url, item_title COLLATE NOCASE, track_number" } -function loadPreviews(locals, field, number, whereClause, filter_field, filter) { - const params = [number] - let sql = `SELECT ${field}, item_url, item_art_url FROM (SELECT ${field}, item_url, item_art_url, row_number() OVER (PARTITION BY ${field} ORDER BY purchased DESC) AS row_number FROM item {JOIN TAG} {WHERE}) WHERE row_number <= ?` +function loadPreviews(locals, field, number, whereClause, account, filter_field, filter) { + const params = [account, number] + let sql = `SELECT ${field}, item_url, item_art_url FROM (SELECT ${field}, item_url, item_art_url, row_number() OVER (PARTITION BY ${field} ORDER BY purchased DESC) AS row_number FROM item {JOIN TAG} WHERE account = ? {WHERE}) WHERE row_number <= ?` sql = sql.replace("{WHERE}", whereClause) if (whereClause) { - params.unshift(filter) + if (filter_field === "band_url") { + params.splice(1, 0, `%${filter}%`) + } else { + params.splice(1, 0, filter) + } } if (filter_field === "tag" && filter) { - sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (item_id)") + sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (account, item_id)") } else { sql = sql.replace("{JOIN TAG}", "") } + const previews = db.prepare(sql).all(params) // TODO: performance? for (const item of locals.items) { @@ -52,20 +61,26 @@ function loadPreviews(locals, field, number, whereClause, filter_field, filter) } } -const schema = z.object({ - arrange: z.enum(["album", "artist", "label", "tag", "track"]), - shape: z.enum(["grid", "list"]), - filter_field: z.enum(["band_name", "band_url", "item_id", "tag"]).optional(), - filter: z.string().optional() -}) +const schema = { + query: z.object({ + arrange: z.enum(["album", "artist", "label", "tag", "track"]), + shape: z.enum(["grid", "list"]), + filter_field: z.enum(["band_name", "band_url", "item_id", "tag"]).optional(), + filter: z.string().optional() + }), + params: z.object({ + account: z.string() + }) +} -router.get("/", defineEventHandler({ +router.get("/:account/", defineEventHandler({ onBeforeResponse: pugSync.compressResponse, handler: async event => { try { - var {arrange, shape, filter, filter_field} = await getValidatedQuery(event, schema.parse) + var {account} = await getValidatedRouterParams(event, schema.params.parse) + var {arrange, shape, filter, filter_field} = await getValidatedQuery(event, schema.query.parse) } catch (e) { - return sendRedirect(event, "/?arrange=album&shape=grid", 302) + return sendRedirect(event, "?arrange=album&shape=grid", 302) } const query = getQuery(event) if (arrange === "track") { @@ -73,7 +88,7 @@ router.get("/", defineEventHandler({ query.shape = "list" } const mode = `${arrange}_${shape}` - const params = [] + const params = [account] let sql = sqls[mode] let whereClause = "" if (filter_field && filter) { @@ -84,23 +99,33 @@ router.get("/", defineEventHandler({ } else { params.push(filter) } - whereClause = `WHERE ${filter_field} ${operator} ?` + whereClause = `AND ${filter_field} ${operator} ?` sql = sql.replace("{ORDER}", "ORDER BY item_title COLLATE NOCASE") } else { sql = sql.replace("{ORDER}", "ORDER BY purchased DESC") } sql = sql.replace("{WHERE}", whereClause) if (filter_field === "tag" && filter) { - sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (item_id)") + sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (account, item_id)") } else { sql = sql.replace("{JOIN TAG}", "") } - const prepared = db.prepare(sql) - if (arrange === "tag") { - prepared.raw() + if (mode === "tag_grid" && ((!filter_field || !filter) || filter_field === "tag")) { + sql = `SELECT tag, count(*) AS count FROM (${sql}) GROUP BY tag ORDER BY count DESC` + } + try { + const prepared = db.prepare(sql) + if (arrange === "tag") { + prepared.raw() + } + var items = prepared.all(params) + } catch (e) { + console.error(sql, params) + throw e } const locals = { - items: prepared.all(params), + items, + account, query, count: { total: db.prepare("SELECT count(*) FROM item").pluck().get(), @@ -123,9 +148,12 @@ router.get("/", defineEventHandler({ } } if (mode === "artist_grid") { - loadPreviews(locals, "band_name", 4, whereClause, filter_field, filter) + loadPreviews(locals, "band_name", 4, whereClause, account, filter_field, filter) } else if (mode === "label_grid") { - loadPreviews(locals, "band_url", 6, whereClause, filter_field, filter) + loadPreviews(locals, "band_url", 6, whereClause, account, filter_field, filter) + } else if (arrange === "tag") { + locals.downloadManager = loadTags.downloadManager + locals.downloader = loadTags.downloadManager.check(account) } return pugSync.render(event, `${arrange}_${shape}.pug`, locals) } diff --git a/routes/load-tags.js b/routes/load-tags.js new file mode 100644 index 0000000..2079e49 --- /dev/null +++ b/routes/load-tags.js @@ -0,0 +1,99 @@ +// @ts-check + +const domino = require("domino") +const {z} = require("zod") +const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3") + +const {sync, db, from, router} = require("../passthrough") +const pugSync = sync.require("../pug-sync") + +const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)") + +class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) { + constructor(account) { + super() + this.account = account + this.processed = 0 + this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account) + this.total = this.untaggedItems.length + this.running = false + this.outcome = null + } + + async _start() { + if (this.running) return + this.running = true + try { + for (const {account, item_id, item_title, item_url} of this.untaggedItems) { + const html = await fetch(item_url).then(res => res.text()) + const doc = domino.createDocument(html) + // @ts-ignore + const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent) + db.transaction(() => { + for (const tag of tags) { + i.run(account, item_id, tag) + } + })() + this.processed++ + } + this.outcome = "Success" + } catch (e) { + console.error(e) + this.outcome = e.toString() + } finally { + this.running = false + } + } +} + +const downloadManager = new class { + /** @type {Map} */ + inProgressTagDownloads = sync.remember(() => new Map()) + + /** @param {string} account */ + check(account) { + return this.inProgressTagDownloads.get(account) || (() => { + const downloader = new TagDownloader(account) + this.inProgressTagDownloads.set(account, downloader) + console.log(`created downloader ${account}`) + setTimeout(() => { + this.resolve(account) + }) + return downloader + })() + } + + /** @param {string} account */ + start(account) { + const downloader = this.check(account) + downloader._start() + return downloader + } + + /** @param {string} account */ + resolve(account) { + const downloader = this.check(account) + if (!downloader.running) { + console.log(`disposed downloader ${account}`) + this.inProgressTagDownloads.delete(account) + } + } +} + +const schema = z.object({ + account: z.string() +}) + +router.get("/api/tag-download", defineEventHandler(async event => { + const {account} = await getValidatedQuery(event, schema.parse) + const downloader = downloadManager.check(account) + return pugSync.render(event, "includes/tag-status.pug", {downloadManager, downloader, account}) +})) + +router.post("/api/tag-download", defineEventHandler(async event => { + const {account} = await readValidatedBody(event, schema.parse) + const downloader = downloadManager.start(account) + return pugSync.render(event, "includes/tag-status.pug", {downloadManager, downloader, account}) +})) + +module.exports.downloadManager = downloadManager diff --git a/scripts/load-tags.js b/scripts/load-tags.js deleted file mode 100644 index 8a18b90..0000000 --- a/scripts/load-tags.js +++ /dev/null @@ -1,30 +0,0 @@ -// @ts-check - -const domino = require("domino") -const sqlite = require("better-sqlite3") - -const db = new sqlite("bc-explorer.db", {fileMustExist: true}) -require("../db/migrate").migrate(db) -Object.assign(require("../passthrough"), {db}) -const {from} = require("../db/orm") - -const i = db.prepare("INSERT OR IGNORE INTO item_tag (item_id, tag) VALUES (?, ?)") - -;(async () => { - const untaggedItems = from("item").select("item_id", "item_title", "item_url").join("item_tag", "item_id", "left").and("WHERE tag IS NULL").all() - console.log(`Downloading tags for ${untaggedItems.length} purchased items`) - let processed = 1 - for (const {item_id, item_title, item_url} of untaggedItems) { - const html = await fetch(item_url).then(res => res.text()) - const doc = domino.createDocument(html) - // @ts-ignore - const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent) - console.log(`[${processed}/${untaggedItems.length}] tagging ${item_title} with ${tags.join(", ")}`) - db.transaction(() => { - for (const tag of tags) { - i.run(item_id, tag) - } - })() - processed++ - } -})() diff --git a/scripts/populate-albums-tracks.js b/scripts/populate-albums-tracks.js index ee74143..8fe67e4 100644 --- a/scripts/populate-albums-tracks.js +++ b/scripts/populate-albums-tracks.js @@ -11,7 +11,7 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8")) assert(collection_summary) const body = JSON.parse(collection_summary.response.content.text) - const {fan_id} = body + const {fan_id, username: account} = body.collection_summary const count = Object.keys(body.collection_summary.tralbum_lookup).length const newestPurchase = Object.values(body.collection_summary.tralbum_lookup).sort((a, b) => new Date(b.purchased).getTime() - new Date(a.purchased).getTime())[0] @@ -36,12 +36,15 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8")) const db = new sqlite("bc-explorer.db") require("../db/migrate").migrate(db) - const columns = ["item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"] + db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id) + + const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"] const upsert_columns = ["added", "updated", "purchased"] const preparedItem = db.prepare(`INSERT OR IGNORE INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`) db.transaction(() => { for (const item of items.items) { preparedItem.run({ + account, ...item, purchased: new Date(item.purchased).getTime(), added: new Date(item.added).getTime(), @@ -52,12 +55,13 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8")) const storedItemCount = db.prepare("SELECT count(*) AS count FROM item").pluck().get() console.log(`Stored ${storedItemCount} purchased items`) - const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (item_id, track_id, title, artist, track_number, duration, mp3) VALUES (@item_id, @track_id, @title, @artist, @track_number, @duration, @mp3)") + const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration, mp3) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration, @mp3)") db.transaction(() => { for (const [key, tracklist] of Object.entries(items.tracklists)) { assert.match(key[0], /[at]/) for (const track of tracklist) { preparedTrack.run({ + account, item_id: key.slice(1), track_id: track.id, mp3: track.file?.["mp3-v0"], @@ -68,5 +72,4 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8")) })() const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track").pluck().get() console.log(`Stored ${storedTrackCount} tracks`) - console.log("To load tag data, please now run node ./scripts/load-tags.js") })() diff --git a/server.js b/start.js similarity index 97% rename from server.js rename to start.js index 5a8f072..8ad1467 100644 --- a/server.js +++ b/start.js @@ -31,7 +31,6 @@ const icons = sync.require("./icons") pugSync.addGlobals({h3, select, icons}) pugSync.createRoute(router, "/", "home.pug") -pugSync.createRoute(router, "/ok", "ok.pug") // Routes