cadence/bc-explorer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Multiple accounts support, load tags online

Browse source
This commit is contained in:
Cadence Ember 2025-04-02 17:03:03 +13:00
parent cd2827791f
commit 26ea869285
14 changed files with 295 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

20
db/migrations/0001-schema.sql
View file

@ -1,6 +1,14 @@
BEGIN TRANSACTION;
CREATE TABLE account (
account TEXT NOT NULL,
fan_id INTEGER NOT NULL,
currency TEXT,
PRIMARY KEY (account)
) WITHOUT ROWID;
CREATE TABLE item (
account TEXT NOT NULL,
item_id INTEGER NOT NULL,
item_type TEXT NOT NULL,
band_id INTEGER NOT NULL,
@ -24,10 +32,11 @@ CREATE TABLE item (
currency STRING NOT NULL,
label TEXT,
label_id INTEGER,
PRIMARY KEY (item_id)
PRIMARY KEY (account, item_id)
) WITHOUT ROWID;
CREATE TABLE track (
account TEXT NOT NULL,
item_id INTEGER NOT NULL,
track_id INTEGER NOT NULL,
title TEXT NOT NULL,
@ -35,15 +44,16 @@ CREATE TABLE track (
track_number INTEGER,
duration NUMERIC NOT NULL,
mp3 TEXT,
PRIMARY KEY (item_id, track_id),
FOREIGN KEY (item_id) REFERENCES item (item_id) ON DELETE CASCADE
PRIMARY KEY (account, item_id, track_id),
FOREIGN KEY (account, item_id) REFERENCES item (account, item_id) ON DELETE CASCADE
) WITHOUT ROWID;
CREATE TABLE item_tag (
account TEXT NOT NULL,
item_id INTEGER NOT NULL,
tag TEXT NOT NULL,
PRIMARY KEY (item_id, tag),
FOREIGN KEY (item_id) REFERENCES item (item_id) ON DELETE CASCADE
PRIMARY KEY (account, item_id, tag),
FOREIGN KEY (account, item_id) REFERENCES item (account, item_id) ON DELETE CASCADE
) WITHOUT ROWID;
COMMIT;

9
db/orm-defs.d.ts vendored
View file

@ -1,5 +1,12 @@
export type Models = {
account: {
account: string
fan_id: number
currency: string | null
}
item: {
account: string
item_id: number
item_type: string
band_id: number
@ -26,6 +33,7 @@ export type Models = {
}
track: {
account: string
item_id: number
track_id: number
title: string
@ -36,6 +44,7 @@ export type Models = {
}
item_tag: {
account: string
item_id: number
tag: string
}

2
pug/album_grid.pug
View file

@ -20,5 +20,5 @@ block view
= item.band_name
- let label = item.band_url.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1")
a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label}))
span.s-tag--sponsor!= icons.get("component", 16)
span.s-tag--sponsor!= icons.get("flower", 16)
= label

2
pug/artist_grid.pug
View file

@ -20,7 +20,7 @@ block view
- let labels = item.labels.split("|").map(x => x.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1"))
each label in labels
a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label}))
span.s-tag--sponsor!= icons.get("component", 16)
span.s-tag--sponsor!= icons.get("flower", 16)
= label
each preview in item.previews
a.d-flex(href=preview.item_url)

20
pug/home.pug Normal file
View file

@ -0,0 +1,20 @@
doctype html
html
head
meta(charset="utf-8")
<meta name="viewport" content="width=device-width, initial-scale=1" />
title BC Explorer
link(rel="stylesheet" type="text/css" href="/static/stacks.min.css")
body.themed.theme-system.overflow-y-scroll
header.s-topbar.ps-sticky.t0
.s-topbar--container.wmx9
.s-topbar--logo
!= icons.get("compass-solid", 24)
.ml4 BC Explorer
.mx-auto.wmx9.py24.px16.g24.s-prose
h1 Select profile
- const names = select("account", "account").pluck().all()
ul
each name in names
li: a(href=`/${name}/`)= name

18
pug/includes/layout.pug
View file

@ -63,14 +63,15 @@ html
ul.s-navigation
li: +navi("arrange", "album", "album", "Album")
li: +navi("arrange", "artist", "people-tag", "Artist")
li: +navi("arrange", "label", "component", "Label")
li: +navi("arrange", "label", "flower", "Label")
//- asana, flower, component, circle-spark, rhombus, sphere, union-alt, color-wheel, community, combine
li: +navi("arrange", "tag", "label", "Tag")
li: +navi("arrange", "track", "music-note", "Track")
.px16
nav
ul.s-navigation.s-navigation__toggle.g0
li: +navi("shape", "grid").brr0!= icons.get("view-grid")
li: +navi("shape", "list").blr0!= icons.get("table-rows")
li: +navi("shape", "list").blr0!= icons.get("align-justify")
.fl-grow1
#player(hx-preserve)
button.s-btn.s-btn__outlined.s-btn__xs!= icons.get("play")
@ -80,12 +81,17 @@ html
block view
div
.ps-sticky.d-flex.fd-column.g12(style="top: 80px")
.ps-sticky.d-flex.fd-column.g12.wmx4(style="top: 80px")
if arrange === "tag"
include tag-status.pug
if filter
.s-sidebarwidget.s-sidebarwidget__blue.d-flex.ai-center.gx16.jc-space-between.p8.pl16
!= icons.get("search")
div Searching for #[strong= filter]
.fl-grow1
.fl-shrink0!= icons.get("search")
.fl-grow1= `Searching for `
if filter_field === "item_id"
strong= items[0].item_title
else
strong= filter
a.s-btn.s-notice--btn(href=and({filter: null, filter_field: null})) Clear
.s-sidebarwidget.wmn3
.s-sidebarwidget--header Collection

32
pug/includes/tag-status.pug Normal file
View file

@ -0,0 +1,32 @@
if downloader.total > 0
.d-flex.jc-center#tag-download
if !downloader.running && !downloader.outcome
form.s-notice.s-notice__info.d-flex.ai-center.p8.gx16.pl16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-post="/api/tag-download")
!= icons.get("info-circle")
div Tag data needs to be downloaded. This will take a while.
input(type="hidden" name="account" value=account)
button.s-btn.s-btn__outlined Download now
else if !downloader.outcome
.s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s")
.d-flex.gx16.ai-center
!= icons.get("cloud-download")
div Downloading tags...
p.mt12 You can keep using BC Explorer while this continues in the background.
.s-progress.mt16
.s-progress--bar(style=`width: ${downloader.processed/downloader.total*100}%`)
.d-flex.jc-space-between.fs-fine
span= downloader.processed
span= downloader.total
else if downloader.outcome === "Success"
.s-notice.s-notice__success.p8.gx16.pl16.d-flex.ai-center.wmn3
!= icons.get("cloud-check")
.fl-grow1 Tags downloaded.
- downloadManager.resolve(account)
a.s-btn.s-btn__outlined(href="") Refresh
else
.s-notice.s-notice__danger.p8.gx16.pl16.d-flex.ai-center.wmn3
!= icons.get("cloud-xmark")
.fl-grow1= downloader.outcome

78
pug/tag_grid.pug
View file

@ -4,46 +4,42 @@ block view
script
| var items =
!= JSON.stringify(items)
| ; var filter_field =
!= JSON.stringify(filter_field || null)
.mx-auto.w100.wmx11.fs-body1
if !items.length
.d-flex.jc-center
.s-notice.d-flex.ai-center.p8.gx16.pl16(role="status")
!= icons.get("info-circle")
div Tag data needs to be downloaded. This will take a while.
button.s-btn.s-btn__outlined Download now
#content(style="cursor: default")
script.
setTimeout(() => {
const content = document.getElementById("content")
content.style.height = `${Math.round(Math.min(content.clientWidth, window.innerHeight)*0.8)}px`
const dark = window.matchMedia?.("(prefers-color-scheme: dark)").matches
WordCloud(content, {
list: items,
fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI Adjusted", "Segoe UI", "Liberation Sans", sans-serif',
fontWeight: "bold",
color: dark ? "random-light": "random-dark",
minSize: 4,
gridSize: Math.round(content.clientWidth / 200),
weightFactor: size => size * content.clientWidth / 180,
rotateRatio: 0,
click: item => {
const highlightedItem = document.querySelector(".wc-hl")?.textContent || item[0]
const newURL = new URL(location)
newURL.searchParams.set("filter", highlightedItem)
newURL.searchParams.set("filter_field", "tag")
newURL.searchParams.set("arrange", "label")
location = newURL
}
.word-cloud(style="cursor: default")
script.
setTimeout(() => {
const content = document.querySelector(".word-cloud")
content.style.height = `${Math.round(Math.min(content.clientWidth, window.innerHeight)*0.8)}px`
const dark = window.matchMedia?.("(prefers-color-scheme: dark)").matches
WordCloud(content, {
list: items,
fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI Adjusted", "Segoe UI", "Liberation Sans", sans-serif',
fontWeight: "bold",
color: dark ? "random-light": "random-dark",
minSize: 4,
gridSize: Math.round(content.clientWidth / 200),
weightFactor: size => size * content.clientWidth / 180,
rotateRatio: 0,
click: item => {
const highlightedItem = document.querySelector(".wc-hl")?.textContent || item[0]
const newURL = new URL(location)
newURL.searchParams.set("filter", highlightedItem)
newURL.searchParams.set("filter_field", "tag")
newURL.searchParams.set("arrange", filter_field ? "album" : "label")
location = newURL
}
})
content.style.background = "none"
content.addEventListener("wordcloudstop", () => {
for (const child of content.children) {
child.addEventListener("mouseenter", () => {
child.classList.add("wc-hl")
})
child.addEventListener("mouseleave", () => {
child.classList.remove("wc-hl")
})
}
})
})
content.addEventListener("wordcloudstop", () => {
for (const child of content.children) {
child.addEventListener("mouseenter", () => {
child.classList.add("wc-hl")
})
child.addEventListener("mouseleave", () => {
child.classList.remove("wc-hl")
})
}
})
})

3
readme.md
View file

@ -16,8 +16,7 @@ but the idea is you can more easily search your whole collection and play it str
4. go to network tab and save all as har, save as scripts/account.har
5. npm install
6. node scripts/populate-albums-tracks.js
7. node scripts/load-tags.js (if you want to be able to search through album tags. takes a while to run)
8. node server.js
8. node start.js
9. http://localhost:2239
## license

94
routes/app.js
View file

@ -1,10 +1,13 @@
// @ts-check
const {z} = require("zod")
const {defineEventHandler, getQuery, getValidatedQuery, sendRedirect, createError} = require("h3")
const {defineEventHandler, getQuery, getValidatedQuery, sendRedirect, createError, getValidatedRouterParams} = require("h3")
const {router, db, sync, select, from} = require("../passthrough")
const pugSync = sync.require("../pug-sync")
/** @type {import("./load-tags")} */
const loadTags = sync.require("./load-tags")
const displayCurrency = "NZD"
const displayCurrencySymbol = "$"
const currencyExchange = new Map([
@ -14,32 +17,38 @@ const currencyExchange = new Map([
["EUR", 1.08],
["GBP", 1.3],
["CAD", 0.7],
["NOK", 0.1]
["NOK", 0.1],
["CHF", 1.13]
])
const sqls = {
album_grid: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY item_id {ORDER}",
album_list: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY item_id ORDER BY band_url, band_name COLLATE NOCASE, item_title COLLATE NOCASE",
artist_grid: "SELECT band_name, count(DISTINCT item_id) AS album_count, group_concat(DISTINCT band_url) AS labels, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
artist_list: "SELECT band_name, count(DISTINCT item_id) AS album_count, band_url, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
label_grid: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
label_list: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (item_id) {JOIN TAG} {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
tag_grid: "SELECT tag, count(*) AS count FROM (SELECT tag, band_url, band_name, item_id, count(*) AS count FROM item_tag INNER JOIN item USING (item_id) GROUP BY tag, band_url) {WHERE} GROUP BY tag ORDER BY count DESC",
track_list: "SELECT * FROM track INNER JOIN item USING (item_id) {JOIN TAG} {WHERE} ORDER BY band_url, item_title COLLATE NOCASE, track_number"
album_grid: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY item_id {ORDER}",
album_list: "SELECT item.*, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY item_id ORDER BY band_url, band_name COLLATE NOCASE, item_title COLLATE NOCASE",
artist_grid: "SELECT band_name, count(DISTINCT item_id) AS album_count, group_concat(DISTINCT band_url) AS labels, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
artist_list: "SELECT band_name, count(DISTINCT item_id) AS album_count, band_url, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_name ORDER BY band_name COLLATE NOCASE",
label_grid: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
label_list: "SELECT iif(count(DISTINCT band_name) = 1, band_name, band_url) AS display_name, band_url, count(DISTINCT item_id) AS album_count, count(DISTINCT band_name) AS artist_count, count(*) AS track_count, iif(sum(duration) > 3600, cast(total(duration)/3600 AS INTEGER) || 'h ' || cast(total(duration)/60%60 AS INTEGER) || 'm', cast(total(duration)/60 AS INTEGER) || 'm') AS total_duration FROM item INNER JOIN track USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} GROUP BY band_url ORDER BY display_name COLLATE NOCASE",
tag_grid: "SELECT tag, count(*) AS count FROM item_tag INNER JOIN item USING (account, item_id) WHERE account = ? {WHERE} GROUP BY tag, band_url ORDER BY count DESC",
track_list: "SELECT * FROM track INNER JOIN item USING (account, item_id) {JOIN TAG} WHERE account = ? {WHERE} ORDER BY band_url, item_title COLLATE NOCASE, track_number"
}
function loadPreviews(locals, field, number, whereClause, filter_field, filter) {
const params = [number]
let sql = `SELECT ${field}, item_url, item_art_url FROM (SELECT ${field}, item_url, item_art_url, row_number() OVER (PARTITION BY ${field} ORDER BY purchased DESC) AS row_number FROM item {JOIN TAG} {WHERE}) WHERE row_number <= ?`
function loadPreviews(locals, field, number, whereClause, account, filter_field, filter) {
const params = [account, number]
let sql = `SELECT ${field}, item_url, item_art_url FROM (SELECT ${field}, item_url, item_art_url, row_number() OVER (PARTITION BY ${field} ORDER BY purchased DESC) AS row_number FROM item {JOIN TAG} WHERE account = ? {WHERE}) WHERE row_number <= ?`
sql = sql.replace("{WHERE}", whereClause)
if (whereClause) {
params.unshift(filter)
if (filter_field === "band_url") {
params.splice(1, 0, `%${filter}%`)
} else {
params.splice(1, 0, filter)
}
}
if (filter_field === "tag" && filter) {
sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (item_id)")
sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (account, item_id)")
} else {
sql = sql.replace("{JOIN TAG}", "")
}
const previews = db.prepare(sql).all(params)
// TODO: performance?
for (const item of locals.items) {
@ -52,20 +61,26 @@ function loadPreviews(locals, field, number, whereClause, filter_field, filter)
}
}
const schema = z.object({
arrange: z.enum(["album", "artist", "label", "tag", "track"]),
shape: z.enum(["grid", "list"]),
filter_field: z.enum(["band_name", "band_url", "item_id", "tag"]).optional(),
filter: z.string().optional()
})
const schema = {
query: z.object({
arrange: z.enum(["album", "artist", "label", "tag", "track"]),
shape: z.enum(["grid", "list"]),
filter_field: z.enum(["band_name", "band_url", "item_id", "tag"]).optional(),
filter: z.string().optional()
}),
params: z.object({
account: z.string()
})
}
router.get("/", defineEventHandler({
router.get("/:account/", defineEventHandler({
onBeforeResponse: pugSync.compressResponse,
handler: async event => {
try {
var {arrange, shape, filter, filter_field} = await getValidatedQuery(event, schema.parse)
var {account} = await getValidatedRouterParams(event, schema.params.parse)
var {arrange, shape, filter, filter_field} = await getValidatedQuery(event, schema.query.parse)
} catch (e) {
return sendRedirect(event, "/?arrange=album&shape=grid", 302)
return sendRedirect(event, "?arrange=album&shape=grid", 302)
}
const query = getQuery(event)
if (arrange === "track") {
@ -73,7 +88,7 @@ router.get("/", defineEventHandler({
query.shape = "list"
}
const mode = `${arrange}_${shape}`
const params = []
const params = [account]
let sql = sqls[mode]
let whereClause = ""
if (filter_field && filter) {
@ -84,23 +99,33 @@ router.get("/", defineEventHandler({
} else {
params.push(filter)
}
whereClause = `WHERE ${filter_field} ${operator} ?`
whereClause = `AND ${filter_field} ${operator} ?`
sql = sql.replace("{ORDER}", "ORDER BY item_title COLLATE NOCASE")
} else {
sql = sql.replace("{ORDER}", "ORDER BY purchased DESC")
}
sql = sql.replace("{WHERE}", whereClause)
if (filter_field === "tag" && filter) {
sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (item_id)")
sql = sql.replace("{JOIN TAG}", "INNER JOIN item_tag USING (account, item_id)")
} else {
sql = sql.replace("{JOIN TAG}", "")
}
const prepared = db.prepare(sql)
if (arrange === "tag") {
prepared.raw()
if (mode === "tag_grid" && ((!filter_field || !filter) || filter_field === "tag")) {
sql = `SELECT tag, count(*) AS count FROM (${sql}) GROUP BY tag ORDER BY count DESC`
}
try {
const prepared = db.prepare(sql)
if (arrange === "tag") {
prepared.raw()
}
var items = prepared.all(params)
} catch (e) {
console.error(sql, params)
throw e
}
const locals = {
items: prepared.all(params),
items,
account,
query,
count: {
total: db.prepare("SELECT count(*) FROM item").pluck().get(),
@ -123,9 +148,12 @@ router.get("/", defineEventHandler({
}
}
if (mode === "artist_grid") {
loadPreviews(locals, "band_name", 4, whereClause, filter_field, filter)
loadPreviews(locals, "band_name", 4, whereClause, account, filter_field, filter)
} else if (mode === "label_grid") {
loadPreviews(locals, "band_url", 6, whereClause, filter_field, filter)
loadPreviews(locals, "band_url", 6, whereClause, account, filter_field, filter)
} else if (arrange === "tag") {
locals.downloadManager = loadTags.downloadManager
locals.downloader = loadTags.downloadManager.check(account)
}
return pugSync.render(event, `${arrange}_${shape}.pug`, locals)
}

99
routes/load-tags.js Normal file
View file

@ -0,0 +1,99 @@
// @ts-check
const domino = require("domino")
const {z} = require("zod")
const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3")
const {sync, db, from, router} = require("../passthrough")
const pugSync = sync.require("../pug-sync")
const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
constructor(account) {
super()
this.account = account
this.processed = 0
this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
this.total = this.untaggedItems.length
this.running = false
this.outcome = null
}
async _start() {
if (this.running) return
this.running = true
try {
for (const {account, item_id, item_title, item_url} of this.untaggedItems) {
const html = await fetch(item_url).then(res => res.text())
const doc = domino.createDocument(html)
// @ts-ignore
const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
db.transaction(() => {
for (const tag of tags) {
i.run(account, item_id, tag)
}
})()
this.processed++
}
this.outcome = "Success"
} catch (e) {
console.error(e)
this.outcome = e.toString()
} finally {
this.running = false
}
}
}
const downloadManager = new class {
/** @type {Map<string, TagDownloader>} */
inProgressTagDownloads = sync.remember(() => new Map())
/** @param {string} account */
check(account) {
return this.inProgressTagDownloads.get(account) || (() => {
const downloader = new TagDownloader(account)
this.inProgressTagDownloads.set(account, downloader)
console.log(`created downloader ${account}`)
setTimeout(() => {
this.resolve(account)
})
return downloader
})()
}
/** @param {string} account */
start(account) {
const downloader = this.check(account)
downloader._start()
return downloader
}
/** @param {string} account */
resolve(account) {
const downloader = this.check(account)
if (!downloader.running) {
console.log(`disposed downloader ${account}`)
this.inProgressTagDownloads.delete(account)
}
}
}
const schema = z.object({
account: z.string()
})
router.get("/api/tag-download", defineEventHandler(async event => {
const {account} = await getValidatedQuery(event, schema.parse)
const downloader = downloadManager.check(account)
return pugSync.render(event, "includes/tag-status.pug", {downloadManager, downloader, account})
}))
router.post("/api/tag-download", defineEventHandler(async event => {
const {account} = await readValidatedBody(event, schema.parse)
const downloader = downloadManager.start(account)
return pugSync.render(event, "includes/tag-status.pug", {downloadManager, downloader, account})
}))
module.exports.downloadManager = downloadManager

30
scripts/load-tags.js
View file

@ -1,30 +0,0 @@
// @ts-check
const domino = require("domino")
const sqlite = require("better-sqlite3")
const db = new sqlite("bc-explorer.db", {fileMustExist: true})
require("../db/migrate").migrate(db)
Object.assign(require("../passthrough"), {db})
const {from} = require("../db/orm")
const i = db.prepare("INSERT OR IGNORE INTO item_tag (item_id, tag) VALUES (?, ?)")
;(async () => {
const untaggedItems = from("item").select("item_id", "item_title", "item_url").join("item_tag", "item_id", "left").and("WHERE tag IS NULL").all()
console.log(`Downloading tags for ${untaggedItems.length} purchased items`)
let processed = 1
for (const {item_id, item_title, item_url} of untaggedItems) {
const html = await fetch(item_url).then(res => res.text())
const doc = domino.createDocument(html)
// @ts-ignore
const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
console.log(`[${processed}/${untaggedItems.length}] tagging ${item_title} with ${tags.join(", ")}`)
db.transaction(() => {
for (const tag of tags) {
i.run(item_id, tag)
}
})()
processed++
}
})()

11
scripts/populate-albums-tracks.js
View file

@ -11,7 +11,7 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
assert(collection_summary)
const body = JSON.parse(collection_summary.response.content.text)
const {fan_id} = body
const {fan_id, username: account} = body.collection_summary
const count = Object.keys(body.collection_summary.tralbum_lookup).length
const newestPurchase = Object.values(body.collection_summary.tralbum_lookup).sort((a, b) => new Date(b.purchased).getTime() - new Date(a.purchased).getTime())[0]
@ -36,12 +36,15 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
const db = new sqlite("bc-explorer.db")
require("../db/migrate").migrate(db)
const columns = ["item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id)
const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
const upsert_columns = ["added", "updated", "purchased"]
const preparedItem = db.prepare(`INSERT OR IGNORE INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`)
db.transaction(() => {
for (const item of items.items) {
preparedItem.run({
account,
...item,
purchased: new Date(item.purchased).getTime(),
added: new Date(item.added).getTime(),
@ -52,12 +55,13 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
const storedItemCount = db.prepare("SELECT count(*) AS count FROM item").pluck().get()
console.log(`Stored ${storedItemCount} purchased items`)
const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (item_id, track_id, title, artist, track_number, duration, mp3) VALUES (@item_id, @track_id, @title, @artist, @track_number, @duration, @mp3)")
const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration, mp3) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration, @mp3)")
db.transaction(() => {
for (const [key, tracklist] of Object.entries(items.tracklists)) {
assert.match(key[0], /[at]/)
for (const track of tracklist) {
preparedTrack.run({
account,
item_id: key.slice(1),
track_id: track.id,
mp3: track.file?.["mp3-v0"],
@ -68,5 +72,4 @@ const har = JSON.parse(fs.readFileSync("scripts/account.har", "utf8"))
})()
const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track").pluck().get()
console.log(`Stored ${storedTrackCount} tracks`)
console.log("To load tag data, please now run node ./scripts/load-tags.js")
})()

1
server.js → start.js
View file

@ -31,7 +31,6 @@ const icons = sync.require("./icons")
pugSync.addGlobals({h3, select, icons})
pugSync.createRoute(router, "/", "home.pug")
pugSync.createRoute(router, "/ok", "ok.pug")
// Routes