From cf6310b89a10065a7a9e27d3389ba10bde2b7fa5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 3 Apr 2025 16:39:46 +1300 Subject: [PATCH] Import accounts from home page --- pug/album_grid.pug | 15 +++--- pug/artist_grid.pug | 13 +++--- pug/collection-loaded.pug | 6 +++ pug/home.pug | 23 ++++++--- pug/includes/layout.pug | 12 +++-- pug/includes/tag-status.pug | 12 +++-- pug/label_grid.pug | 19 ++++---- readme.md | 11 +++-- routes/app.js | 38 ++++++++------- routes/load-collection.js | 93 +++++++++++++++++++++++++++++++++++++ routes/load-tags.js | 41 ++++++++++++++-- start.js | 1 + 12 files changed, 222 insertions(+), 62 deletions(-) create mode 100644 pug/collection-loaded.pug create mode 100644 routes/load-collection.js diff --git a/pug/album_grid.pug b/pug/album_grid.pug index 4431d36..c196547 100644 --- a/pug/album_grid.pug +++ b/pug/album_grid.pug @@ -10,14 +10,15 @@ block view p.fs-body3.mb8= item.item_title .d-flex.fw-wrap.g4 if item.why - a.s-tag.s-tag__xs(title=item.why).fc-orange-400(href=and({filter_field: "why", filter: "reviewed"})) + a.s-tag.s-tag__xs.fc-orange-400(title=item.why href=and({filter_field: "why", filter: "reviewed"})) != icons.get("star-solid", 16) - a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "item_id", filter: item.item_id, filter_fuzzy: null})) - span.s-tag--sponsor!= icons.get("music-note", 16) - = item.track_count - span.s-tag.s-tag__xs - span.s-tag--sponsor!= icons.get("compact-disc", 16) - = item.total_duration + if hasFullTrackData + a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "item_id", filter: item.item_id, filter_fuzzy: null})) + span.s-tag--sponsor!= icons.get("music-note", 16) + = item.track_count + span.s-tag.s-tag__xs + span.s-tag--sponsor!= icons.get("compact-disc", 16) + = item.total_duration a.s-tag.s-tag__xs(href=and({filter_field: "band_name", filter: item.band_name, filter_fuzzy: null})) span.s-tag--sponsor!= icons.get("people-tag", 16) = item.band_name diff --git a/pug/artist_grid.pug b/pug/artist_grid.pug index 86e69af..26c7614 100644 --- a/pug/artist_grid.pug +++ b/pug/artist_grid.pug @@ -11,12 +11,13 @@ block view a.s-tag.s-tag__xs(href=and({arrange: "album", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null})) span.s-tag--sponsor!= icons.get("album", 16) = item.album_count - a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null})) - span.s-tag--sponsor!= icons.get("music-note", 16) - = item.track_count - span.s-tag.s-tag__xs - span.s-tag--sponsor!= icons.get("compact-disc", 16) - = item.total_duration + if hasFullTrackData + a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null})) + span.s-tag--sponsor!= icons.get("music-note", 16) + = item.track_count + span.s-tag.s-tag__xs + span.s-tag--sponsor!= icons.get("compact-disc", 16) + = item.total_duration - let labels = item.labels.split("|").map(x => x.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1")) each label in labels a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label, filter_fuzzy: null})) diff --git a/pug/collection-loaded.pug b/pug/collection-loaded.pug new file mode 100644 index 0000000..270564e --- /dev/null +++ b/pug/collection-loaded.pug @@ -0,0 +1,6 @@ +.s-notice.s-notice__success + .d-flex.ai-center.gx16 + != icons.get("check-circle") + .fl-grow1 Imported #{storedItemCount}/#{count} purchases and #{storedTrackCount} tracks. + .mt16 + a.s-link(href=`/${account}/`) Check it out diff --git a/pug/home.pug b/pug/home.pug index 41c78b6..96d8ca2 100644 --- a/pug/home.pug +++ b/pug/home.pug @@ -5,6 +5,8 @@ html title BC Explorer link(rel="stylesheet" type="text/css" href="/static/stacks.min.css") + script(src="/static/htmx.js") + meta(name="htmx-config" content='{"requestClass":"is-loading"}') body.themed.theme-system.overflow-y-scroll header.s-topbar.ps-sticky.t0 .s-topbar--container.wmx9 @@ -12,9 +14,18 @@ html != icons.get("compass-solid", 24) .ml4 BC Explorer - .mx-auto.wmx9.py24.px16.g24.s-prose - h1 Select profile - - const names = select("account", "account").pluck().all() - ul - each name in names - li: a(href=`/${name}/`)= name + .mx-auto.wmx9.py24.px16.g24 + .s-prose + h1 Select profile + - const names = select("account", "account").pluck().all() + ul + each name in names + li: a(href=`/${name}/`)= name + + form.mt32(hx-post="/api/load-collection" hx-target="#results" hx-indicator="#submit-username") + h2 Add your profile + .d-flex.gy4.fd-column.ps-relative + label.s-label(for="username") Bandcamp username + input.s-input.wmx3#username(name="account" placeholder="Enter your Bandcamp username here") + button.s-btn.s-btn__filled.my16#submit-username Load collection + #results.d-flex diff --git a/pug/includes/layout.pug b/pug/includes/layout.pug index 6e6bb4d..2f61f22 100644 --- a/pug/includes/layout.pug +++ b/pug/includes/layout.pug @@ -28,6 +28,13 @@ html .s-navigation--item.is-loading svg, .s-tag.is-loading svg, .s-sidebarwidget.is-loading svg { visibility: hidden; } + .s-btn__icon.is-loading { + --_li-offset: 0.7em; + --_il-size: 1.5em; + } + .s-btn__icon.is-loading svg { + display: none; + } .s-navigation__toggle.s-navigation { --_na-item-bg: var(--black-150); } @@ -98,7 +105,7 @@ html strong= filter a.s-btn.s-notice--btn(href=and({filter: null, filter_field: null, filter_fuzzy: null})) Clear else - form.d-flex.ai-stretch.gx8.jc-space-between.baw0 + form.d-flex.ai-stretch.gx8.jc-space-between.baw0(hx-indicator="#search-submit") input.s-input(name="filter" placeholder="Search" autocomplete="off").fl-grow1 input(type="hidden" name="filter_field" value= ( arrange === "artist" ? "band_name" @@ -108,11 +115,10 @@ html input(type="hidden" name="filter_fuzzy" value="true") input(type="hidden" name="arrange" value=arrange) input(type="hidden" name="shape" value=shape) - button.s-btn.s-btn__xs.s-btn__outlined.s-btn__muted!= icons.get("search") + button.s-btn.s-btn__xs.s-btn__icon.s-btn__outlined.s-btn__muted#search-submit(style="height: 38px")!= icons.get("search") .s-sidebarwidget.wmn3 .s-sidebarwidget--header Collection - a.s-sidebarwidget--action Refresh table.s-sidebarwidget--content.s-sidebarwidget__items tr.s-sidebarwidget--item th items diff --git a/pug/includes/tag-status.pug b/pug/includes/tag-status.pug index 2d2f8a6..06d6d77 100644 --- a/pug/includes/tag-status.pug +++ b/pug/includes/tag-status.pug @@ -1,5 +1,5 @@ if downloader.total > 0 - .d-flex.jc-center#tag-download + #tag-download if !downloader.running && !downloader.outcome form.s-notice.s-notice__info.d-flex.ai-center.p8.gx16.pl16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-post="/api/tag-download") != icons.get("info-circle") @@ -8,15 +8,17 @@ if downloader.total > 0 button.s-btn.s-btn__outlined Download now else if !downloader.outcome - .s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s") + .s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s" hx-indicator="null") .d-flex.gx16.ai-center != icons.get("cloud-download") - div Downloading tags... + .fl-grow1 Downloading tags... + #tag-status-indicator p.mt12 You can keep using BC Explorer while this continues in the background. + - let percentage = `${Math.round(downloader.processed/downloader.total*100)}%` .s-progress.mt16 - .s-progress--bar(style=`width: ${downloader.processed/downloader.total*100}%`) + .s-progress--bar(style=`width: ${percentage}`) .d-flex.jc-space-between.fs-fine - span= downloader.processed + span= percentage span= downloader.total else if downloader.outcome === "Success" diff --git a/pug/label_grid.pug b/pug/label_grid.pug index 4f46b0e..9dd0ef9 100644 --- a/pug/label_grid.pug +++ b/pug/label_grid.pug @@ -13,15 +13,16 @@ block view a.s-tag.s-tag__xs(href=and({arrange: "album", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null})) span.s-tag--sponsor!= icons.get("album", 16) = item.album_count - a.s-tag.s-tag__xs(href=and({arrange: "artist", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null})) - span.s-tag--sponsor!= icons.get("people-tag", 16) - = item.artist_count - a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null})) - span.s-tag--sponsor!= icons.get("music-note", 16) - = item.track_count - span.s-tag.s-tag__xs - span.s-tag--sponsor!= icons.get("compact-disc", 16) - = item.total_duration + a.s-tag.s-tag__xs(href=and({arrange: "artist", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null})) + span.s-tag--sponsor!= icons.get("people-tag", 16) + = item.artist_count + if hasFullTrackData + a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null})) + span.s-tag--sponsor!= icons.get("music-note", 16) + = item.track_count + span.s-tag.s-tag__xs + span.s-tag--sponsor!= icons.get("compact-disc", 16) + = item.total_duration each preview in item.previews a.d-flex(href=preview.item_url) img(loading="lazy" src=preview.item_art_url width=210 height=210 style="height: auto; width: auto; max-height: 70px") diff --git a/readme.md b/readme.md index e6c4e27..f18bb3e 100644 --- a/readme.md +++ b/readme.md @@ -10,14 +10,19 @@ but the idea is you can more easily search your whole collection and play it str ## how to use +1. npm install +2. node start.js +3. http://localhost:2239 + +## import more reliable statistics + +the default data is pretty close, but you can do this to get the exact data + 1. log into bandcamp in your browser 2. in the top right, click the button to view your collection, should take you to a url like https://bandcamp.com/cloudrac3r 3. open devtools and reload the page 4. go to network tab and save all as har, save as scripts/account.har -5. npm install 6. node scripts/populate-albums-tracks.js -8. node start.js -9. http://localhost:2239 ## license diff --git a/routes/app.js b/routes/app.js index b95161b..235caad 100644 --- a/routes/app.js +++ b/routes/app.js @@ -11,14 +11,15 @@ const loadTags = sync.require("./load-tags") const displayCurrency = "NZD" const displayCurrencySymbol = "$" const currencyExchange = new Map([ - ["USD", 1], - ["JPY", 0.0067], - ["NZD", 0.57], + ["AUD", 0.63], + ["CAD", 0.7], + ["CHF", 1.13], ["EUR", 1.08], ["GBP", 1.3], - ["CAD", 0.7], + ["JPY", 0.0067], ["NOK", 0.1], - ["CHF", 1.13] + ["NZD", 0.57], + ["USD", 1], ]) const sqls = { @@ -136,23 +137,23 @@ router.get("/:account/", defineEventHandler({ account, query, count: { - total: db.prepare("SELECT count(*) FROM item").pluck().get(), - runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track").pluck().get(), - albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album'").pluck().get(), - singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track'").pluck().get(), - free: db.prepare("SELECT count(*) FROM item WHERE price = 0").pluck().get(), - paid: db.prepare("SELECT count(*) FROM item WHERE price > 0").pluck().get(), - tracks: db.prepare("SELECT count(*) FROM track").pluck().get(), - avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (item_id) WHERE item_type = 'album' GROUP BY item_id)").pluck().get()*10)/10, - tags: db.prepare("SELECT count(*) FROM item_tag").pluck().get(), - avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag GROUP BY item_id)").pluck().get()*10)/10, - lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag GROUP BY tag HAVING count(*) = 1)").pluck().get(), - value: Math.round(select("item", ["currency", "price"]).all().map(c => { + total: db.prepare("SELECT count(*) FROM item WHERE account = ?").pluck().get(account), + runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track WHERE account = ?").pluck().get(account), + albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album' AND account = ?").pluck().get(account), + singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track' AND account = ?").pluck().get(account), + free: db.prepare("SELECT count(*) FROM item WHERE price = 0 AND account = ?").pluck().get(account), + paid: db.prepare("SELECT count(*) FROM item WHERE price > 0 AND account = ?").pluck().get(account), + tracks: db.prepare("SELECT count(*) FROM track WHERE account = ?").pluck().get(account), + avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (account, item_id) WHERE item_type = 'album' AND account = ? GROUP BY item_id)").pluck().get(account)*10)/10, + tags: db.prepare("SELECT count(*) FROM item_tag WHERE account = ?").pluck().get(account), + avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag WHERE account = ? GROUP BY item_id)").pluck().get(account)*10)/10, + lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag WHERE account = ? GROUP BY tag HAVING count(*) = 1)").pluck().get(account), + value: Math.round(select("item", ["currency", "price"], {account}).all().map(c => { return (currencyExchange.get(c.currency) || 0.6) * c.price / (currencyExchange.get(displayCurrency) || 1) / 10 }).reduce((a, c) => a + c, 0)) * 10, displayCurrency, displayCurrencySymbol, - taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get() + taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item WHERE account = ? group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get(account) } } if (mode === "artist_grid") { @@ -163,6 +164,7 @@ router.get("/:account/", defineEventHandler({ locals.downloadManager = loadTags.downloadManager locals.downloader = loadTags.downloadManager.check(account) } + locals.hasFullTrackData = locals.count.tracks > locals.count.total return pugSync.render(event, `${arrange}_${shape}.pug`, locals) } })) diff --git a/routes/load-collection.js b/routes/load-collection.js new file mode 100644 index 0000000..39d1428 --- /dev/null +++ b/routes/load-collection.js @@ -0,0 +1,93 @@ +// @ts-check + +const assert = require("assert/strict") +const fs = require("fs") +const sqlite = require("better-sqlite3") +const domino = require("domino") +const {defineEventHandler, readValidatedBody} = require("h3") +const {z} = require("zod") + +const {sync, db, router} = require("../passthrough") +const pugSync = sync.require("../pug-sync") + +async function loadCollection(inputUsername) { + assert.match(inputUsername, /^[a-z0-9_-]+$/) + const html = await fetch(`https://bandcamp.com/${inputUsername}`).then(res => res.text()) + const doc = domino.createDocument(html) + + const first = doc.querySelector(".collection-item-container") + assert(first) + const token = first.getAttribute("data-token") + assert(token) + + const tokenParts = token.split(":") + tokenParts[0] = String(+tokenParts[0] + 1) // ensure we get the first item + const customToken = tokenParts.join(":") + + const pagedataText = doc.querySelector("#pagedata")?.getAttribute("data-blob") + assert(pagedataText) + const pagedata = JSON.parse(pagedataText) + const fan_id = pagedata.fan_data.fan_id + const account = pagedata.fan_data.username + const count = pagedata.collection_data.item_count + + const items = await fetch("https://bandcamp.com/api/fancollection/1/collection_items", { + method: "POST", + body: JSON.stringify({ + fan_id, + older_than_token: customToken, + count + }) + }).then(res => res.json()) + + db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id) + + const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"] + const upsert_columns = ["added", "updated", "purchased"] + const preparedItem = db.prepare(`INSERT INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`) + db.transaction(() => { + for (const item of items.items) { + preparedItem.run({ + account, + ...item, + purchased: new Date(item.purchased).getTime(), + added: new Date(item.added).getTime(), + updated: new Date(item.updated).getTime() + }) + } + })() + const storedItemCount = db.prepare("SELECT count(*) AS count FROM item WHERE account = ?").pluck().get(account) + + const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration)") + db.transaction(() => { + for (const [key, tracklist] of Object.entries(items.tracklists)) { + assert.match(key[0], /[at]/) + for (const track of tracklist) { + preparedTrack.run({ + account, + item_id: key.slice(1), + track_id: track.id, + ...track + }) + } + } + })() + const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track WHERE account = ?").pluck().get(account) + + return { + storedItemCount, + storedTrackCount, + count, + account + } +} + +const schema = z.object({ + account: z.string() +}) + +router.post("/api/load-collection", defineEventHandler(async event => { + const {account} = await readValidatedBody(event, schema.parse) + const result = await loadCollection(account) + return pugSync.render(event, "collection-loaded.pug", result) +})) diff --git a/routes/load-tags.js b/routes/load-tags.js index 1d1e63b..a061158 100644 --- a/routes/load-tags.js +++ b/routes/load-tags.js @@ -7,14 +7,15 @@ const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3") const {sync, db, from, router} = require("../passthrough") const pugSync = sync.require("../pug-sync") -const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)") +const insertTag = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)") +const insertTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (?, ?, ?, ?, ?, ?, ?)") class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) { constructor(account) { super() this.account = account this.processed = 0 - this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account) + this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url, band_name FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account) this.total = this.untaggedItems.length this.running = false this.outcome = null @@ -24,16 +25,46 @@ class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) { if (this.running) return this.running = true try { - for (const {account, item_id, item_title, item_url} of this.untaggedItems) { - const html = await fetch(item_url).then(res => res.text()) + for (const {account, item_id, item_title, item_url, band_name} of this.untaggedItems) { + const res = await fetch(item_url) + + // delete unreachable items, otherwise it will perpetually try to download tags for them + if (res.status === 404) { + db.prepare("DELETE FROM item WHERE account = ? AND item_id = ?").run(account, item_id) + this.processed++ + continue + } + + const html = await res.text() const doc = domino.createDocument(html) + // @ts-ignore const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent) db.transaction(() => { for (const tag of tags) { - i.run(account, item_id, tag) + insertTag.run(account, item_id, tag) } })() + + // @ts-ignore + const tracks = [...doc.querySelectorAll(".track_row_view").cache] + db.transaction(() => { + for (const track of tracks) { + const track_number = parseInt(track.querySelector(".track_number").textContent) + let title = track.querySelector(".track-title").textContent + let artist = band_name + const match = title.match(/^([^-]*) - (.*)$/) + if (match) { + artist = match[1] + title = match[2] + } + const duration = track.querySelector(".time").textContent.split(":").reverse().reduce((a, c, i) => 60**i * c + a, 0) + console.log(track_number, title, artist, duration) + if (!track_number || !title || !artist || !duration) continue + insertTrack.run(account, item_id, track_number, title, artist, track_number, duration) + } + })() + this.processed++ } this.outcome = "Success" diff --git a/start.js b/start.js index 8ad1467..697ca2c 100644 --- a/start.js +++ b/start.js @@ -35,6 +35,7 @@ pugSync.createRoute(router, "/", "home.pug") // Routes sync.require("./routes/app") +sync.require("./routes/load-collection") // Files