cadence/bc-explorer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Import accounts from home page

Browse source
This commit is contained in:
Cadence Ember 2025-04-03 16:39:46 +13:00
parent cba0552ed0
commit cf6310b89a
12 changed files with 222 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

3
pug/album_grid.pug
View file

@ -10,8 +10,9 @@ block view
p.fs-body3.mb8= item.item_title
.d-flex.fw-wrap.g4
if item.why
a.s-tag.s-tag__xs(title=item.why).fc-orange-400(href=and({filter_field: "why", filter: "reviewed"}))
a.s-tag.s-tag__xs.fc-orange-400(title=item.why href=and({filter_field: "why", filter: "reviewed"}))
!= icons.get("star-solid", 16)
if hasFullTrackData
a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "item_id", filter: item.item_id, filter_fuzzy: null}))
span.s-tag--sponsor!= icons.get("music-note", 16)
= item.track_count

1
pug/artist_grid.pug
View file

@ -11,6 +11,7 @@ block view
a.s-tag.s-tag__xs(href=and({arrange: "album", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null}))
span.s-tag--sponsor!= icons.get("album", 16)
= item.album_count
if hasFullTrackData
a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null}))
span.s-tag--sponsor!= icons.get("music-note", 16)
= item.track_count

6
pug/collection-loaded.pug Normal file
View file

@ -0,0 +1,6 @@
.s-notice.s-notice__success
.d-flex.ai-center.gx16
!= icons.get("check-circle")
.fl-grow1 Imported #{storedItemCount}/#{count} purchases and #{storedTrackCount} tracks.
.mt16
a.s-link(href=`/${account}/`) Check it out

13
pug/home.pug
View file

@ -5,6 +5,8 @@ html
<meta name="viewport" content="width=device-width, initial-scale=1" />
title BC Explorer
link(rel="stylesheet" type="text/css" href="/static/stacks.min.css")
script(src="/static/htmx.js")
meta(name="htmx-config" content='{"requestClass":"is-loading"}')
body.themed.theme-system.overflow-y-scroll
header.s-topbar.ps-sticky.t0
.s-topbar--container.wmx9
@ -12,9 +14,18 @@ html
!= icons.get("compass-solid", 24)
.ml4 BC Explorer
.mx-auto.wmx9.py24.px16.g24.s-prose
.mx-auto.wmx9.py24.px16.g24
.s-prose
h1 Select profile
- const names = select("account", "account").pluck().all()
ul
each name in names
li: a(href=`/${name}/`)= name
form.mt32(hx-post="/api/load-collection" hx-target="#results" hx-indicator="#submit-username")
h2 Add your profile
.d-flex.gy4.fd-column.ps-relative
label.s-label(for="username") Bandcamp username
input.s-input.wmx3#username(name="account" placeholder="Enter your Bandcamp username here")
button.s-btn.s-btn__filled.my16#submit-username Load collection
#results.d-flex

12
pug/includes/layout.pug
View file

@ -28,6 +28,13 @@ html
.s-navigation--item.is-loading svg, .s-tag.is-loading svg, .s-sidebarwidget.is-loading svg {
visibility: hidden;
}
.s-btn__icon.is-loading {
--_li-offset: 0.7em;
--_il-size: 1.5em;
}
.s-btn__icon.is-loading svg {
display: none;
}
.s-navigation__toggle.s-navigation {
--_na-item-bg: var(--black-150);
}
@ -98,7 +105,7 @@ html
strong= filter
a.s-btn.s-notice--btn(href=and({filter: null, filter_field: null, filter_fuzzy: null})) Clear
else
form.d-flex.ai-stretch.gx8.jc-space-between.baw0
form.d-flex.ai-stretch.gx8.jc-space-between.baw0(hx-indicator="#search-submit")
input.s-input(name="filter" placeholder="Search" autocomplete="off").fl-grow1
input(type="hidden" name="filter_field" value=
( arrange === "artist" ? "band_name"
@ -108,11 +115,10 @@ html
input(type="hidden" name="filter_fuzzy" value="true")
input(type="hidden" name="arrange" value=arrange)
input(type="hidden" name="shape" value=shape)
button.s-btn.s-btn__xs.s-btn__outlined.s-btn__muted!= icons.get("search")
button.s-btn.s-btn__xs.s-btn__icon.s-btn__outlined.s-btn__muted#search-submit(style="height: 38px")!= icons.get("search")
.s-sidebarwidget.wmn3
.s-sidebarwidget--header Collection
a.s-sidebarwidget--action Refresh
table.s-sidebarwidget--content.s-sidebarwidget__items
tr.s-sidebarwidget--item
th items

12
pug/includes/tag-status.pug
View file

@ -1,5 +1,5 @@
if downloader.total > 0
.d-flex.jc-center#tag-download
#tag-download
if !downloader.running && !downloader.outcome
form.s-notice.s-notice__info.d-flex.ai-center.p8.gx16.pl16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-post="/api/tag-download")
!= icons.get("info-circle")
@ -8,15 +8,17 @@ if downloader.total > 0
button.s-btn.s-btn__outlined Download now
else if !downloader.outcome
.s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s")
.s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s" hx-indicator="null")
.d-flex.gx16.ai-center
!= icons.get("cloud-download")
div Downloading tags...
.fl-grow1 Downloading tags...
#tag-status-indicator
p.mt12 You can keep using BC Explorer while this continues in the background.
- let percentage = `${Math.round(downloader.processed/downloader.total*100)}%`
.s-progress.mt16
.s-progress--bar(style=`width: ${downloader.processed/downloader.total*100}%`)
.s-progress--bar(style=`width: ${percentage}`)
.d-flex.jc-space-between.fs-fine
span= downloader.processed
span= percentage
span= downloader.total
else if downloader.outcome === "Success"

1
pug/label_grid.pug
View file

@ -16,6 +16,7 @@ block view
a.s-tag.s-tag__xs(href=and({arrange: "artist", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
span.s-tag--sponsor!= icons.get("people-tag", 16)
= item.artist_count
if hasFullTrackData
a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
span.s-tag--sponsor!= icons.get("music-note", 16)
= item.track_count

11
readme.md
View file

@ -10,14 +10,19 @@ but the idea is you can more easily search your whole collection and play it str
## how to use
1. npm install
2. node start.js
3. http://localhost:2239
## import more reliable statistics
the default data is pretty close, but you can do this to get the exact data
1. log into bandcamp in your browser
2. in the top right, click the button to view your collection, should take you to a url like https://bandcamp.com/cloudrac3r
3. open devtools and reload the page
4. go to network tab and save all as har, save as scripts/account.har
5. npm install
6. node scripts/populate-albums-tracks.js
8. node start.js
9. http://localhost:2239
## license

38
routes/app.js
View file

@ -11,14 +11,15 @@ const loadTags = sync.require("./load-tags")
const displayCurrency = "NZD"
const displayCurrencySymbol = "$"
const currencyExchange = new Map([
["USD", 1],
["JPY", 0.0067],
["NZD", 0.57],
["AUD", 0.63],
["CAD", 0.7],
["CHF", 1.13],
["EUR", 1.08],
["GBP", 1.3],
["CAD", 0.7],
["JPY", 0.0067],
["NOK", 0.1],
["CHF", 1.13]
["NZD", 0.57],
["USD", 1],
])
const sqls = {
@ -136,23 +137,23 @@ router.get("/:account/", defineEventHandler({
account,
query,
count: {
total: db.prepare("SELECT count(*) FROM item").pluck().get(),
runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track").pluck().get(),
albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album'").pluck().get(),
singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track'").pluck().get(),
free: db.prepare("SELECT count(*) FROM item WHERE price = 0").pluck().get(),
paid: db.prepare("SELECT count(*) FROM item WHERE price > 0").pluck().get(),
tracks: db.prepare("SELECT count(*) FROM track").pluck().get(),
avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (item_id) WHERE item_type = 'album' GROUP BY item_id)").pluck().get()*10)/10,
tags: db.prepare("SELECT count(*) FROM item_tag").pluck().get(),
avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag GROUP BY item_id)").pluck().get()*10)/10,
lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag GROUP BY tag HAVING count(*) = 1)").pluck().get(),
value: Math.round(select("item", ["currency", "price"]).all().map(c => {
total: db.prepare("SELECT count(*) FROM item WHERE account = ?").pluck().get(account),
runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track WHERE account = ?").pluck().get(account),
albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album' AND account = ?").pluck().get(account),
singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track' AND account = ?").pluck().get(account),
free: db.prepare("SELECT count(*) FROM item WHERE price = 0 AND account = ?").pluck().get(account),
paid: db.prepare("SELECT count(*) FROM item WHERE price > 0 AND account = ?").pluck().get(account),
tracks: db.prepare("SELECT count(*) FROM track WHERE account = ?").pluck().get(account),
avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (account, item_id) WHERE item_type = 'album' AND account = ? GROUP BY item_id)").pluck().get(account)*10)/10,
tags: db.prepare("SELECT count(*) FROM item_tag WHERE account = ?").pluck().get(account),
avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag WHERE account = ? GROUP BY item_id)").pluck().get(account)*10)/10,
lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag WHERE account = ? GROUP BY tag HAVING count(*) = 1)").pluck().get(account),
value: Math.round(select("item", ["currency", "price"], {account}).all().map(c => {
return (currencyExchange.get(c.currency) || 0.6) * c.price / (currencyExchange.get(displayCurrency) || 1) / 10
}).reduce((a, c) => a + c, 0)) * 10,
displayCurrency,
displayCurrencySymbol,
taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get()
taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item WHERE account = ? group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get(account)
}
}
if (mode === "artist_grid") {
@ -163,6 +164,7 @@ router.get("/:account/", defineEventHandler({
locals.downloadManager = loadTags.downloadManager
locals.downloader = loadTags.downloadManager.check(account)
}
locals.hasFullTrackData = locals.count.tracks > locals.count.total
return pugSync.render(event, `${arrange}_${shape}.pug`, locals)
}
}))

93
routes/load-collection.js Normal file
View file

@ -0,0 +1,93 @@
// @ts-check
const assert = require("assert/strict")
const fs = require("fs")
const sqlite = require("better-sqlite3")
const domino = require("domino")
const {defineEventHandler, readValidatedBody} = require("h3")
const {z} = require("zod")
const {sync, db, router} = require("../passthrough")
const pugSync = sync.require("../pug-sync")
async function loadCollection(inputUsername) {
assert.match(inputUsername, /^[a-z0-9_-]+$/)
const html = await fetch(`https://bandcamp.com/${inputUsername}`).then(res => res.text())
const doc = domino.createDocument(html)
const first = doc.querySelector(".collection-item-container")
assert(first)
const token = first.getAttribute("data-token")
assert(token)
const tokenParts = token.split(":")
tokenParts[0] = String(+tokenParts[0] + 1) // ensure we get the first item
const customToken = tokenParts.join(":")
const pagedataText = doc.querySelector("#pagedata")?.getAttribute("data-blob")
assert(pagedataText)
const pagedata = JSON.parse(pagedataText)
const fan_id = pagedata.fan_data.fan_id
const account = pagedata.fan_data.username
const count = pagedata.collection_data.item_count
const items = await fetch("https://bandcamp.com/api/fancollection/1/collection_items", {
method: "POST",
body: JSON.stringify({
fan_id,
older_than_token: customToken,
count
})
}).then(res => res.json())
db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id)
const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
const upsert_columns = ["added", "updated", "purchased"]
const preparedItem = db.prepare(`INSERT INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`)
db.transaction(() => {
for (const item of items.items) {
preparedItem.run({
account,
...item,
purchased: new Date(item.purchased).getTime(),
added: new Date(item.added).getTime(),
updated: new Date(item.updated).getTime()
})
}
})()
const storedItemCount = db.prepare("SELECT count(*) AS count FROM item WHERE account = ?").pluck().get(account)
const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration)")
db.transaction(() => {
for (const [key, tracklist] of Object.entries(items.tracklists)) {
assert.match(key[0], /[at]/)
for (const track of tracklist) {
preparedTrack.run({
account,
item_id: key.slice(1),
track_id: track.id,
...track
})
}
}
})()
const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track WHERE account = ?").pluck().get(account)
return {
storedItemCount,
storedTrackCount,
count,
account
}
}
const schema = z.object({
account: z.string()
})
router.post("/api/load-collection", defineEventHandler(async event => {
const {account} = await readValidatedBody(event, schema.parse)
const result = await loadCollection(account)
return pugSync.render(event, "collection-loaded.pug", result)
}))

41
routes/load-tags.js
View file

@ -7,14 +7,15 @@ const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3")
const {sync, db, from, router} = require("../passthrough")
const pugSync = sync.require("../pug-sync")
const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
const insertTag = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
const insertTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (?, ?, ?, ?, ?, ?, ?)")
class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
constructor(account) {
super()
this.account = account
this.processed = 0
this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url, band_name FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
this.total = this.untaggedItems.length
this.running = false
this.outcome = null
@ -24,16 +25,46 @@ class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
if (this.running) return
this.running = true
try {
for (const {account, item_id, item_title, item_url} of this.untaggedItems) {
const html = await fetch(item_url).then(res => res.text())
for (const {account, item_id, item_title, item_url, band_name} of this.untaggedItems) {
const res = await fetch(item_url)
// delete unreachable items, otherwise it will perpetually try to download tags for them
if (res.status === 404) {
db.prepare("DELETE FROM item WHERE account = ? AND item_id = ?").run(account, item_id)
this.processed++
continue
}
const html = await res.text()
const doc = domino.createDocument(html)
// @ts-ignore
const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
db.transaction(() => {
for (const tag of tags) {
i.run(account, item_id, tag)
insertTag.run(account, item_id, tag)
}
})()
// @ts-ignore
const tracks = [...doc.querySelectorAll(".track_row_view").cache]
db.transaction(() => {
for (const track of tracks) {
const track_number = parseInt(track.querySelector(".track_number").textContent)
let title = track.querySelector(".track-title").textContent
let artist = band_name
const match = title.match(/^([^-]*) - (.*)$/)
if (match) {
artist = match[1]
title = match[2]
}
const duration = track.querySelector(".time").textContent.split(":").reverse().reduce((a, c, i) => 60**i * c + a, 0)
console.log(track_number, title, artist, duration)
if (!track_number || !title || !artist || !duration) continue
insertTrack.run(account, item_id, track_number, title, artist, track_number, duration)
}
})()
this.processed++
}
this.outcome = "Success"

1
start.js
View file

@ -35,6 +35,7 @@ pugSync.createRoute(router, "/", "home.pug")
// Routes
sync.require("./routes/app")
sync.require("./routes/load-collection")
// Files