Import accounts from home page

pug/album_grid.pug

@@ -10,14 +10,15 @@ block view
             p.fs-body3.mb8= item.item_title
           .d-flex.fw-wrap.g4
             if item.why
-              a.s-tag.s-tag__xs(title=item.why).fc-orange-400(href=and({filter_field: "why", filter: "reviewed"}))
+              a.s-tag.s-tag__xs.fc-orange-400(title=item.why href=and({filter_field: "why", filter: "reviewed"}))
                 != icons.get("star-solid", 16)
-            a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "item_id", filter: item.item_id, filter_fuzzy: null}))
+            if hasFullTrackData
-              span.s-tag--sponsor!= icons.get("music-note", 16)
+              a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "item_id", filter: item.item_id, filter_fuzzy: null}))
-              = item.track_count
+                span.s-tag--sponsor!= icons.get("music-note", 16)
-            span.s-tag.s-tag__xs
+                = item.track_count
-              span.s-tag--sponsor!= icons.get("compact-disc", 16)
+              span.s-tag.s-tag__xs
-              = item.total_duration
+                span.s-tag--sponsor!= icons.get("compact-disc", 16)
+                = item.total_duration
             a.s-tag.s-tag__xs(href=and({filter_field: "band_name", filter: item.band_name, filter_fuzzy: null}))
               span.s-tag--sponsor!= icons.get("people-tag", 16)
               = item.band_name

pug/artist_grid.pug

@@ -11,12 +11,13 @@ block view
               a.s-tag.s-tag__xs(href=and({arrange: "album", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null}))
                 span.s-tag--sponsor!= icons.get("album", 16)
                 = item.album_count
-              a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null}))
+              if hasFullTrackData
-                span.s-tag--sponsor!= icons.get("music-note", 16)
+                a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_name", filter: item.band_name, filter_fuzzy: null}))
-                = item.track_count
+                  span.s-tag--sponsor!= icons.get("music-note", 16)
-              span.s-tag.s-tag__xs
+                  = item.track_count
-                span.s-tag--sponsor!= icons.get("compact-disc", 16)
+                span.s-tag.s-tag__xs
-                = item.total_duration
+                  span.s-tag--sponsor!= icons.get("compact-disc", 16)
+                  = item.total_duration
               - let labels = item.labels.split("|").map(x => x.replace(/https?:\/\/(.*?)\.bandcamp\.com.*/, "$1"))
               each label in labels
                 a.s-tag.s-tag__xs(href=and({filter_field: "band_url", filter: label, filter_fuzzy: null}))

pug/collection-loaded.pug

@@ -0,0 +1,6 @@
+.s-notice.s-notice__success
+  .d-flex.ai-center.gx16
+    != icons.get("check-circle")
+    .fl-grow1 Imported #{storedItemCount}/#{count} purchases and #{storedTrackCount} tracks.
+  .mt16
+    a.s-link(href=`/${account}/`) Check it out

pug/home.pug

@@ -5,6 +5,8 @@ html
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     title BC Explorer
     link(rel="stylesheet" type="text/css" href="/static/stacks.min.css")
+    script(src="/static/htmx.js")
+    meta(name="htmx-config" content='{"requestClass":"is-loading"}')
   body.themed.theme-system.overflow-y-scroll
     header.s-topbar.ps-sticky.t0
       .s-topbar--container.wmx9
@@ -12,9 +14,18 @@ html
             != icons.get("compass-solid", 24)
             .ml4 BC Explorer
 
-    .mx-auto.wmx9.py24.px16.g24.s-prose
+    .mx-auto.wmx9.py24.px16.g24
-      h1 Select profile
+      .s-prose
-      - const names = select("account", "account").pluck().all()
+        h1 Select profile
-      ul
+        - const names = select("account", "account").pluck().all()
-        each name in names
+        ul
-          li: a(href=`/${name}/`)= name
+          each name in names
+            li: a(href=`/${name}/`)= name
+
+      form.mt32(hx-post="/api/load-collection" hx-target="#results" hx-indicator="#submit-username")
+        h2 Add your profile
+        .d-flex.gy4.fd-column.ps-relative
+          label.s-label(for="username") Bandcamp username
+          input.s-input.wmx3#username(name="account" placeholder="Enter your Bandcamp username here")
+        button.s-btn.s-btn__filled.my16#submit-username Load collection
+        #results.d-flex

pug/includes/layout.pug

@@ -28,6 +28,13 @@ html
       .s-navigation--item.is-loading svg, .s-tag.is-loading svg, .s-sidebarwidget.is-loading svg {
         visibility: hidden;
       }
+      .s-btn__icon.is-loading {
+        --_li-offset: 0.7em;
+        --_il-size: 1.5em;
+      }
+      .s-btn__icon.is-loading svg {
+        display: none;
+      }
       .s-navigation__toggle.s-navigation {
         --_na-item-bg: var(--black-150);
       }
@@ -98,7 +105,7 @@ html
                   strong= filter
               a.s-btn.s-notice--btn(href=and({filter: null, filter_field: null, filter_fuzzy: null})) Clear
           else
-            form.d-flex.ai-stretch.gx8.jc-space-between.baw0
+            form.d-flex.ai-stretch.gx8.jc-space-between.baw0(hx-indicator="#search-submit")
               input.s-input(name="filter" placeholder="Search" autocomplete="off").fl-grow1
               input(type="hidden" name="filter_field" value=
                 ( arrange === "artist" ? "band_name"
@@ -108,11 +115,10 @@ html
               input(type="hidden" name="filter_fuzzy" value="true")
               input(type="hidden" name="arrange" value=arrange)
               input(type="hidden" name="shape" value=shape)
-              button.s-btn.s-btn__xs.s-btn__outlined.s-btn__muted!= icons.get("search")
+              button.s-btn.s-btn__xs.s-btn__icon.s-btn__outlined.s-btn__muted#search-submit(style="height: 38px")!= icons.get("search")
 
           .s-sidebarwidget.wmn3
             .s-sidebarwidget--header Collection
-              a.s-sidebarwidget--action Refresh
             table.s-sidebarwidget--content.s-sidebarwidget__items
               tr.s-sidebarwidget--item
                 th items

pug/includes/tag-status.pug

@@ -1,5 +1,5 @@
 if downloader.total > 0
-  .d-flex.jc-center#tag-download
+  #tag-download
     if !downloader.running && !downloader.outcome
       form.s-notice.s-notice__info.d-flex.ai-center.p8.gx16.pl16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-post="/api/tag-download")
         != icons.get("info-circle")
@@ -8,15 +8,17 @@ if downloader.total > 0
         button.s-btn.s-btn__outlined Download now
 
     else if !downloader.outcome
-      .s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s")
+      .s-notice.p16(role="status" hx-swap="outerHTML" hx-target="#tag-download" hx-get=`/api/tag-download?account=${account}` hx-trigger="every 5s" hx-indicator="null")
         .d-flex.gx16.ai-center
           != icons.get("cloud-download")
-          div Downloading tags...
+          .fl-grow1 Downloading tags...
+          #tag-status-indicator
         p.mt12 You can keep using BC Explorer while this continues in the background.
+        - let percentage = `${Math.round(downloader.processed/downloader.total*100)}%`
         .s-progress.mt16
-          .s-progress--bar(style=`width: ${downloader.processed/downloader.total*100}%`)
+          .s-progress--bar(style=`width: ${percentage}`)
         .d-flex.jc-space-between.fs-fine
-          span= downloader.processed
+          span= percentage
           span= downloader.total
 
     else if downloader.outcome === "Success"

pug/label_grid.pug

@@ -13,15 +13,16 @@ block view
               a.s-tag.s-tag__xs(href=and({arrange: "album", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
                 span.s-tag--sponsor!= icons.get("album", 16)
                 = item.album_count
-              a.s-tag.s-tag__xs(href=and({arrange: "artist", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
+                a.s-tag.s-tag__xs(href=and({arrange: "artist", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
-                span.s-tag--sponsor!= icons.get("people-tag", 16)
+                  span.s-tag--sponsor!= icons.get("people-tag", 16)
-                = item.artist_count
+                  = item.artist_count
-              a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
+              if hasFullTrackData
-                span.s-tag--sponsor!= icons.get("music-note", 16)
+                a.s-tag.s-tag__xs(href=and({arrange: "track", filter_field: "band_url", filter: minBandURL, filter_fuzzy: null}))
-                = item.track_count
+                  span.s-tag--sponsor!= icons.get("music-note", 16)
-              span.s-tag.s-tag__xs
+                  = item.track_count
-                span.s-tag--sponsor!= icons.get("compact-disc", 16)
+                span.s-tag.s-tag__xs
-                = item.total_duration
+                  span.s-tag--sponsor!= icons.get("compact-disc", 16)
+                  = item.total_duration
           each preview in item.previews
             a.d-flex(href=preview.item_url)
               img(loading="lazy" src=preview.item_art_url width=210 height=210 style="height: auto; width: auto; max-height: 70px")

readme.md

@@ -10,14 +10,19 @@ but the idea is you can more easily search your whole collection and play it str
 
 ## how to use
 
+1. npm install
+2. node start.js
+3. http://localhost:2239
+
+## import more reliable statistics
+
+the default data is pretty close, but you can do this to get the exact data
+
 1. log into bandcamp in your browser
 2. in the top right, click the button to view your collection, should take you to a url like https://bandcamp.com/cloudrac3r
 3. open devtools and reload the page
 4. go to network tab and save all as har, save as scripts/account.har
-5. npm install
 6. node scripts/populate-albums-tracks.js
-8. node start.js
-9. http://localhost:2239
 
 ## license
 

routes/app.js

@@ -11,14 +11,15 @@ const loadTags = sync.require("./load-tags")
 const displayCurrency = "NZD"
 const displayCurrencySymbol = "$"
 const currencyExchange = new Map([
-	["USD", 1],
+	["AUD", 0.63],
-	["JPY", 0.0067],
+	["CAD", 0.7],
-	["NZD", 0.57],
+	["CHF", 1.13],
 	["EUR", 1.08],
 	["GBP", 1.3],
-	["CAD", 0.7],
+	["JPY", 0.0067],
 	["NOK", 0.1],
-	["CHF", 1.13]
+	["NZD", 0.57],
+	["USD", 1],
 ])
 
 const sqls = {
@@ -136,23 +137,23 @@ router.get("/:account/", defineEventHandler({
 			account,
 			query,
 			count: {
-				total: db.prepare("SELECT count(*) FROM item").pluck().get(),
+				total: db.prepare("SELECT count(*) FROM item WHERE account = ?").pluck().get(account),
-				runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track").pluck().get(),
+				runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track WHERE account = ?").pluck().get(account),
-				albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album'").pluck().get(),
+				albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album' AND account = ?").pluck().get(account),
-				singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track'").pluck().get(),
+				singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track' AND account = ?").pluck().get(account),
-				free: db.prepare("SELECT count(*) FROM item WHERE price = 0").pluck().get(),
+				free: db.prepare("SELECT count(*) FROM item WHERE price = 0 AND account = ?").pluck().get(account),
-				paid: db.prepare("SELECT count(*) FROM item WHERE price > 0").pluck().get(),
+				paid: db.prepare("SELECT count(*) FROM item WHERE price > 0 AND account = ?").pluck().get(account),
-				tracks: db.prepare("SELECT count(*) FROM track").pluck().get(),
+				tracks: db.prepare("SELECT count(*) FROM track WHERE account = ?").pluck().get(account),
-				avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (item_id) WHERE item_type = 'album' GROUP BY item_id)").pluck().get()*10)/10,
+				avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (account, item_id) WHERE item_type = 'album' AND account = ? GROUP BY item_id)").pluck().get(account)*10)/10,
-				tags: db.prepare("SELECT count(*) FROM item_tag").pluck().get(),
+				tags: db.prepare("SELECT count(*) FROM item_tag WHERE account = ?").pluck().get(account),
-				avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag GROUP BY item_id)").pluck().get()*10)/10,
+				avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag WHERE account = ? GROUP BY item_id)").pluck().get(account)*10)/10,
-				lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag GROUP BY tag HAVING count(*) = 1)").pluck().get(),
+				lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag WHERE account = ? GROUP BY tag HAVING count(*) = 1)").pluck().get(account),
-				value: Math.round(select("item", ["currency", "price"]).all().map(c => {
+				value: Math.round(select("item", ["currency", "price"], {account}).all().map(c => {
 					return (currencyExchange.get(c.currency) || 0.6) * c.price / (currencyExchange.get(displayCurrency) || 1) / 10
 				}).reduce((a, c) => a + c, 0)) * 10,
 				displayCurrency,
 				displayCurrencySymbol,
-				taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get()
+				taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item WHERE account = ? group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get(account)
 			}
 		}
 		if (mode === "artist_grid") {
@@ -163,6 +164,7 @@ router.get("/:account/", defineEventHandler({
 			locals.downloadManager = loadTags.downloadManager
 			locals.downloader = loadTags.downloadManager.check(account)
 		}
+		locals.hasFullTrackData = locals.count.tracks > locals.count.total
 		return pugSync.render(event, `${arrange}_${shape}.pug`, locals)
 	}
 }))

routes/load-collection.js

@@ -0,0 +1,93 @@
+// @ts-check
+
+const assert = require("assert/strict")
+const fs = require("fs")
+const sqlite = require("better-sqlite3")
+const domino = require("domino")
+const {defineEventHandler, readValidatedBody} = require("h3")
+const {z} = require("zod")
+
+const {sync, db, router} = require("../passthrough")
+const pugSync = sync.require("../pug-sync")
+
+async function loadCollection(inputUsername) {
+	assert.match(inputUsername, /^[a-z0-9_-]+$/)
+	const html = await fetch(`https://bandcamp.com/${inputUsername}`).then(res => res.text())
+	const doc = domino.createDocument(html)
+
+	const first = doc.querySelector(".collection-item-container")
+	assert(first)
+	const token = first.getAttribute("data-token")
+	assert(token)
+
+	const tokenParts = token.split(":")
+	tokenParts[0] = String(+tokenParts[0] + 1) // ensure we get the first item
+	const customToken = tokenParts.join(":")
+
+	const pagedataText = doc.querySelector("#pagedata")?.getAttribute("data-blob")
+	assert(pagedataText)
+	const pagedata = JSON.parse(pagedataText)
+	const fan_id = pagedata.fan_data.fan_id
+	const account = pagedata.fan_data.username
+	const count = pagedata.collection_data.item_count
+
+	const items = await fetch("https://bandcamp.com/api/fancollection/1/collection_items", {
+		method: "POST",
+		body: JSON.stringify({
+			fan_id,
+			older_than_token: customToken,
+			count
+		})
+	}).then(res => res.json())
+
+	db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id)
+
+	const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
+	const upsert_columns = ["added", "updated", "purchased"]
+	const preparedItem = db.prepare(`INSERT INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`)
+	db.transaction(() => {
+		for (const item of items.items) {
+			preparedItem.run({
+				account,
+				...item,
+				purchased: new Date(item.purchased).getTime(),
+				added: new Date(item.added).getTime(),
+				updated: new Date(item.updated).getTime()
+			})
+		}
+	})()
+	const storedItemCount = db.prepare("SELECT count(*) AS count FROM item WHERE account = ?").pluck().get(account)
+
+	const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration)")
+	db.transaction(() => {
+		for (const [key, tracklist] of Object.entries(items.tracklists)) {
+			assert.match(key[0], /[at]/)
+			for (const track of tracklist) {
+				preparedTrack.run({
+					account,
+					item_id: key.slice(1),
+					track_id: track.id,
+					...track
+				})
+			}
+		}
+	})()
+	const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track WHERE account = ?").pluck().get(account)
+
+	return {
+		storedItemCount,
+		storedTrackCount,
+		count,
+		account
+	}
+}
+
+const schema = z.object({
+	account: z.string()
+})
+
+router.post("/api/load-collection", defineEventHandler(async event => {
+	const {account} = await readValidatedBody(event, schema.parse)
+	const result = await loadCollection(account)
+	return pugSync.render(event, "collection-loaded.pug", result)
+}))

routes/load-tags.js

@@ -7,14 +7,15 @@ const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3")
 const {sync, db, from, router} = require("../passthrough")
 const pugSync = sync.require("../pug-sync")
 
-const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
+const insertTag = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
+const insertTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (?, ?, ?, ?, ?, ?, ?)")
 
 class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
 	constructor(account) {
 		super()
 		this.account = account
 		this.processed = 0
-		this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
+		this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url, band_name FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
 		this.total = this.untaggedItems.length
 		this.running = false
 		this.outcome = null
@@ -24,16 +25,46 @@ class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
 		if (this.running) return
 		this.running = true
 		try {
-			for (const {account, item_id, item_title, item_url} of this.untaggedItems) {
+			for (const {account, item_id, item_title, item_url, band_name} of this.untaggedItems) {
-				const html = await fetch(item_url).then(res => res.text())
+				const res = await fetch(item_url)
+
+				// delete unreachable items, otherwise it will perpetually try to download tags for them
+				if (res.status === 404) {
+					db.prepare("DELETE FROM item WHERE account = ? AND item_id = ?").run(account, item_id)
+					this.processed++
+					continue
+				}
+
+				const html = await res.text()
 				const doc = domino.createDocument(html)
+
 				// @ts-ignore
 				const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
 				db.transaction(() => {
 					for (const tag of tags) {
-						i.run(account, item_id, tag)
+						insertTag.run(account, item_id, tag)
 					}
 				})()
+
+				// @ts-ignore
+				const tracks = [...doc.querySelectorAll(".track_row_view").cache]
+				db.transaction(() => {
+					for (const track of tracks) {
+						const track_number = parseInt(track.querySelector(".track_number").textContent)
+						let title = track.querySelector(".track-title").textContent
+						let artist = band_name
+						const match = title.match(/^([^-]*) - (.*)$/)
+						if (match) {
+							artist = match[1]
+							title = match[2]
+						}
+						const duration = track.querySelector(".time").textContent.split(":").reverse().reduce((a, c, i) => 60**i * c + a, 0)
+						console.log(track_number, title, artist, duration)
+						if (!track_number || !title || !artist || !duration) continue
+						insertTrack.run(account, item_id, track_number, title, artist, track_number, duration)
+					}
+				})()
+
 				this.processed++
 			}
 			this.outcome = "Success"

start.js

@@ -35,6 +35,7 @@ pugSync.createRoute(router, "/", "home.pug")
 // Routes
 
 sync.require("./routes/app")
+sync.require("./routes/load-collection")
 
 // Files