cadence/bc-explorer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Load huge collections pagewise

Browse source
This commit is contained in:
Cadence Ember 2025-04-17 15:26:06 +12:00
parent b812b8b17b
commit 12f8b8ffd0
1 changed files with 27 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

25
routes/load-collection.js
View file

@ -39,14 +39,31 @@ async function loadCollection(inputUsername) {
const account = pagedata.fan_data.username const account = pagedata.fan_data.username
const count = pagedata.collection_data.item_count const count = pagedata.collection_data.item_count
const items = await fetch("https://bandcamp.com/api/fancollection/1/collection_items", { let items = []
const tracklists = {}
/** @type {string?} */
let lastToken = customToken
let remaining = count
while (lastToken) {
const pageSize = remaining > 2000 ? 2000 : remaining
console.log(`load page of collection - ${account} ${lastToken} - load ${pageSize} - progress ${items.length}/${count}`)
const root = await fetch("https://bandcamp.com/api/fancollection/1/collection_items", {
method: "POST", method: "POST",
body: JSON.stringify({ body: JSON.stringify({
fan_id, fan_id,
older_than_token: customToken, older_than_token: customToken,
count count: pageSize
}) })
}).then(res => res.json()) }).then(res => res.json())
// counter
remaining -= root.items.length
// add returned items
items = items.concat(root.items)
Object.assign(tracklists, root.tracklists)
// last token for next page
lastToken = root.last_token
if (items.length === count) lastToken = null
}
db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id) db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id)
@ -54,7 +71,7 @@ async function loadCollection(inputUsername) {
const upsert_columns = ["added", "updated", "purchased", "why", "featured_track", "also_collected_count", "featured_track_title", "featured_track_number", "featured_track_duration"] const upsert_columns = ["added", "updated", "purchased", "why", "featured_track", "also_collected_count", "featured_track_title", "featured_track_number", "featured_track_duration"]
const preparedItem = db.prepare(`INSERT INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`) const preparedItem = db.prepare(`INSERT INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`)
db.transaction(() => { db.transaction(() => {
for (const item of items.items) { for (const item of items) {
if (!item.tralbum_type.match(/[at]/)) continue // p=product and s=subscription not supported if (!item.tralbum_type.match(/[at]/)) continue // p=product and s=subscription not supported
try { try {
preparedItem.run({ preparedItem.run({
@ -79,7 +96,7 @@ async function loadCollection(inputUsername) {
const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration)") const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration)")
db.transaction(() => { db.transaction(() => {
for (const [key, tracklist] of Object.entries(items.tracklists)) { for (const [key, tracklist] of Object.entries(tracklists)) {
if (!key[0].match(/[at]/)) continue if (!key[0].match(/[at]/)) continue
for (const track of tracklist) { for (const track of tracklist) {
preparedTrack.run({ preparedTrack.run({