Import accounts from home page
This commit is contained in:
parent
cba0552ed0
commit
cf6310b89a
12 changed files with 222 additions and 62 deletions
|
|
@ -11,14 +11,15 @@ const loadTags = sync.require("./load-tags")
|
|||
const displayCurrency = "NZD"
|
||||
const displayCurrencySymbol = "$"
|
||||
const currencyExchange = new Map([
|
||||
["USD", 1],
|
||||
["JPY", 0.0067],
|
||||
["NZD", 0.57],
|
||||
["AUD", 0.63],
|
||||
["CAD", 0.7],
|
||||
["CHF", 1.13],
|
||||
["EUR", 1.08],
|
||||
["GBP", 1.3],
|
||||
["CAD", 0.7],
|
||||
["JPY", 0.0067],
|
||||
["NOK", 0.1],
|
||||
["CHF", 1.13]
|
||||
["NZD", 0.57],
|
||||
["USD", 1],
|
||||
])
|
||||
|
||||
const sqls = {
|
||||
|
|
@ -136,23 +137,23 @@ router.get("/:account/", defineEventHandler({
|
|||
account,
|
||||
query,
|
||||
count: {
|
||||
total: db.prepare("SELECT count(*) FROM item").pluck().get(),
|
||||
runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track").pluck().get(),
|
||||
albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album'").pluck().get(),
|
||||
singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track'").pluck().get(),
|
||||
free: db.prepare("SELECT count(*) FROM item WHERE price = 0").pluck().get(),
|
||||
paid: db.prepare("SELECT count(*) FROM item WHERE price > 0").pluck().get(),
|
||||
tracks: db.prepare("SELECT count(*) FROM track").pluck().get(),
|
||||
avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (item_id) WHERE item_type = 'album' GROUP BY item_id)").pluck().get()*10)/10,
|
||||
tags: db.prepare("SELECT count(*) FROM item_tag").pluck().get(),
|
||||
avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag GROUP BY item_id)").pluck().get()*10)/10,
|
||||
lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag GROUP BY tag HAVING count(*) = 1)").pluck().get(),
|
||||
value: Math.round(select("item", ["currency", "price"]).all().map(c => {
|
||||
total: db.prepare("SELECT count(*) FROM item WHERE account = ?").pluck().get(account),
|
||||
runtime: db.prepare("SELECT iif(sum(duration) > 86400, cast(total(duration)/86400 AS INTEGER) || 'd ' || cast(total(duration)/3600%24 AS INTEGER) || 'h', cast(total(duration)/3600 AS INTEGER) || 'h') FROM track WHERE account = ?").pluck().get(account),
|
||||
albums: db.prepare("SELECT count(*) FROM item WHERE item_type = 'album' AND account = ?").pluck().get(account),
|
||||
singles: db.prepare("SELECT count(*) FROM item WHERE item_type = 'track' AND account = ?").pluck().get(account),
|
||||
free: db.prepare("SELECT count(*) FROM item WHERE price = 0 AND account = ?").pluck().get(account),
|
||||
paid: db.prepare("SELECT count(*) FROM item WHERE price > 0 AND account = ?").pluck().get(account),
|
||||
tracks: db.prepare("SELECT count(*) FROM track WHERE account = ?").pluck().get(account),
|
||||
avgTracks: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM track INNER JOIN item USING (account, item_id) WHERE item_type = 'album' AND account = ? GROUP BY item_id)").pluck().get(account)*10)/10,
|
||||
tags: db.prepare("SELECT count(*) FROM item_tag WHERE account = ?").pluck().get(account),
|
||||
avgTags: Math.round(db.prepare("SELECT avg(count) FROM (SELECT count(*) AS count FROM item_tag WHERE account = ? GROUP BY item_id)").pluck().get(account)*10)/10,
|
||||
lonelyTags: db.prepare("SELECT count(*) FROM (SELECT tag FROM item_tag WHERE account = ? GROUP BY tag HAVING count(*) = 1)").pluck().get(account),
|
||||
value: Math.round(select("item", ["currency", "price"], {account}).all().map(c => {
|
||||
return (currencyExchange.get(c.currency) || 0.6) * c.price / (currencyExchange.get(displayCurrency) || 1) / 10
|
||||
}).reduce((a, c) => a + c, 0)) * 10,
|
||||
displayCurrency,
|
||||
displayCurrencySymbol,
|
||||
taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get()
|
||||
taste: db.prepare("with popularity (a) as (select avg(also_collected_count) from item WHERE account = ? group by band_url) select sum(iif(a >= 0 and a < 20, 1, 0)) as cold, sum(iif(a >= 20 and a < 200, 1, 0)) as warm, sum(iif(a >= 200 and a < 2000, 1, 0)) as hot, sum(iif(a >= 2000, 1, 0)) as supernova from popularity").raw().get(account)
|
||||
}
|
||||
}
|
||||
if (mode === "artist_grid") {
|
||||
|
|
@ -163,6 +164,7 @@ router.get("/:account/", defineEventHandler({
|
|||
locals.downloadManager = loadTags.downloadManager
|
||||
locals.downloader = loadTags.downloadManager.check(account)
|
||||
}
|
||||
locals.hasFullTrackData = locals.count.tracks > locals.count.total
|
||||
return pugSync.render(event, `${arrange}_${shape}.pug`, locals)
|
||||
}
|
||||
}))
|
||||
|
|
|
|||
93
routes/load-collection.js
Normal file
93
routes/load-collection.js
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
// @ts-check
|
||||
|
||||
const assert = require("assert/strict")
|
||||
const fs = require("fs")
|
||||
const sqlite = require("better-sqlite3")
|
||||
const domino = require("domino")
|
||||
const {defineEventHandler, readValidatedBody} = require("h3")
|
||||
const {z} = require("zod")
|
||||
|
||||
const {sync, db, router} = require("../passthrough")
|
||||
const pugSync = sync.require("../pug-sync")
|
||||
|
||||
async function loadCollection(inputUsername) {
|
||||
assert.match(inputUsername, /^[a-z0-9_-]+$/)
|
||||
const html = await fetch(`https://bandcamp.com/${inputUsername}`).then(res => res.text())
|
||||
const doc = domino.createDocument(html)
|
||||
|
||||
const first = doc.querySelector(".collection-item-container")
|
||||
assert(first)
|
||||
const token = first.getAttribute("data-token")
|
||||
assert(token)
|
||||
|
||||
const tokenParts = token.split(":")
|
||||
tokenParts[0] = String(+tokenParts[0] + 1) // ensure we get the first item
|
||||
const customToken = tokenParts.join(":")
|
||||
|
||||
const pagedataText = doc.querySelector("#pagedata")?.getAttribute("data-blob")
|
||||
assert(pagedataText)
|
||||
const pagedata = JSON.parse(pagedataText)
|
||||
const fan_id = pagedata.fan_data.fan_id
|
||||
const account = pagedata.fan_data.username
|
||||
const count = pagedata.collection_data.item_count
|
||||
|
||||
const items = await fetch("https://bandcamp.com/api/fancollection/1/collection_items", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
fan_id,
|
||||
older_than_token: customToken,
|
||||
count
|
||||
})
|
||||
}).then(res => res.json())
|
||||
|
||||
db.prepare("INSERT OR IGNORE INTO account (account, fan_id) VALUES (?, ?)").run(account, fan_id)
|
||||
|
||||
const columns = ["account", "item_id", "item_type", "band_id", "added", "updated", "purchased", "featured_track", "why", "also_collected_count", "item_title", "item_url", "item_art_url", "band_name", "band_url", "featured_track_title", "featured_track_number", "featured_track_duration", "album_id", "album_title", "price", "currency", "label", "label_id"]
|
||||
const upsert_columns = ["added", "updated", "purchased"]
|
||||
const preparedItem = db.prepare(`INSERT INTO item (${columns.join(", ")}) VALUES (${columns.map(x => "@" + x).join(", ")}) ON CONFLICT DO UPDATE SET ${upsert_columns.map(x => `${x} = @${x}`).join(", ")}`)
|
||||
db.transaction(() => {
|
||||
for (const item of items.items) {
|
||||
preparedItem.run({
|
||||
account,
|
||||
...item,
|
||||
purchased: new Date(item.purchased).getTime(),
|
||||
added: new Date(item.added).getTime(),
|
||||
updated: new Date(item.updated).getTime()
|
||||
})
|
||||
}
|
||||
})()
|
||||
const storedItemCount = db.prepare("SELECT count(*) AS count FROM item WHERE account = ?").pluck().get(account)
|
||||
|
||||
const preparedTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (@account, @item_id, @track_id, @title, @artist, @track_number, @duration)")
|
||||
db.transaction(() => {
|
||||
for (const [key, tracklist] of Object.entries(items.tracklists)) {
|
||||
assert.match(key[0], /[at]/)
|
||||
for (const track of tracklist) {
|
||||
preparedTrack.run({
|
||||
account,
|
||||
item_id: key.slice(1),
|
||||
track_id: track.id,
|
||||
...track
|
||||
})
|
||||
}
|
||||
}
|
||||
})()
|
||||
const storedTrackCount = db.prepare("SELECT count(*) AS count FROM track WHERE account = ?").pluck().get(account)
|
||||
|
||||
return {
|
||||
storedItemCount,
|
||||
storedTrackCount,
|
||||
count,
|
||||
account
|
||||
}
|
||||
}
|
||||
|
||||
const schema = z.object({
|
||||
account: z.string()
|
||||
})
|
||||
|
||||
router.post("/api/load-collection", defineEventHandler(async event => {
|
||||
const {account} = await readValidatedBody(event, schema.parse)
|
||||
const result = await loadCollection(account)
|
||||
return pugSync.render(event, "collection-loaded.pug", result)
|
||||
}))
|
||||
|
|
@ -7,14 +7,15 @@ const {getValidatedQuery, readValidatedBody, defineEventHandler} = require("h3")
|
|||
const {sync, db, from, router} = require("../passthrough")
|
||||
const pugSync = sync.require("../pug-sync")
|
||||
|
||||
const i = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
|
||||
const insertTag = db.prepare("INSERT OR IGNORE INTO item_tag (account, item_id, tag) VALUES (?, ?, ?)")
|
||||
const insertTrack = db.prepare("INSERT OR IGNORE INTO track (account, item_id, track_id, title, artist, track_number, duration) VALUES (?, ?, ?, ?, ?, ?, ?)")
|
||||
|
||||
class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
|
||||
constructor(account) {
|
||||
super()
|
||||
this.account = account
|
||||
this.processed = 0
|
||||
this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
|
||||
this.untaggedItems = db.prepare("SELECT account, item_id, item_title, item_url, band_name FROM item LEFT JOIN item_tag USING (account, item_id) WHERE account = ? AND tag IS NULL").all(account)
|
||||
this.total = this.untaggedItems.length
|
||||
this.running = false
|
||||
this.outcome = null
|
||||
|
|
@ -24,16 +25,46 @@ class TagDownloader extends sync.reloadClassMethods(() => TagDownloader) {
|
|||
if (this.running) return
|
||||
this.running = true
|
||||
try {
|
||||
for (const {account, item_id, item_title, item_url} of this.untaggedItems) {
|
||||
const html = await fetch(item_url).then(res => res.text())
|
||||
for (const {account, item_id, item_title, item_url, band_name} of this.untaggedItems) {
|
||||
const res = await fetch(item_url)
|
||||
|
||||
// delete unreachable items, otherwise it will perpetually try to download tags for them
|
||||
if (res.status === 404) {
|
||||
db.prepare("DELETE FROM item WHERE account = ? AND item_id = ?").run(account, item_id)
|
||||
this.processed++
|
||||
continue
|
||||
}
|
||||
|
||||
const html = await res.text()
|
||||
const doc = domino.createDocument(html)
|
||||
|
||||
// @ts-ignore
|
||||
const tags = [...doc.querySelectorAll(".tag").cache].map(e => e.textContent)
|
||||
db.transaction(() => {
|
||||
for (const tag of tags) {
|
||||
i.run(account, item_id, tag)
|
||||
insertTag.run(account, item_id, tag)
|
||||
}
|
||||
})()
|
||||
|
||||
// @ts-ignore
|
||||
const tracks = [...doc.querySelectorAll(".track_row_view").cache]
|
||||
db.transaction(() => {
|
||||
for (const track of tracks) {
|
||||
const track_number = parseInt(track.querySelector(".track_number").textContent)
|
||||
let title = track.querySelector(".track-title").textContent
|
||||
let artist = band_name
|
||||
const match = title.match(/^([^-]*) - (.*)$/)
|
||||
if (match) {
|
||||
artist = match[1]
|
||||
title = match[2]
|
||||
}
|
||||
const duration = track.querySelector(".time").textContent.split(":").reverse().reduce((a, c, i) => 60**i * c + a, 0)
|
||||
console.log(track_number, title, artist, duration)
|
||||
if (!track_number || !title || !artist || !duration) continue
|
||||
insertTrack.run(account, item_id, track_number, title, artist, track_number, duration)
|
||||
}
|
||||
})()
|
||||
|
||||
this.processed++
|
||||
}
|
||||
this.outcome = "Success"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue