From 5e0c7153b9c2388cc269c3553d88e3d501c4ab78 Mon Sep 17 00:00:00 2001 From: Jane Petrovna Date: Sun, 17 Apr 2022 00:24:59 -0400 Subject: [PATCH] ingest now downloads borders --- package.json | 1 + pnpm-lock.yaml | 41 ++++++++++++++++++++++++++++ prisma/schema.prisma | 7 +++-- util/ingest.js | 65 ++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 109 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 5e3276f..5b9637a 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "dotenv": "^16.0.0", "next": "12.1.4", "next-auth": "^4.3.1", + "node-fetch": "^3.2.3", "prisma": "^3.12.0", "react": "18.0.0", "react-dom": "18.0.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index db1d77a..be0e922 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,7 @@ specifiers: eslint-config-next: 12.1.4 next: 12.1.4 next-auth: ^4.3.1 + node-fetch: ^3.2.3 prisma: ^3.12.0 react: 18.0.0 react-dom: 18.0.0 @@ -18,6 +19,7 @@ dependencies: dotenv: 16.0.0 next: 12.1.4_react-dom@18.0.0+react@18.0.0 next-auth: 4.3.1_react-dom@18.0.0+react@18.0.0 + node-fetch: 3.2.3 prisma: 3.12.0 react: 18.0.0 react-dom: 18.0.0_react@18.0.0 @@ -492,6 +494,11 @@ packages: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} dev: true + /data-uri-to-buffer/4.0.0: + resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==} + engines: {node: '>= 12'} + dev: false + /debug/2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} dependencies: @@ -871,6 +878,14 @@ packages: reusify: 1.0.4 dev: true + /fetch-blob/3.1.5: + resolution: {integrity: sha512-N64ZpKqoLejlrwkIAnb9iLSA3Vx/kjgzpcDhygcqJ2KKjky8nCgUQ+dzXtbrLaWZGZNmNfQTsiQ0weZ1svglHg==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: false + /file-entry-cache/6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -904,6 +919,13 @@ packages: resolution: {integrity: sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==} dev: true + /formdata-polyfill/4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.1.5 + dev: false + /fs.realpath/1.0.0: resolution: {integrity: sha1-FQStJSMVjKpA20onh8sBQRmU6k8=} dev: true @@ -1348,6 +1370,20 @@ packages: - babel-plugin-macros dev: false + /node-domexception/1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + + /node-fetch/3.2.3: + resolution: {integrity: sha512-AXP18u4pidSZ1xYXRDPY/8jdv3RAozIt/WLNR/MBGZAz+xjtlr90RvCnsvHQRiXyWliZF/CpytExp32UU67/SA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.0 + fetch-blob: 3.1.5 + formdata-polyfill: 4.0.10 + dev: false + /oauth/0.9.15: resolution: {integrity: sha1-vR/vr2hslrdUda7VGWQS/2DPucE=} dev: false @@ -1830,6 +1866,11 @@ packages: resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==} dev: true + /web-streams-polyfill/3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: false + /which-boxed-primitive/1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} dependencies: diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f37715e..5b7f6f8 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -66,6 +66,9 @@ model ApplicationUserData { } model BorderImage { - id Int @id @default(autoincrement()) - imageName String @unique + id Int @id @default(autoincrement()) + imageName String @unique + appId Int? + appName String? + borderName String? } diff --git a/util/ingest.js b/util/ingest.js index cd863c5..951271a 100644 --- a/util/ingest.js +++ b/util/ingest.js @@ -1,4 +1,3 @@ -console.log("a"); const fs = require("fs"); const path = require("path"); const Prisma = require("@prisma/client"); @@ -14,9 +13,10 @@ const folder = path.join( cwd.includes("util") ? ".." : "", "public/images" ); + const list = fs.readdirSync(folder); -(async () => { +let catalogue = async () => { let numAdded = 0; for (let item of list) { // console.log(item); @@ -31,6 +31,7 @@ const list = fs.readdirSync(folder); data: { id: 0, imageName: item, + borderName: "Default", }, }); numAdded++; @@ -46,4 +47,62 @@ const list = fs.readdirSync(folder); } } console.log(`Processed ${numAdded} new images.`); -})(); +}; + +let download = async () => { + const fetch = await import("node-fetch"); + const filePath = path.join( + cwd, + cwd.includes("util") ? ".." : "", + "util/border_data.json" + ); + const json_data = JSON.parse(fs.readFileSync(filePath)); + let numAdded = 0; + for (let value of json_data) { + if (!value) continue; + const filename_regex = + /https:\/\/cdn.akamai.steamstatic.com\/steamcommunity\/public\/images\/items\/\d+\/(.+\.png)/gi; + const filename = filename_regex.exec(value.borderURL)[1]; + + if (!list.includes(filename)) { + console.log("N", filename); + let data = await fetch.default(value.borderURL); + data.body.pipe(fs.createWriteStream(path.join(folder, filename))); + } + + const result = await prisma.borderImage.findFirst({ + where: { + imageName: filename, + }, + }); + + if (!result.appId || !result.appName || !result.borderName) { + const added = await prisma.borderImage.upsert({ + create: { + imageName: filename, + appId: value.appInfo.appid, + appName: value.appInfo.name, + borderName: value.name, + }, + update: { + imageName: filename, + appId: value.appInfo.appid, + appName: value.appInfo.name, + borderName: value.name, + }, + where: { + imageName: filename, + }, + }); + console.log(added); + numAdded++; + } + } + console.log(`Upserted ${numAdded} values.`); +}; + +download().then(() => { + catalogue().then(() => { + console.log("done."); + }); +});