ingest now downloads borders

This commit is contained in:
jane 2022-04-17 00:24:59 -04:00
parent 19a94f927e
commit 5e0c7153b9
4 changed files with 109 additions and 5 deletions

View file

@ -15,6 +15,7 @@
"dotenv": "^16.0.0",
"next": "12.1.4",
"next-auth": "^4.3.1",
"node-fetch": "^3.2.3",
"prisma": "^3.12.0",
"react": "18.0.0",
"react-dom": "18.0.0"

View file

@ -8,6 +8,7 @@ specifiers:
eslint-config-next: 12.1.4
next: 12.1.4
next-auth: ^4.3.1
node-fetch: ^3.2.3
prisma: ^3.12.0
react: 18.0.0
react-dom: 18.0.0
@ -18,6 +19,7 @@ dependencies:
dotenv: 16.0.0
next: 12.1.4_react-dom@18.0.0+react@18.0.0
next-auth: 4.3.1_react-dom@18.0.0+react@18.0.0
node-fetch: 3.2.3
prisma: 3.12.0
react: 18.0.0
react-dom: 18.0.0_react@18.0.0
@ -492,6 +494,11 @@ packages:
resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==}
dev: true
/data-uri-to-buffer/4.0.0:
resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==}
engines: {node: '>= 12'}
dev: false
/debug/2.6.9:
resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
dependencies:
@ -871,6 +878,14 @@ packages:
reusify: 1.0.4
dev: true
/fetch-blob/3.1.5:
resolution: {integrity: sha512-N64ZpKqoLejlrwkIAnb9iLSA3Vx/kjgzpcDhygcqJ2KKjky8nCgUQ+dzXtbrLaWZGZNmNfQTsiQ0weZ1svglHg==}
engines: {node: ^12.20 || >= 14.13}
dependencies:
node-domexception: 1.0.0
web-streams-polyfill: 3.2.1
dev: false
/file-entry-cache/6.0.1:
resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==}
engines: {node: ^10.12.0 || >=12.0.0}
@ -904,6 +919,13 @@ packages:
resolution: {integrity: sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==}
dev: true
/formdata-polyfill/4.0.10:
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
dependencies:
fetch-blob: 3.1.5
dev: false
/fs.realpath/1.0.0:
resolution: {integrity: sha1-FQStJSMVjKpA20onh8sBQRmU6k8=}
dev: true
@ -1348,6 +1370,20 @@ packages:
- babel-plugin-macros
dev: false
/node-domexception/1.0.0:
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
engines: {node: '>=10.5.0'}
dev: false
/node-fetch/3.2.3:
resolution: {integrity: sha512-AXP18u4pidSZ1xYXRDPY/8jdv3RAozIt/WLNR/MBGZAz+xjtlr90RvCnsvHQRiXyWliZF/CpytExp32UU67/SA==}
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
dependencies:
data-uri-to-buffer: 4.0.0
fetch-blob: 3.1.5
formdata-polyfill: 4.0.10
dev: false
/oauth/0.9.15:
resolution: {integrity: sha1-vR/vr2hslrdUda7VGWQS/2DPucE=}
dev: false
@ -1830,6 +1866,11 @@ packages:
resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==}
dev: true
/web-streams-polyfill/3.2.1:
resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
engines: {node: '>= 8'}
dev: false
/which-boxed-primitive/1.0.2:
resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==}
dependencies:

View file

@ -66,6 +66,9 @@ model ApplicationUserData {
}
model BorderImage {
id Int @id @default(autoincrement())
imageName String @unique
id Int @id @default(autoincrement())
imageName String @unique
appId Int?
appName String?
borderName String?
}

View file

@ -1,4 +1,3 @@
console.log("a");
const fs = require("fs");
const path = require("path");
const Prisma = require("@prisma/client");
@ -14,9 +13,10 @@ const folder = path.join(
cwd.includes("util") ? ".." : "",
"public/images"
);
const list = fs.readdirSync(folder);
(async () => {
let catalogue = async () => {
let numAdded = 0;
for (let item of list) {
// console.log(item);
@ -31,6 +31,7 @@ const list = fs.readdirSync(folder);
data: {
id: 0,
imageName: item,
borderName: "Default",
},
});
numAdded++;
@ -46,4 +47,62 @@ const list = fs.readdirSync(folder);
}
}
console.log(`Processed ${numAdded} new images.`);
})();
};
let download = async () => {
const fetch = await import("node-fetch");
const filePath = path.join(
cwd,
cwd.includes("util") ? ".." : "",
"util/border_data.json"
);
const json_data = JSON.parse(fs.readFileSync(filePath));
let numAdded = 0;
for (let value of json_data) {
if (!value) continue;
const filename_regex =
/https:\/\/cdn.akamai.steamstatic.com\/steamcommunity\/public\/images\/items\/\d+\/(.+\.png)/gi;
const filename = filename_regex.exec(value.borderURL)[1];
if (!list.includes(filename)) {
console.log("N", filename);
let data = await fetch.default(value.borderURL);
data.body.pipe(fs.createWriteStream(path.join(folder, filename)));
}
const result = await prisma.borderImage.findFirst({
where: {
imageName: filename,
},
});
if (!result.appId || !result.appName || !result.borderName) {
const added = await prisma.borderImage.upsert({
create: {
imageName: filename,
appId: value.appInfo.appid,
appName: value.appInfo.name,
borderName: value.name,
},
update: {
imageName: filename,
appId: value.appInfo.appid,
appName: value.appInfo.name,
borderName: value.name,
},
where: {
imageName: filename,
},
});
console.log(added);
numAdded++;
}
}
console.log(`Upserted ${numAdded} values.`);
};
download().then(() => {
catalogue().then(() => {
console.log("done.");
});
});