ingest now downloads borders

package.json

@@ -15,6 +15,7 @@
     "dotenv": "^16.0.0",
     "next": "12.1.4",
     "next-auth": "^4.3.1",
+    "node-fetch": "^3.2.3",
     "prisma": "^3.12.0",
     "react": "18.0.0",
     "react-dom": "18.0.0"

pnpm-lock.yaml

@@ -8,6 +8,7 @@ specifiers:
   eslint-config-next: 12.1.4
   next: 12.1.4
   next-auth: ^4.3.1
+  node-fetch: ^3.2.3
   prisma: ^3.12.0
   react: 18.0.0
   react-dom: 18.0.0
@@ -18,6 +19,7 @@ dependencies:
   dotenv: 16.0.0
   next: 12.1.4_react-dom@18.0.0+react@18.0.0
   next-auth: 4.3.1_react-dom@18.0.0+react@18.0.0
+  node-fetch: 3.2.3
   prisma: 3.12.0
   react: 18.0.0
   react-dom: 18.0.0_react@18.0.0
@@ -492,6 +494,11 @@ packages:
     resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==}
     dev: true
 
+  /data-uri-to-buffer/4.0.0:
+    resolution: {integrity: sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==}
+    engines: {node: '>= 12'}
+    dev: false
+
   /debug/2.6.9:
     resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==}
     dependencies:
@@ -871,6 +878,14 @@ packages:
       reusify: 1.0.4
     dev: true
 
+  /fetch-blob/3.1.5:
+    resolution: {integrity: sha512-N64ZpKqoLejlrwkIAnb9iLSA3Vx/kjgzpcDhygcqJ2KKjky8nCgUQ+dzXtbrLaWZGZNmNfQTsiQ0weZ1svglHg==}
+    engines: {node: ^12.20 || >= 14.13}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 3.2.1
+    dev: false
+
   /file-entry-cache/6.0.1:
     resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==}
     engines: {node: ^10.12.0 || >=12.0.0}
@@ -904,6 +919,13 @@ packages:
     resolution: {integrity: sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==}
     dev: true
 
+  /formdata-polyfill/4.0.10:
+    resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
+    engines: {node: '>=12.20.0'}
+    dependencies:
+      fetch-blob: 3.1.5
+    dev: false
+
   /fs.realpath/1.0.0:
     resolution: {integrity: sha1-FQStJSMVjKpA20onh8sBQRmU6k8=}
     dev: true
@@ -1348,6 +1370,20 @@ packages:
       - babel-plugin-macros
     dev: false
 
+  /node-domexception/1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: false
+
+  /node-fetch/3.2.3:
+    resolution: {integrity: sha512-AXP18u4pidSZ1xYXRDPY/8jdv3RAozIt/WLNR/MBGZAz+xjtlr90RvCnsvHQRiXyWliZF/CpytExp32UU67/SA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      data-uri-to-buffer: 4.0.0
+      fetch-blob: 3.1.5
+      formdata-polyfill: 4.0.10
+    dev: false
+
   /oauth/0.9.15:
     resolution: {integrity: sha1-vR/vr2hslrdUda7VGWQS/2DPucE=}
     dev: false
@@ -1830,6 +1866,11 @@ packages:
     resolution: {integrity: sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA==}
     dev: true
 
+  /web-streams-polyfill/3.2.1:
+    resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==}
+    engines: {node: '>= 8'}
+    dev: false
+
   /which-boxed-primitive/1.0.2:
     resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==}
     dependencies:

prisma/schema.prisma

@@ -66,6 +66,9 @@ model ApplicationUserData {
 }
 
 model BorderImage {
-  id        Int    @id @default(autoincrement())
-  imageName String @unique
+  id         Int     @id @default(autoincrement())
+  imageName  String  @unique
+  appId      Int?
+  appName    String?
+  borderName String?
 }

util/ingest.js

@@ -1,4 +1,3 @@
-console.log("a");
 const fs = require("fs");
 const path = require("path");
 const Prisma = require("@prisma/client");
@@ -14,9 +13,10 @@ const folder = path.join(
   cwd.includes("util") ? ".." : "",
   "public/images"
 );
+
 const list = fs.readdirSync(folder);
 
-(async () => {
+let catalogue = async () => {
   let numAdded = 0;
   for (let item of list) {
     // console.log(item);
@@ -31,6 +31,7 @@ const list = fs.readdirSync(folder);
         data: {
           id: 0,
           imageName: item,
+          borderName: "Default",
         },
       });
       numAdded++;
@@ -46,4 +47,62 @@ const list = fs.readdirSync(folder);
     }
   }
   console.log(`Processed ${numAdded} new images.`);
-})();
+};
+
+let download = async () => {
+  const fetch = await import("node-fetch");
+  const filePath = path.join(
+    cwd,
+    cwd.includes("util") ? ".." : "",
+    "util/border_data.json"
+  );
+  const json_data = JSON.parse(fs.readFileSync(filePath));
+  let numAdded = 0;
+  for (let value of json_data) {
+    if (!value) continue;
+    const filename_regex =
+      /https:\/\/cdn.akamai.steamstatic.com\/steamcommunity\/public\/images\/items\/\d+\/(.+\.png)/gi;
+    const filename = filename_regex.exec(value.borderURL)[1];
+
+    if (!list.includes(filename)) {
+      console.log("N", filename);
+      let data = await fetch.default(value.borderURL);
+      data.body.pipe(fs.createWriteStream(path.join(folder, filename)));
+    }
+
+    const result = await prisma.borderImage.findFirst({
+      where: {
+        imageName: filename,
+      },
+    });
+
+    if (!result.appId || !result.appName || !result.borderName) {
+      const added = await prisma.borderImage.upsert({
+        create: {
+          imageName: filename,
+          appId: value.appInfo.appid,
+          appName: value.appInfo.name,
+          borderName: value.name,
+        },
+        update: {
+          imageName: filename,
+          appId: value.appInfo.appid,
+          appName: value.appInfo.name,
+          borderName: value.name,
+        },
+        where: {
+          imageName: filename,
+        },
+      });
+      console.log(added);
+      numAdded++;
+    }
+  }
+  console.log(`Upserted ${numAdded} values.`);
+};
+
+download().then(() => {
+  catalogue().then(() => {
+    console.log("done.");
+  });
+});