pasta

lib/borders.js

@@ -9,8 +9,11 @@ export const getBorderById = async (id) => {
   });
 };
 
-export const getAllBorders = async () => {
+export const getAllBorders = async (limit = undefined, cursor = undefined) => {
-  return await prisma.borderImage.findMany();
+  return await prisma.borderImage.findMany({
+    take: limit,
+    cursor: cursor,
+  });
 };
 
 export const getUserBorders = async (req) => {

package.json

@@ -7,7 +7,8 @@
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
-    "ingest": "node util/ingest.js"
+    "ingest": "node util/ingest.js",
+    "get_data": "node util/get_data.js"
   },
   "dependencies": {
     "@next-auth/prisma-adapter": "^1.0.3",

util/get_data.js

@@ -0,0 +1,114 @@
+const dotenv = require("dotenv");
+dotenv.config();
+const { URL, URLSearchParams } = require("url");
+const fs = require("fs");
+const path = require("path");
+
+const key = process.env.STEAM_API_KEY;
+
+let appIds = {};
+
+async function makeReq(cursor) {
+  const fetch = await import("node-fetch");
+  const url = new URL(
+    "https://api.steampowered.com/ILoyaltyRewardsService/QueryRewardItems/v1/"
+  );
+
+  const searchParams = new URLSearchParams({
+    key,
+    "community_item_classes[0]": 14,
+  });
+
+  if (cursor !== null) searchParams.set("cursor", cursor); // pagination
+  url.search = searchParams;
+
+  const req = await fetch.default(url);
+  const json = await req.json();
+  return json.response;
+}
+
+async function getAppInfo(appid) {
+  if (appIds[appid]) {
+    return appIds[appid];
+  }
+
+  const fetch = await import("node-fetch");
+  const url = new URL(
+    "https://api.steampowered.com/IStoreService/GetAppList/v1/"
+  );
+
+  // what the fuck this is a hack
+  // todo: paginate all app info and store it instead of querying the api 447 times
+  const searchParams = new URLSearchParams({
+    key,
+    last_appid: parseInt(appid) - 1,
+    max_results: 1,
+  });
+  url.search = searchParams;
+
+  const req = await fetch.default(url);
+  const json = await req.json();
+  appIds[appid] = json.response.apps[0];
+  return json.response.apps[0];
+}
+
+async function main() {
+  const borders = [];
+  let cursor = null;
+
+  let total = 0;
+  let done = 0;
+
+  while (true) {
+    const json = await makeReq(cursor);
+
+    cursor = json.next_cursor;
+    total = json.total_count;
+    if (json.count === 0) break; // found all borders
+
+    for (const border of json.definitions) {
+      const appid = border.appid;
+      const name = border.community_item_data.item_name;
+
+      const filename_large = border.community_item_data.item_image_large;
+      if (filename_large == "c05c0155855b74c28e0f6e9417d4afa3c99d76ef.png") {
+        console.log(border);
+      }
+      const filename_small = border.community_item_data.item_image_small;
+
+      const appInfo = await getAppInfo(appid);
+
+      const borderURL = `https://cdn.akamai.steamstatic.com/steamcommunity/public/images/items/${appid}`;
+
+      if (filename_large != undefined) {
+        borders.push({
+          name: `${name}-LARGE`,
+          borderURL: `${borderURL}/${filename_large}`,
+          appInfo,
+        });
+      }
+
+      if (filename_small != undefined) {
+        borders.push({
+          name: `${name}-SMALL`,
+          borderURL: `${borderURL}/${filename_small}`,
+          appInfo,
+        });
+      }
+
+      done++;
+      console.log(`${done}/${total}`);
+    }
+  }
+
+  const cwd = process.cwd();
+  const filePath = path.join(
+    cwd,
+    cwd.includes("util") ? ".." : "",
+    "util/border_data.json"
+  );
+
+  fs.writeFileSync(filePath, JSON.stringify(borders));
+}
+
+main();

util/ingest.js

@@ -40,6 +40,7 @@ let catalogue = async () => {
       const added = await prisma.borderImage.create({
         data: {
           imageName: item,
+          appName: "Manually added",
         },
       });
       numAdded++;
@@ -58,16 +59,23 @@ let download = async () => {
   );
   const json_data = JSON.parse(fs.readFileSync(filePath));
   let numAdded = 0;
-  for (let value of json_data) {
+  for (let key in json_data) {
+    let value = json_data[key];
     if (!value) continue;
     const filename_regex =
-      /https:\/\/cdn.akamai.steamstatic.com\/steamcommunity\/public\/images\/items\/\d+\/(.+\.png)/gi;
+      /https:\/\/cdn.akamai.steamstatic.com\/steamcommunity\/public\/images\/items\/\d+\/\/?(.+\.(a?png|gif))/gi;
-    const filename = filename_regex.exec(value.borderURL)[1];
+    const file_match = filename_regex.exec(value.borderURL);
+    if (!file_match || file_match == null) {
+      console.error("MATCH ERROR", value.borderURL);
+    }
+    const filename = file_match[1].replace("apng", "png");
 
     if (!list.includes(filename)) {
       console.log("N", filename);
-      let data = await fetch.default(value.borderURL);
+      setTimeout(async () => {
-      data.body.pipe(fs.createWriteStream(path.join(folder, filename)));
+        let data = await fetch.default(value.borderURL);
+        data.body.pipe(fs.createWriteStream(path.join(folder, filename)));
+      }, key * 100);
     }
 
     const result = await prisma.borderImage.findFirst({
@@ -76,7 +84,11 @@ let download = async () => {
       },
     });
 
-    if (!result.appId || !result.appName || !result.borderName) {
+    if (
+      result?.appId != value.appInfo.appid ||
+      result?.appName != value.appInfo.name ||
+      result?.borderName != value.name
+    ) {
       const added = await prisma.borderImage.upsert({
         create: {
           imageName: filename,
@@ -94,7 +106,7 @@ let download = async () => {
           imageName: filename,
         },
       });
-      console.log(added);
+      // console.log(added);
       numAdded++;
     }
   }
@@ -104,5 +116,6 @@ let download = async () => {
 download().then(() => {
   catalogue().then(() => {
     console.log("done.");
+    console.log("waiting for image catch up to finish...");
   });
 });