fedimbed: proxy

package.json

@@ -30,7 +30,8 @@
     "sh": "github:Cynosphere/sh",
     "sharp": "^0.32.0",
     "spitroast": "^1.4.3",
-    "sqlite3": "^5.1.6"
+    "sqlite3": "^5.1.6",
+    "undici": "^7.10.0"
   },
   "devDependencies": {
     "eslint": "^8.39.0",

pnpm-lock.yaml

@@ -47,6 +47,9 @@ importers:
       sqlite3:
         specifier: ^5.1.6
         version: 5.1.6(encoding@0.1.13)
+      undici:
+        specifier: ^7.10.0
+        version: 7.10.0
     devDependencies:
       eslint:
         specifier: ^8.39.0
@@ -1508,6 +1511,10 @@ packages:
     resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==}
     engines: {node: '>=10'}
 
+  undici@7.10.0:
+    resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==}
+    engines: {node: '>=20.18.1'}
+
   unique-filename@1.1.1:
     resolution: {integrity: sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==}
 
@@ -3320,6 +3327,8 @@ snapshots:
 
   type-fest@0.20.2: {}
 
+  undici@7.10.0: {}
+
   unique-filename@1.1.1:
     dependencies:
       unique-slug: 2.0.2

src/modules/fedimbed.js

@@ -1,6 +1,7 @@
 const fs = require("node:fs");
 const httpSignature = require("@peertube/http-signature");
 const {XMLParser} = require("fast-xml-parser");
+const {ProxyAgent} = require("undici");
 
 const events = require("#lib/events.js");
 const logger = require("#lib/logger.js");
@@ -113,6 +114,9 @@ const numberFormatter = Intl.NumberFormat("en-US");
 
 const domainCache = new Map();
 domainCache.set("cohost.org", "cohost"); // no nodeinfo
+domainCache.set("truthsocial.com", "Truth Social"); // no nodeinfo
+
+const proxyAgent = new ProxyAgent(hf.config.proxy);
 
 function firefoxUseragent() {
   const now = Math.floor(Date.now() / 1000);
@@ -125,11 +129,9 @@ async function resolvePlatform(url) {
   const urlObj = new URL(url);
   if (domainCache.has(urlObj.hostname)) return domainCache.get(urlObj.hostname);
 
-  const ffua = firefoxUseragent();
-
   try {
     const res = await fetch(urlObj.origin + "/.well-known/nodeinfo", {
-      headers: {"User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT},
+      headers: {"User-Agent": FRIENDLY_USERAGENT},
     }).then((res) => res.text());
 
     if (!res.startsWith("{")) {
@@ -147,7 +149,7 @@ async function resolvePlatform(url) {
     }
 
     const nodeinfo = await fetch(probe.links[probe.links.length - 1].href, {
-      headers: {"User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT},
+      headers: {"User-Agent": FRIENDLY_USERAGENT},
     }).then((res) => res.json());
 
     if (!nodeinfo?.software?.name) {
@@ -183,6 +185,7 @@ async function getCrawledData(url, color, platformName) {
       headers: {
         "User-Agent": urlObj.hostname === "truthsocial.com" ? firefoxUseragent() : FRIENDLY_USERAGENT,
       },
+      dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
     })
       .then((res) => res.text())
       .catch(() => {});
@@ -781,6 +784,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) {
           "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT,
           Accept: "application/activity+json",
         },
+        dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
       }).then((res) => res.text());
     } catch (err) {
       logger.error("fedimbed", `Failed to signed fetch "${url}", retrying unsigned: ${err}`);
@@ -792,6 +796,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) {
             "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT,
             Accept: "application/activity+json",
           },
+          dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
         }).then((res) => res.text());
       } catch (err) {
         logger.error("fedimbed", `Failed to fetch "${url}": ${err}`);
@@ -883,6 +888,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) {
             headers: Object.assign(headers, {
               "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT,
             }),
+            dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
           })
         ).then((res) => res.text());
       } catch (err) {
@@ -896,6 +902,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) {
               headers: Object.assign(headers, {
                 "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT,
               }),
+              dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
             })
           ).then((res) => res.text());
         } catch (err) {
@@ -932,23 +939,28 @@ async function fetchPost(url, platform, forceMastoAPI = false) {
 }
 
 async function getStatsAS(post) {
+  const urlObj = new URL(post);
+  const ffua = firefoxUseragent();
+
   // agony
   let replyCount = 0;
   if (post?.replies?.id)
     try {
       const selfReplies = await signedFetch(post.replies.id + "?page=true", {
         headers: {
-          "User-Agent": FRIENDLY_USERAGENT,
+          "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT,
           Accept: "application/activity+json",
         },
+        dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
       }).then((res) => res.json());
       replyCount += selfReplies?.items?.length ?? 0;
 
       const otherReplies = await signedFetch(post.replies.id + "?page=true&only_other_accounts=true", {
         headers: {
-          "User-Agent": FRIENDLY_USERAGENT,
+          "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT,
           Accept: "application/activity+json",
         },
+        dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null,
       }).then((res) => res.json());
       replyCount += otherReplies?.items?.length ?? 0;
     } catch {