diff --git a/package.json b/package.json index b3a2db6..c003ddd 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "sh": "github:Cynosphere/sh", "sharp": "^0.32.0", "spitroast": "^1.4.3", - "sqlite3": "^5.1.6" + "sqlite3": "^5.1.6", + "undici": "^7.10.0" }, "devDependencies": { "eslint": "^8.39.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ff848ef..d77c888 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,9 @@ importers: sqlite3: specifier: ^5.1.6 version: 5.1.6(encoding@0.1.13) + undici: + specifier: ^7.10.0 + version: 7.10.0 devDependencies: eslint: specifier: ^8.39.0 @@ -1508,6 +1511,10 @@ packages: resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} engines: {node: '>=10'} + undici@7.10.0: + resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==} + engines: {node: '>=20.18.1'} + unique-filename@1.1.1: resolution: {integrity: sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==} @@ -3320,6 +3327,8 @@ snapshots: type-fest@0.20.2: {} + undici@7.10.0: {} + unique-filename@1.1.1: dependencies: unique-slug: 2.0.2 diff --git a/src/modules/fedimbed.js b/src/modules/fedimbed.js index 9631db5..013b5df 100644 --- a/src/modules/fedimbed.js +++ b/src/modules/fedimbed.js @@ -1,6 +1,7 @@ const fs = require("node:fs"); const httpSignature = require("@peertube/http-signature"); const {XMLParser} = require("fast-xml-parser"); +const {ProxyAgent} = require("undici"); const events = require("#lib/events.js"); const logger = require("#lib/logger.js"); @@ -113,6 +114,9 @@ const numberFormatter = Intl.NumberFormat("en-US"); const domainCache = new Map(); domainCache.set("cohost.org", "cohost"); // no nodeinfo +domainCache.set("truthsocial.com", "Truth Social"); // no nodeinfo + +const proxyAgent = new ProxyAgent(hf.config.proxy); function firefoxUseragent() { const now = Math.floor(Date.now() / 1000); @@ -125,11 +129,9 @@ async function resolvePlatform(url) { const urlObj = new URL(url); if (domainCache.has(urlObj.hostname)) return domainCache.get(urlObj.hostname); - const ffua = firefoxUseragent(); - try { const res = await fetch(urlObj.origin + "/.well-known/nodeinfo", { - headers: {"User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT}, + headers: {"User-Agent": FRIENDLY_USERAGENT}, }).then((res) => res.text()); if (!res.startsWith("{")) { @@ -147,7 +149,7 @@ async function resolvePlatform(url) { } const nodeinfo = await fetch(probe.links[probe.links.length - 1].href, { - headers: {"User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT}, + headers: {"User-Agent": FRIENDLY_USERAGENT}, }).then((res) => res.json()); if (!nodeinfo?.software?.name) { @@ -183,6 +185,7 @@ async function getCrawledData(url, color, platformName) { headers: { "User-Agent": urlObj.hostname === "truthsocial.com" ? firefoxUseragent() : FRIENDLY_USERAGENT, }, + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }) .then((res) => res.text()) .catch(() => {}); @@ -781,6 +784,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) { "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT, Accept: "application/activity+json", }, + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }).then((res) => res.text()); } catch (err) { logger.error("fedimbed", `Failed to signed fetch "${url}", retrying unsigned: ${err}`); @@ -792,6 +796,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) { "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT, Accept: "application/activity+json", }, + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }).then((res) => res.text()); } catch (err) { logger.error("fedimbed", `Failed to fetch "${url}": ${err}`); @@ -883,6 +888,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) { headers: Object.assign(headers, { "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT, }), + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }) ).then((res) => res.text()); } catch (err) { @@ -896,6 +902,7 @@ async function fetchPost(url, platform, forceMastoAPI = false) { headers: Object.assign(headers, { "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT, }), + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }) ).then((res) => res.text()); } catch (err) { @@ -932,23 +939,28 @@ async function fetchPost(url, platform, forceMastoAPI = false) { } async function getStatsAS(post) { + const urlObj = new URL(post); + const ffua = firefoxUseragent(); + // agony let replyCount = 0; if (post?.replies?.id) try { const selfReplies = await signedFetch(post.replies.id + "?page=true", { headers: { - "User-Agent": FRIENDLY_USERAGENT, + "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT, Accept: "application/activity+json", }, + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }).then((res) => res.json()); replyCount += selfReplies?.items?.length ?? 0; const otherReplies = await signedFetch(post.replies.id + "?page=true&only_other_accounts=true", { headers: { - "User-Agent": FRIENDLY_USERAGENT, + "User-Agent": urlObj.hostname === "truthsocial.com" ? ffua : FRIENDLY_USERAGENT, Accept: "application/activity+json", }, + dispatcher: urlObj.hostname === "truthsocial.com" ? proxyAgent : null, }).then((res) => res.json()); replyCount += otherReplies?.items?.length ?? 0; } catch {