guh

.rgignore

@@ -0,0 +1 @@
+data/

src/index.js

@@ -56,6 +56,7 @@ global.hf = {
   events,
   timer,
   database,
+  event_stats: {},
 };
 
 const {formatUsername} = require("#util/misc.js");
@@ -69,7 +70,7 @@ for (const file of fs.readdirSync(resolve(__dirname, "modules"), {withFileTypes:
     require(`#modules/${file.name}`);
     logger.info("hf:modules", `Loaded module: "${file.name}"`);
   } catch (err) {
-    logger.error("hf:modules", `Failed to load "${file.name}": ${err}`);
+    logger.error("hf:modules", `Failed to load "${file.name}": ${err.stack}`);
   }
 }
 
@@ -89,9 +90,7 @@ bot.on("messageCreate", async (msg) => {
 
     await CommandDispatcher(msg);
   } catch (err) {
-    const stack = (err?.stack ?? err.message).split("\n");
+    logger.error("hf:main", `Failed to dispatch command: ${err.stack}`);
-    const error = stack.shift();
-    logger.error("hf:main", `Failed to dispatch command: ${error}\n\t${stack.join("\n\t")}`);
   }
 });
 bot.on("messageUpdate", async (msg, oldMsg) => {
@@ -101,9 +100,7 @@ bot.on("messageUpdate", async (msg, oldMsg) => {
       await CommandDispatcher(msg);
     }
   } catch (err) {
-    const stack = (err?.stack ?? err.message).split("\n");
+    logger.error("hf:main", `Failed to dispatch command update: ${err.stack}`);
-    const error = stack.shift();
-    logger.error("hf:main", `Failed to dispatch command update: ${error}\n\t${stack.join("\n\t")}`);
   }
 });
 bot.on("messageReactionAdd", async (msg, reaction, reactor) => {
@@ -133,18 +130,14 @@ bot.on("messageReactionAdd", async (msg, reaction, reactor) => {
 
     await msg.delete("Command sender requested output deletion.");
   } catch (err) {
-    const stack = (err?.stack ?? err.message).split("\n");
+    logger.error("hf:main", `Failed to self-delete message: ${err.stack}`);
-    const error = stack.shift();
-    logger.error("hf:main", `Failed to self-delete message: ${error}\n\t${stack.join("\n\t")}`);
   }
 });
 bot.on("interactionCreate", async (interaction) => {
   try {
     await InteractionDispatcher(interaction);
   } catch (err) {
-    const stack = (err?.stack ?? err.message).split("\n");
+    logger.error("hf:main", `Failed to dispatch interaction command: ${err.stack}`);
-    const error = stack.shift();
-    logger.error("hf:main", `Failed to dispatch interaction command: ${error}\n\t${stack.join("\n\t")}`);
   }
 });
 
@@ -160,7 +153,7 @@ bot.once("ready", async () => {
       });
     }
   } catch (err) {
-    logger.error("hf:main", `Failed to send startup message, API probably broken currently.\n${err}`);
+    logger.error("hf:main", `Failed to send startup message, API probably broken currently.\n${err.stack}`);
   }
   bot.on("ready", () => {
     logger.info("hf:main", "Reconnected to Discord.");
@@ -209,23 +202,23 @@ bot.once("ready", async () => {
     try {
       await bot.requestHandler.request("PUT", APIEndpoints.COMMANDS(bot.application.id), true, commands);
     } catch (err) {
-      logger.error("hf:main", `Failed to update interaction commands, API probably broken currently.\n${err}`);
+      logger.error("hf:main", `Failed to update interaction commands, API probably broken currently.\n${err.stack}`);
     }
   }
 });
 
 bot.on("error", (err) => {
-  logger.error("hf:main", "Catching error: " + err);
+  logger.error("hf:main", `Catching error: ${err.stack}`);
 });
 bot.on("warn", (err) => {
-  logger.warn("hf:main", "Catching warn: " + err);
+  logger.warn("hf:main", `Catching warn: ${err}`);
 });
 
 bot.on("shardDisconnect", (err, id) => {
-  logger.verbose("hf:shard", `Disconnecting from shard ${id}: ${err}`);
+  logger.verbose("hf:shard", `Disconnecting from shard ${id}: ${err?.stack ?? err ?? "no error???"}`);
 });
 bot.on("shardResume", (id) => {
-  logger.verbose("hf:shard", "Resuming on shard " + id);
+  logger.verbose("hf:shard", `Resuming on shard ${id}`);
 });
 bot.on("shardPreReady", (id) => {
   logger.verbose("hf:shard", `Shard ${id} getting ready`);
@@ -234,7 +227,14 @@ bot.on("shardReady", (id) => {
   logger.verbose("hf:shard", `Shard ${id} ready`);
 });
 bot.on("unknown", (packet, id) => {
-  logger.verbose("hf:main", `Shard ${id} caught unknown packet: ${JSON.stringify(packet)}`);
+  logger.verbose("hf:main", `Shard ${id} caught unknown packet:\n  ${JSON.stringify(packet)}`);
+});
+
+bot.on("rawWS", (packet) => {
+  if (packet.op === 0 && packet.t != null) {
+    if (!hf.event_stats[packet.t]) hf.event_stats[packet.t] = 0;
+    hf.event_stats[packet.t]++;
+  }
 });
 
 instead("spawn", bot.shards, function (args, orig) {

src/modules/bot.js

@@ -221,3 +221,17 @@ settings.callback = async function (msg, line, [cmd, key, value]) {
   }
 };
 hf.registerCommand(settings);
+
+const socketstats = new Command("socketstats");
+socketstats.category = CATEGORY;
+socketstats.helpText = "List the counts of socket events this session";
+socketstats.callback = function () {
+  let out = "```c\n";
+  for (const [event, count] of Object.entries(hf.event_stats)) {
+    out += `"${event}": ${count}\n`;
+  }
+  out += "```";
+
+  return out;
+};
+hf.registerCommand(socketstats);

src/modules/codePreviews.js

@@ -60,12 +60,12 @@ async function processFile(link, originalLink, spoiler = false, linkFile = false
     fileName = `[${fileName}](<${originalLink}>)`;
   }
 
-  const lineStr = urlObj.hash.match(/#L\d+(-L?\d+)?/)?.[0];
+  const lineStr = urlObj.hash.match(/#L\d+(C\d+)?(-L?\d+)?(C\d+)?/)?.[0];
   let startLine, endLine;
   let entireFile = false;
 
   if (lineStr) {
-    const [start, end] = lineStr.match(/\d+/g);
+    const [start, end] = lineStr.match(/(?<=[L-]{1,2})\d+/g);
     if (!end) {
       startLine = endLine = start;
     } else {

src/modules/fedimbed.js

@@ -21,8 +21,8 @@ const BSKY_POST_REGEX =
   /^\/profile\/(did:plc:[a-z0-9]+|(did:web:)?[a-z0-9][a-z0-9.-]+[a-z0-9]*)\/post\/([a-z0-9]+)\/?$/i;
 
 const PATH_REGEX = {
-  mastodon: /^\/@(.+?)\/(\d+)\/?/,
+  mastodon: /^\/@(.+?)\/([a-z0-9]+?)\/?/i,
-  mastodon2: /^\/(.+?)\/statuses\/\d+\/?/,
+  mastodon2: /^\/(.+?)\/statuses\/([a-z0-9]+?)\/?/i,
   pleroma: /^\/objects\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/?/,
   pleroma2: /^\/notice\/[A-Za-z0-9]+\/?/,
   misskey: /^\/notes\/[a-z0-9]+\/?/,
@@ -30,12 +30,12 @@ const PATH_REGEX = {
   lemmy: /^\/post\/\d+\/?/,
   honk: /^\/u\/(.+?)\/h\/(.+?)\/?/,
   pixelfed: /^\/p\/(.+?)\/(.+?)\/?/,
-  cohost: /^\/[A-Za-z0-9]+\/post\/\d+-[A-Za-z0-9-]+\/?/,
+  //cohost: /^\/[A-Za-z0-9]+\/post\/\d+-[A-Za-z0-9-]+\/?/,
   bluesky: BSKY_POST_REGEX,
 };
 
 const PLATFORM_COLORS = {
-  mastodon: 0x2791da,
+  mastodon: 0x6363ff,
   pleroma: 0xfba457,
   akkoma: 0x593196,
   misskey: 0x99c203,
@@ -46,8 +46,9 @@ const PLATFORM_COLORS = {
   birdsitelive: 0x1da1f2,
   iceshrimp: 0x8e82f9, // YCbCr interpolated as the accent color is a gradient
   pixelfed: 0x10c5f8,
-  cohost: 0x83254f,
+  //cohost: 0x83254f,
   bluesky: 0x0085ff,
+  twitter: 0xff6c60, // Nitter accent color
 };
 
 const BSKY_DOMAINS = [
@@ -67,6 +68,7 @@ const BSKY_DOMAINS = [
 /*const TW_DOMAINS = [
   "tw.c7.pm",
   "tw.counter-strike.gay",
+  "xcancel.com",
   "twitter.com",
   "x.com",
   "fxtwitter.com",
@@ -84,36 +86,49 @@ async function resolvePlatform(url) {
   const urlObj = new URL(url);
   if (domainCache.has(urlObj.hostname)) return domainCache.get(urlObj.hostname);
 
-  const res = await fetch(urlObj.origin + "/.well-known/nodeinfo", {
+  try {
-    headers: {"User-Agent": FRIENDLY_USERAGENT},
+    const res = await fetch(urlObj.origin + "/.well-known/nodeinfo", {
-  }).then((res) => res.text());
+      headers: {"User-Agent": FRIENDLY_USERAGENT},
+    }).then((res) => res.text());
 
-  if (!res.startsWith("{")) {
+    if (!res.startsWith("{")) {
-    logger.error("fedimbed", `No nodeinfo for "${urlObj.hostname}"???`);
+      logger.error("fedimbed", `No nodeinfo for "${urlObj.hostname}"???`);
-    domainCache.set(urlObj.hostname, null);
+      domainCache.set(urlObj.hostname, null);
-    return null;
+      return null;
+    }
+
+    const probe = JSON.parse(res);
+
+    if (!probe?.links) {
+      logger.error("fedimbed", `No nodeinfo for "${urlObj.hostname}"???`);
+      domainCache.set(urlObj.hostname, null);
+      return null;
+    }
+
+    const nodeinfo = await fetch(probe.links[probe.links.length - 1].href, {
+      headers: {"User-Agent": FRIENDLY_USERAGENT},
+    }).then((res) => res.json());
+
+    if (!nodeinfo?.software?.name) {
+      logger.error("fedimbed", `Got nodeinfo for "${urlObj.hostname}", but missing software name.`);
+      domainCache.set(urlObj.hostname, null);
+      return null;
+    }
+
+    domainCache.set(urlObj.hostname, nodeinfo.software.name);
+    return nodeinfo.software.name;
+  } catch (err) {
+    logger.error("fedimbed", `Failed to get nodeinfo for "${url}": ${err}`);
+    return "<nodeinfo failed>";
   }
+}
 
-  const probe = JSON.parse(res);
+function normalizePlatform(platform) {
-
+  return platform
-  if (!probe?.links) {
+    .replace("gotosocial", "GoToSocial")
-    logger.error("fedimbed", `No nodeinfo for "${urlObj.hostname}"???`);
+    .replace("birdsitelive", '"Twitter" (BirdsiteLive)')
-    domainCache.set(urlObj.hostname, null);
+    .replace(/^(.)/, (_, c) => c.toUpperCase())
-    return null;
+    .replace("Cohost", "cohost");
-  }
-
-  const nodeinfo = await fetch(probe.links[probe.links.length - 1].href, {
-    headers: {"User-Agent": FRIENDLY_USERAGENT},
-  }).then((res) => res.json());
-
-  if (!nodeinfo?.software?.name) {
-    logger.error("fedimbed", `Got nodeinfo for "${urlObj.hostname}", but missing software name.`);
-    domainCache.set(urlObj.hostname, null);
-    return null;
-  }
-
-  domainCache.set(urlObj.hostname, nodeinfo.software.name);
-  return nodeinfo.software.name;
 }
 
 const keyId = "https://hf.c7.pm/actor#main-key";
@@ -400,7 +415,7 @@ async function bluesky(msg, url, spoiler = false) {
   if (data.thread.parent) {
     const reply = data.thread.parent.post;
     mainEmbed.author = {
-      name: `Replying to: ${reply.author.displayName} (${reply.author.handle})`,
+      name: `Replying to: ${reply.author.displayName} (@${reply.author.handle})`,
       icon_url: "https://cdn.discordapp.com/emojis/1308640078825787412.png",
       url: `https://bsky.app/profile/${reply.author.handle}/post/${reply.uri.substring(
         reply.uri.lastIndexOf("/") + 1
@@ -461,6 +476,17 @@ async function bluesky(msg, url, spoiler = false) {
           videos.push({url: videoUrl, desc: post.embed.alt, type: contentType});
 
           embeds.push({...mainEmbed, fields: [{name: "\u200b", value: `[Video Link](${videoUrl})`}]});
+        } else if (post.embed.media.$type === "app.bsky.embed.external#view") {
+          if (post.embed.media.external.uri.includes("tenor.com")) {
+            const url = new URL(post.embed.media.external.uri);
+            url.searchParams.delete("hh");
+            url.searchParams.delete("ww");
+            embeds.push({...mainEmbed, image: {url: url.toString()}});
+          } else {
+            embeds.push(mainEmbed);
+          }
+        } else {
+          embeds.push(mainEmbed);
         }
 
         const quoteData = await blueskyQuoteEmbed(post.embed.record.record);
@@ -564,104 +590,53 @@ async function bluesky(msg, url, spoiler = false) {
   };
 }
 
-async function processUrl(msg, url, spoiler = false, command = false) {
+async function fetchPost(url, platform, forceMastoAPI = false) {
-  let canFedi = await hasFlag(msg.guildID, "fedimbed");
+  let urlObj = new URL(url);
-  let canBsky = await hasFlag(msg.guildID, "bskyEmbeds");
+  let postData;
 
-  if (command === true) {
+  if (!forceMastoAPI) {
-    canFedi = true;
+    let rawPostData;
-    canBsky = true;
-  }
-
-  let invalidUrl = false;
-  let urlObj;
-  try {
-    urlObj = new URL(url);
-  } catch {
-    invalidUrl = true;
-  }
-
-  if (invalidUrl) return {};
-
-  if (BSKY_DOMAINS.includes(urlObj.hostname.toLowerCase())) {
-    if (canBsky) {
-      return await bluesky(msg, url, spoiler);
-    } else {
-      return {};
-    }
-  }
-  if (!canFedi) return {};
-
-  // some lemmy instances have old reddit frontend subdomains
-  // but these frontends are just frontends and dont actually expose the API
-  if (urlObj.hostname.startsWith("old.")) {
-    urlObj.hostname = urlObj.hostname.replace("old.", "");
-    url = urlObj.href;
-  }
-
-  let platform = (await resolvePlatform(url)) ?? "<no nodeinfo>";
-  let color = PLATFORM_COLORS[platform];
-  let platformName = platform
-    .replace("gotosocial", "GoToSocial")
-    .replace("birdsitelive", '"Twitter" (BirdsiteLive)')
-    .replace(/^(.)/, (_, c) => c.toUpperCase())
-    .replace("Cohost", "cohost");
-
-  const images = [];
-  const videos = [];
-  const audios = [];
-  let content,
-    cw,
-    author,
-    timestamp,
-    title,
-    poll,
-    emotes = [],
-    sensitive = false;
-
-  // Fetch post
-  let rawPostData;
-  try {
-    rawPostData = await signedFetch(url, {
-      headers: {
-        "User-Agent": FRIENDLY_USERAGENT,
-        Accept: "application/activity+json",
-      },
-    }).then((res) => res.text());
-  } catch (err) {
-    logger.error("fedimbed", `Failed to signed fetch "${url}", retrying unsigned: ${err}`);
-  }
-  if (!rawPostData) {
     try {
-      rawPostData = await fetch(url, {
+      rawPostData = await signedFetch(url, {
         headers: {
           "User-Agent": FRIENDLY_USERAGENT,
           Accept: "application/activity+json",
         },
       }).then((res) => res.text());
     } catch (err) {
-      logger.error("fedimbed", `Failed to fetch "${url}": ${err}`);
+      logger.error("fedimbed", `Failed to signed fetch "${url}", retrying unsigned: ${err}`);
     }
-  }
+    if (!rawPostData) {
-
+      try {
-  let postData;
+        rawPostData = await fetch(url, {
-  if (rawPostData?.startsWith("{")) {
+          headers: {
-    try {
+            "User-Agent": FRIENDLY_USERAGENT,
-      postData = JSON.parse(rawPostData);
+            Accept: "application/activity+json",
-    } catch (err) {
+          },
-      logger.error("fedimbed", `Failed to decode JSON for "${url}": ${err}\n  "${rawPostData}"`);
+        }).then((res) => res.text());
+      } catch (err) {
+        logger.error("fedimbed", `Failed to fetch "${url}": ${err}`);
+      }
     }
-  } else {
-    logger.warn("fedimbed", `Got non-JSON for "${url}": ${rawPostData}`);
-  }
 
-  if (postData?.error) {
+    if (rawPostData?.startsWith("{")) {
-    logger.error("fedimbed", `Received error for "${url}": ${postData.error}`);
+      try {
+        postData = JSON.parse(rawPostData);
+      } catch (err) {
+        logger.error("fedimbed", `Failed to decode JSON for "${url}": ${err}\n  "${rawPostData}"`);
+      }
+    } else {
+      logger.warn("fedimbed", `Got non-JSON for "${url}": ${rawPostData}`);
+    }
+
+    if (postData?.error) {
+      logger.error("fedimbed", `Received error for "${url}": ${postData.error}`);
+    }
   }
 
   if (!postData) {
-    // We failed to get post.
+    // We failed to get post. (or we forced ourselves to be here)
-    // Assume it was due to AFM or forced HTTP signatures and use MastoAPI
+    // Assume it was due to AFM or some other issue and (try to) use MastoAPI (or equivalent)
 
     // Follow redirect from /object since we need the ID from /notice
     if (PATH_REGEX.pleroma.test(urlObj.pathname)) {
@@ -698,10 +673,17 @@ async function processUrl(msg, url, spoiler = false, command = false) {
         noteId = noteId.split("#")[0];
       }
       logger.verbose("fedimbed", "Misskey post ID: " + noteId);
-      redirUrl = urlObj.origin + "/api/notes/show/";
+
-      options.method = "POST";
+      // iceshrimp has an entire .NET rewrite and only supports MastoAPI it seems
-      options.body = JSON.stringify({noteId});
+      // *should* be fine for older iceshrimp-js instances
-      headers["Content-Type"] = "application/json";
+      if (platform == "iceshrimp") {
+        redirUrl = urlObj.origin + "/api/v1/statuses/" + noteId;
+      } else {
+        redirUrl = urlObj.origin + "/api/notes/show/";
+        options.method = "POST";
+        options.body = JSON.stringify({noteId});
+        headers["Content-Type"] = "application/json";
+      }
     } else {
       logger.error("fedimbed", `Missing MastoAPI replacement for "${platform}"`);
     }
@@ -748,119 +730,215 @@ async function processUrl(msg, url, spoiler = false, command = false) {
 
       if (!postData2) {
         logger.warn("fedimbed", `Bailing trying to re-embed "${url}": Failed to get post from normal and MastoAPI.`);
+        return null;
       } else if (postData2.error) {
         logger.error(
           "fedimbed",
           `Bailing trying to re-embed "${url}", MastoAPI gave us error: ${JSON.stringify(postData2.error)}`
         );
-      } else {
+        return null;
-        cw = postData2.spoiler_warning ?? postData2.spoiler_text ?? postData2.cw;
+      }
-        content =
-          postData2.akkoma?.source?.content ??
-          postData2.pleroma?.content?.["text/plain"] ??
-          postData2.text ??
-          postData2.content;
-        author = {
-          name:
-            postData2.account?.display_name ??
-            postData2.account?.username ??
-            postData2.user?.name ??
-            postData2.user?.username,
-          handle:
-            postData2.account?.fqn ?? `${postData2.account?.username ?? postData2.user?.username}@${urlObj.hostname}`,
-          url: postData2.account?.url ?? `${urlObj.origin}/@${postData2.account?.username ?? postData2.user?.username}`,
-          avatar: postData2.account?.avatar ?? postData2.user?.avatarUrl,
-        };
-        timestamp = postData2.created_at ?? postData2.createdAt;
-        emotes = postData2.emojis.filter((x) => !x.name.endsWith("#.")).map((x) => ({name: `:${x.name}:`, url: x.url}));
-        sensitive = postData2.sensitive;
 
-        const attachments = postData2.media_attachments ?? postData2.files;
+      postData2._fedimbed_mastoapi = true;
-        if (attachments) {
+      return postData2;
-          for (const attachment of attachments) {
+    } else {
-            const contentType = await fetch(attachment.url, {
+      return null;
-              method: "HEAD",
+    }
-            }).then((res) => res.headers.get("Content-Type"));
+  } else {
+    return postData;
+  }
+}
 
-            if (contentType) {
+async function processUrl(msg, url, spoiler = false, command = false) {
-              if (contentType.startsWith("image/")) {
+  let canFedi = await hasFlag(msg.guildID, "fedimbed");
-                images.push({
+  let canBsky = await hasFlag(msg.guildID, "bskyEmbeds");
-                  url: attachment.url,
-                  desc: attachment.description ?? attachment.comment,
-                  type: contentType,
-                });
-              } else if (contentType.startsWith("video/")) {
-                videos.push({
-                  url: attachment.url,
-                  desc: attachment.description ?? attachment.comment,
-                  type: contentType,
-                });
-              } else if (contentType.startsWith("audio/")) {
-                audios.push({
-                  url: attachment.url,
-                  desc: attachment.description ?? attachment.comment,
-                  type: contentType,
-                });
-              }
-            } else {
-              const type = attachment.type?.toLowerCase();
 
-              const fileType =
+  if (command === true) {
-                attachment.pleroma?.mime_type ?? type.indexOf("/") > -1
+    canFedi = true;
-                  ? type
+    canBsky = true;
-                  : type +
+  }
-                    "/" +
+
-                    (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? type == "image"
+  let invalidUrl = false;
-                      ? "png"
+  let urlObj;
-                      : type == "video"
+  try {
-                      ? "mp4"
+    urlObj = new URL(url);
-                      : "mpeg");
+  } catch {
-              if (type.startsWith("image")) {
+    invalidUrl = true;
-                images.push({
+  }
-                  url: attachment.url,
+
-                  desc: attachment.description ?? attachment.comment,
+  if (invalidUrl) return {};
-                  type: fileType,
+
-                });
+  if (BSKY_DOMAINS.includes(urlObj.hostname.toLowerCase())) {
-              } else if (type.startsWith("video")) {
+    if (canBsky) {
-                videos.push({
+      return await bluesky(msg, url, spoiler);
-                  url: attachment.url,
+    } else {
-                  desc: attachment.description ?? attachment.comment,
+      return {};
-                  type: fileType,
+    }
-                });
+  }
-              } else if (type.startsWith("audio")) {
+  if (!canFedi) return {};
-                audios.push({
+
-                  url: attachment.url,
+  // some lemmy instances have old reddit frontend subdomains
-                  desc: attachment.description ?? attachment.comment,
+  // but these frontends are just frontends and dont actually expose the API
-                  type: fileType,
+  if (urlObj.hostname.startsWith("old.")) {
-                });
+    urlObj.hostname = urlObj.hostname.replace("old.", "");
-              }
+    url = urlObj.href;
-            }
+  }
+
+  let platform = (await resolvePlatform(url)) ?? "<no nodeinfo>";
+  let color = PLATFORM_COLORS[platform];
+  let platformName = normalizePlatform(platform);
+
+  const images = [];
+  const videos = [];
+  const audios = [];
+  let content,
+    cw,
+    author,
+    timestamp,
+    title,
+    poll,
+    context,
+    contextUrl,
+    emotes = [],
+    sensitive = false;
+
+  // Fetch post
+  const postData = await fetchPost(url, platform);
+
+  if (!postData) {
+    return {};
+  } else if (postData._fedimbed_mastoapi) {
+    if (postData.url) {
+      const realUrlObj = new URL(postData.url);
+      if (realUrlObj.origin != urlObj.origin) {
+        platform = await resolvePlatform(postData.url);
+        color = PLATFORM_COLORS[platform];
+        platformName = normalizePlatform(platform);
+        url = postData.url;
+        urlObj = realUrlObj;
+      }
+    }
+
+    cw = postData.spoiler_warning ?? postData.spoiler_text ?? postData.cw;
+    content =
+      postData.akkoma?.source?.content ??
+      postData.pleroma?.content?.["text/plain"] ??
+      postData.text ??
+      postData.content;
+    author = {
+      name:
+        postData.account?.display_name ?? postData.account?.username ?? postData.user?.name ?? postData.user?.username,
+      handle: postData.account?.fqn ?? `${postData.account?.username ?? postData.user?.username}@${urlObj.hostname}`,
+      url: postData.account?.url ?? `${urlObj.origin}/@${postData.account?.username ?? postData.user?.username}`,
+      avatar: postData.account?.avatar ?? postData.user?.avatarUrl,
+    };
+    timestamp = postData.created_at ?? postData.createdAt;
+    emotes = postData.emojis
+      .filter((x) => !(x.name ?? x.shortcode)?.endsWith("#."))
+      .map((x) => ({name: `:${x.name ?? x.shortcode}:`, url: x.url}));
+    sensitive = postData.sensitive;
+
+    if (postData.in_reply_to_id) {
+      // this url is a dummy and will failed if gone to normally
+      const replyData = await fetchPost(
+        `https://${urlObj.origin}/@fedimbed_reply_fake_user_sorry/${postData.in_reply_to_id}`,
+        platform,
+        true
+      );
+      if (replyData) {
+        contextUrl = replyData.url;
+        context = `Replying to: ${
+          replyData.account?.display_name ??
+          replyData.account?.username ??
+          replyData.user?.name ??
+          replyData.user?.username
+        } (${
+          replyData.account?.fqn ?? `${replyData.account?.username ?? replyData.user?.username}@${urlObj.hostname}`
+        })`;
+      }
+    }
+
+    const attachments = postData.media_attachments ?? postData.files;
+    if (attachments) {
+      for (const attachment of attachments) {
+        const contentType = await fetch(attachment.url, {
+          method: "HEAD",
+        }).then((res) => res.headers.get("Content-Type"));
+
+        if (contentType) {
+          if (contentType.startsWith("image/")) {
+            images.push({
+              url: attachment.url,
+              desc: attachment.description ?? attachment.comment,
+              type: contentType,
+            });
+          } else if (contentType.startsWith("video/")) {
+            videos.push({
+              url: attachment.url,
+              desc: attachment.description ?? attachment.comment,
+              type: contentType,
+            });
+          } else if (contentType.startsWith("audio/")) {
+            audios.push({
+              url: attachment.url,
+              desc: attachment.description ?? attachment.comment,
+              type: contentType,
+            });
+          }
+        } else {
+          const type = attachment.type?.toLowerCase();
+
+          const fileType =
+            attachment.pleroma?.mime_type ?? type.indexOf("/") > -1
+              ? type
+              : type +
+                "/" +
+                (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? type == "image" ? "png" : type == "video" ? "mp4" : "mpeg");
+          if (type.startsWith("image")) {
+            images.push({
+              url: attachment.url,
+              desc: attachment.description ?? attachment.comment,
+              type: fileType,
+            });
+          } else if (type.startsWith("video")) {
+            videos.push({
+              url: attachment.url,
+              desc: attachment.description ?? attachment.comment,
+              type: fileType,
+            });
+          } else if (type.startsWith("audio")) {
+            audios.push({
+              url: attachment.url,
+              desc: attachment.description ?? attachment.comment,
+              type: fileType,
+            });
           }
         }
-        if (!spoiler && postData2.sensitive && attachments.length > 0) {
-          spoiler = true;
-        }
-
-        if (postData2.poll) {
-          poll = {
-            end: new Date(postData2.poll.expires_at),
-            total: postData2.poll.votes_count,
-            options: postData2.poll.options.map((o) => ({
-              name: o.title,
-              count: o.votes_count,
-            })),
-          };
-        }
       }
     }
+    if (!spoiler && postData.sensitive && attachments.length > 0) {
+      spoiler = true;
+    }
+
+    if (postData.poll) {
+      poll = {
+        end: new Date(postData.poll.expires_at),
+        total: postData.poll.votes_count,
+        options: postData.poll.options.map((o) => ({
+          name: o.title,
+          count: o.votes_count,
+        })),
+      };
+    }
   } else {
     if (postData.id) {
       const realUrlObj = new URL(postData.id);
       if (realUrlObj.origin != urlObj.origin) {
         platform = await resolvePlatform(postData.id);
         color = PLATFORM_COLORS[platform];
-        platformName = platform.replace("gotosocial", "GoToSocial").replace(/^(.)/, (_, c) => c.toUpperCase());
+        platformName = normalizePlatform(platform);
         url = postData.id;
+        urlObj = realUrlObj;
       }
     }
 
@@ -876,90 +954,135 @@ async function processUrl(msg, url, spoiler = false, command = false) {
       emotes = tag.filter((x) => !!x.icon).map((x) => ({name: x.name, url: x.icon.url}));
     }
 
-    // NB: gts doesnt send singular attachments as array
+    if (postData.inReplyTo) {
-    const attachments = Array.isArray(postData.attachment) ? postData.attachment : [postData.attachment];
+      contextUrl = postData.inReplyTo;
-    for (const attachment of attachments) {
+      context = "Replying to: ";
-      if (attachment.mediaType) {
-        if (attachment.mediaType.startsWith("video/")) {
-          videos.push({
-            url: attachment.url,
-            desc: attachment.name ?? attachment.description ?? attachment.comment,
-            type: attachment.mediaType,
-          });
-        } else if (attachment.mediaType.startsWith("image/")) {
-          images.push({
-            url: attachment.url,
-            desc: attachment.name ?? attachment.description ?? attachment.comment,
-            type: attachment.mediaType,
-          });
-        } else if (attachment.mediaType.startsWith("audio/")) {
-          audios.push({
-            url: attachment.url,
-            desc: attachment.name ?? attachment.description ?? attachment.comment,
-            type: attachment.mediaType,
-          });
-        }
-      } else if (attachment.url) {
-        const contentType = await fetch(attachment.url, {
-          method: "HEAD",
-        }).then((res) => res.headers.get("Content-Type"));
 
-        if (contentType) {
+      const replyData = await fetchPost(postData.inReplyTo, platform);
-          if (contentType.startsWith("image/")) {
+      if (replyData) {
-            images.push({
+        if (replyData._fedimbed_mastoapi) {
-              url: attachment.url,
+          context += `${
-              desc: attachment.name ?? attachment.description ?? attachment.comment,
+            replyData.account?.display_name ??
-              type: contentType,
+            replyData.account?.username ??
-            });
+            replyData.user?.name ??
-          } else if (contentType.startsWith("video/")) {
+            replyData.user?.username
-            videos.push({
+          } (@${
-              url: attachment.url,
+            replyData.account?.fqn ?? `${replyData.account?.username ?? replyData.user?.username}@${urlObj.hostname}`
-              desc: attachment.name ?? attachment.description ?? attachment.comment,
+          })`;
-              type: contentType,
-            });
-          } else if (contentType.startsWith("audio/")) {
-            audios.push({
-              url: attachment.url,
-              desc: attachment.name ?? attachment.description ?? attachment.comment,
-              type: contentType,
-            });
-          }
         } else {
-          const type = attachment.type?.toLowerCase();
+          const authorData = await signedFetch(replyData.actor ?? replyData.attributedTo, {
+            headers: {
+              "User-Agent": FRIENDLY_USERAGENT,
+              Accept: "application/activity+json",
+            },
+          })
+            .then((res) => res.json())
+            .catch((err) => {
+              /*if (platform !== "cohost")*/ logger.error("fedimbed", `Failed to get author for "${url}": ${err}`);
+            });
 
-          const fileType =
+          if (authorData) {
-            type.indexOf("/") > -1
+            const authorUrlObj = new URL(authorData.url ?? authorData.id);
-              ? type
+            context += `${authorData.name} (${authorData.preferredUsername}@${authorUrlObj.hostname})`;
-              : type +
+          } else {
-                "/" +
+            // bootleg author
-                (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? type == "image" ? "png" : type == "video" ? "mp4" : "mpeg");
+            const authorUrl = replyData.actor ?? replyData.attributedTo;
-          if (type.startsWith("image")) {
+            const authorUrlObj = new URL(authorUrl);
-            images.push({
+            const name = authorUrlObj.pathname.substring(authorUrlObj.pathname.lastIndexOf("/") + 1);
-              url: attachment.url,
+            context += `${name}@${authorUrlObj.hostname}`;
-              desc: attachment.name ?? attachment.description ?? attachment.comment,
-              type: fileType,
-            });
-          } else if (type.startsWith("video")) {
-            videos.push({
-              url: attachment.url,
-              desc: attachment.name ?? attachment.description ?? attachment.comment,
-              type: fileType,
-            });
-          } else if (type.startsWith("audio")) {
-            audios.push({
-              url: attachment.url,
-              desc: attachment.name ?? attachment.description ?? attachment.comment,
-              type: fileType,
-            });
           }
         }
       } else {
-        logger.warn("fedimbed", `Unhandled attachment structure! ${JSON.stringify(attachment)}`);
+        context += "<failed to get reply author>";
       }
     }
 
-    if (!spoiler && postData.sensitive && attachments.length > 0) {
+    if (postData.attachment != null) {
-      spoiler = true;
+      // NB: gts doesnt send singular attachments as array
+      const attachments = Array.isArray(postData.attachment) ? postData.attachment : [postData.attachment];
+      for (const attachment of attachments) {
+        if (attachment.mediaType) {
+          if (attachment.mediaType.startsWith("video/")) {
+            videos.push({
+              url: attachment.url,
+              desc: attachment.name ?? attachment.description ?? attachment.comment,
+              type: attachment.mediaType,
+            });
+          } else if (attachment.mediaType.startsWith("image/")) {
+            images.push({
+              url: attachment.url,
+              desc: attachment.name ?? attachment.description ?? attachment.comment,
+              type: attachment.mediaType,
+            });
+          } else if (attachment.mediaType.startsWith("audio/")) {
+            audios.push({
+              url: attachment.url,
+              desc: attachment.name ?? attachment.description ?? attachment.comment,
+              type: attachment.mediaType,
+            });
+          }
+        } else if (attachment.url) {
+          const contentType = await fetch(attachment.url, {
+            method: "HEAD",
+          }).then((res) => res.headers.get("Content-Type"));
+
+          if (contentType) {
+            if (contentType.startsWith("image/")) {
+              images.push({
+                url: attachment.url,
+                desc: attachment.name ?? attachment.description ?? attachment.comment,
+                type: contentType,
+              });
+            } else if (contentType.startsWith("video/")) {
+              videos.push({
+                url: attachment.url,
+                desc: attachment.name ?? attachment.description ?? attachment.comment,
+                type: contentType,
+              });
+            } else if (contentType.startsWith("audio/")) {
+              audios.push({
+                url: attachment.url,
+                desc: attachment.name ?? attachment.description ?? attachment.comment,
+                type: contentType,
+              });
+            }
+          } else {
+            const type = attachment.type?.toLowerCase();
+
+            const fileType =
+              type.indexOf("/") > -1
+                ? type
+                : type +
+                  "/" +
+                  (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? type == "image" ? "png" : type == "video" ? "mp4" : "mpeg");
+            if (type.startsWith("image")) {
+              images.push({
+                url: attachment.url,
+                desc: attachment.name ?? attachment.description ?? attachment.comment,
+                type: fileType,
+              });
+            } else if (type.startsWith("video")) {
+              videos.push({
+                url: attachment.url,
+                desc: attachment.name ?? attachment.description ?? attachment.comment,
+                type: fileType,
+              });
+            } else if (type.startsWith("audio")) {
+              audios.push({
+                url: attachment.url,
+                desc: attachment.name ?? attachment.description ?? attachment.comment,
+                type: fileType,
+              });
+            }
+          }
+        } else {
+          logger.warn("fedimbed", `Unhandled attachment structure! ${JSON.stringify(attachment)}`);
+        }
+      }
+
+      if (!spoiler && postData.sensitive && attachments.length > 0) {
+        spoiler = true;
+      }
     }
 
     if (postData.image?.url) {
@@ -985,8 +1108,7 @@ async function processUrl(msg, url, spoiler = false, command = false) {
     })
       .then((res) => res.json())
       .catch((err) => {
-        // only posts can be activity+json right now, reduce log spam
+        /*if (platform !== "cohost")*/ logger.error("fedimbed", `Failed to get author for "${url}": ${err}`);
-        if (platform !== "cohost") logger.error("fedimbed", `Failed to get author for "${url}": ${err}`);
       });
 
     if (authorData) {
@@ -998,7 +1120,7 @@ async function processUrl(msg, url, spoiler = false, command = false) {
         avatar: authorData.icon?.url,
       };
     } else {
-      // bootleg author, mainly for cohost
+      // bootleg author
       const authorUrl = postData.actor ?? postData.attributedTo;
       const authorUrlObj = new URL(authorUrl);
       const name = authorUrlObj.pathname.substring(authorUrlObj.pathname.lastIndexOf("/") + 1);
@@ -1077,6 +1199,12 @@ async function processUrl(msg, url, spoiler = false, command = false) {
           name: user,
           url: author.url,
         }
+      : context
+      ? {
+          name: context,
+          url: contextUrl,
+          icon_url: "https://cdn.discordapp.com/emojis/1308640078825787412.png",
+        }
       : null,
     footer: {
       text: platformName,
@@ -1384,6 +1512,7 @@ events.add("messageCreate", "fedimbed", async function (msg) {
           logger.verbose("fedimbed", `Hit "${service}" for "${url}", processing now.`);
           try {
             const {response, sendWait} = await processUrl(msg, url, hasSpoiler);
+            if (!response) return;
             await msg.channel.createMessage(response).then(() => {
               if (sendWait) {
                 msg.removeReaction("\uD83D\uDCE4");
@@ -1394,7 +1523,7 @@ events.add("messageCreate", "fedimbed", async function (msg) {
               }
             });
           } catch (err) {
-            logger.error("fedimbed", `Error processing "${url}":\n` + err.stack);
+            logger.error("fedimbed", `Error processing "${url}":\n${err.stack}`);
           }
           break;
         }

src/modules/foxwells.js

@@ -263,3 +263,33 @@ async function processReaction(_msg, reaction, user) {
 
 events.add("messageReactionAdd", "vinboard", processReaction);
 events.add("messageReactionRemove", "vinboard", processReaction);
+
+/* join request logger */
+/*const STAFF_CHANNEL_ID = "662007759738372096";
+
+function processJoinRequest(data) {
+  if (data.status !== "SUBMITTED") return;
+  if (!data.request) return;
+  if (data.request.guild_id !== FOXWELLS_GUILD_ID) return;
+
+  const channel = hf.bot.guilds.get(FOXWELLS_GUILD_ID).channels.get(STAFF_CHANNEL_ID);
+  const userId = data.request.user_id;
+
+  channel.createMessage({
+    embeds: [
+      {
+        title: "New join request",
+        description: `<@${userId}> (\`${userId}\`)`,
+        fields: data.request.form_responses
+          .filter((field) => field.field_type !== "TERMS")
+          .map((field) => ({name: field.label, value: `\`\`\`\n${field.response}\n\`\`\``})),
+      },
+    ],
+  });
+}
+
+events.add("unknown", "foxwells_joinrequest", (packet) => {
+  if (packet.t === "GUILD_JOIN_REQUEST_UPDATE") {
+    processJoinRequest(packet.d);
+  }
+});*/

src/modules/utility/guildinfo.js

@@ -601,7 +601,7 @@ guildinfo.callback = async function (msg, line, args, {nolocal, debug}) {
         invite = await hf.bot.requestHandler.request(
           "GET",
           `/invites/${guild.instant_invite.replace(
-            /(https?:\/\/)?discord(\.gg|(app)?.com\/invite)\//,
+            /(https?:\/\/)?(canary\.|ptb\.)?discord(\.gg|(app)?.com\/invite)\//,
             ""
           )}?with_counts=true&with_expiration=true`
         );

src/modules/utility/lookupinvite.js

@@ -23,7 +23,7 @@ lookupinvite.addAlias("ii");
 lookupinvite.callback = async function (msg, line) {
   if (!line || line == "") return "Arguments required.";
 
-  line = line.replace(/(https?:\/\/)?discord(\.gg|(app)?.com\/invite)\//, "");
+  line = line.replace(/(https?:\/\/)?(canary\.|ptb\.)?discord(\.gg|(app)?.com\/invite)\//, "");
 
   if (decodeURIComponent(line).indexOf("../") > -1) return "nuh uh";
 

src/util/html.js

@@ -32,14 +32,14 @@ function parseHtmlEntities(str) {
 
 function htmlToMarkdown(str, images = true, embed = true) {
   str = str.replaceAll("\\", "\\\\");
-  str = str.replace(/<style(\s+[^>]+)?>(.|\n)*?<\/style>/gi, "");
+  str = str.replace(/<style(\s*[^>]+)?>(.|\n)*?<\/style>/gi, "");
-  str = str.replace(/<a (\s+[^>]+)?href="([^"]+?)"(\s+[^>]+)?>(.+?)<\/a>/gi, (_, __, url, ___, text) => {
+  str = str.replace(/<a (\s*[^>]+)?href="([^"]+?)"(\s*[^>]+)?>(.+?)<\/a>/gi, (_, __, url, ___, text) => {
     url = url.replace(/^\/\//, "https://").replace("\\#", "#");
     return url == text ? url : `[${text}](${embed ? "" : "&lt;"}${url}${embed ? "" : "&gt;"})`;
   });
   if (images)
     str = str.replace(
-      /<img (\s+[^>]+)?src="([^"]+?)"(\s+[^>]+)?(alt|title)="([^"]+?)"(\s+[^>]+)?\/>/gi,
+      /<img (\s*[^>]+)?src="([^"]+?)"(\s*[^>]+)?(alt|title)="([^"]+?)"(\s*[^>]+)?\/>/gi,
       `[$5](${embed ? "" : "&lt;"}$2${embed ? "" : "&gt;"})`
     );
   str = str.replace(/<\/?\s*br\s*\/?>/gi, "\n");
@@ -49,7 +49,7 @@ function htmlToMarkdown(str, images = true, embed = true) {
   );
   str = str.replace(/<\/?p>/gi, "\n");
   str = str.replace(/<dd>((.|\n)*?)<\/dd>/gi, (_, inner) => "\u3000\u3000" + inner.split("\n").join("\n\u3000\u3000"));
-  str = str.replace(/<ol(\s+[^>]+)?>((.|\n)*?)<\/ol>/gi, (_, __, inner) => {
+  str = str.replace(/<ol(\s*[^>]+)?>((.|\n)*?)<\/ol>/gi, (_, __, inner) => {
     let index = 0;
     return inner
       .replace(/<li>/gi, () => {
@@ -59,7 +59,7 @@ function htmlToMarkdown(str, images = true, embed = true) {
       .replace(/<\/li>/gi, "\n")
       .replaceAll("\n\n", "\n");
   });
-  str = str.replace(/<ul(\s+[^>]+)?>((.|\n)*?)<\/ul>/gi, (_, __, inner) => {
+  str = str.replace(/<ul(\s*[^>]+)?>((.|\n)*?)<\/ul>/gi, (_, __, inner) => {
     let index = 0;
     return inner
       .replace(/<li>/gi, () => {
@@ -69,17 +69,18 @@ function htmlToMarkdown(str, images = true, embed = true) {
       .replace(/<\/li>/gi, "\n")
       .replaceAll("\n\n", "\n");
   });
-  str = str.replace(/<\/?code(\s+[^>]+)?>/gi, "`");
+  str = str.replace(/<\/?span(\s*[^>]+)?>/gi, "");
-  str = str.replace(/<\/?em(\s+[^>]+)?>/gi, "_");
+  str = str.replace(/<\/?code(\s*[^>]+)?>/gi, "`");
-  str = str.replace(/<\/?i(\s+[^>]+)?>/gi, "_");
+  str = str.replace(/<\/?em(\s*[^>]+)?>/gi, "_");
-  str = str.replace(/<\/?b(\s+[^>]+)?>/gi, "**");
+  str = str.replace(/<\/?i(\s*[^>]+)?>/gi, "_");
-  str = str.replace(/<\/?u(\s+[^>]+)?>/gi, "__");
+  str = str.replace(/<\/?b(\s*[^>]+)?>/gi, "**");
-  str = str.replace(/<\/?s(\s+[^>]+)?>/gi, "~~");
+  str = str.replace(/<\/?u(\s*[^>]+)?>/gi, "__");
-  str = str.replace(/<h1(\s+[^>]+)?>/gi, "# ");
+  str = str.replace(/<\/?s(\s*[^>]+)?>/gi, "~~");
-  str = str.replace(/<h2(\s+[^>]+)?>/gi, "## ");
+  str = str.replace(/<h1(\s*[^>]+)?>/gi, "# ");
-  str = str.replace(/<h3(\s+[^>]+)?>/gi, "### ");
+  str = str.replace(/<h2(\s*[^>]+)?>/gi, "## ");
-  str = str.replace(/<\/?h4(\s+[^>]+)?>/gi, "**");
+  str = str.replace(/<h3(\s*[^>]+)?>/gi, "### ");
-  str = str.replace(/<(math|noscript)(\s+[^>]+)?>((.|\n)*?)<\/(math|noscript)>/gi, "");
+  str = str.replace(/<\/?h4(\s*[^>]+)?>/gi, "**");
+  str = str.replace(/<(math|noscript)(\s*[^>]+)?>((.|\n)*?)<\/(math|noscript)>/gi, "");
   str = str.replace(/<[^>]+?>/gi, "");
   str = parseHtmlEntities(str);
   // whyyyyyyyyyyyy