const {MessageFlags} = require("@projectdysnomia/dysnomia").Constants; const fs = require("node:fs"); const path = require("node:path"); const httpSignature = require("@peertube/http-signature"); const events = require("../lib/events.js"); const logger = require("../lib/logger.js"); const {hasFlag} = require("../lib/guildSettings.js"); const {parseHtmlEntities, getUploadLimit} = require("../lib/utils.js"); const FRIENDLY_USERAGENT = "HiddenPhox/fedimbed (https://gitdab.com/Cynosphere/HiddenPhox)"; const URLS_REGEX = /(?:\s|^)(\|\|\s*)?(https?:\/\/[^\s<]+[^<.,:;"'\]\s])(\s+\|\|)?/g; const SPOILER_REGEX = /(?:\s|^)\|\|([\s\S]+?)\|\|/; const PATH_REGEX = { mastodon: /^\/@(.+?)\/(\d+)\/?/, mastodon2: /^\/(.+?)\/statuses\/\d+\/?/, pleroma: /^\/objects\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/?/, pleroma2: /^\/notice\/[A-Za-z0-9]+\/?/, misskey: /^\/notes\/[a-z0-9]+\/?/, gotosocial: /^\/@(.+?)\/statuses\/[0-9A-Z]+\/?/, lemmy: /^\/post\/\d+\/?/, }; const PLATFORM_COLORS = { mastodon: 0x2791da, pleroma: 0xfba457, akkoma: 0x593196, misskey: 0x99c203, calckey: 0x31748f, firefish: 0xf07a5b, // YCbCr interpolated color from the two logo colors gotosocial: 0xff853e, lemmy: 0x14854f, birdsitelive: 0x1da1f2, }; const domainCache = new Map(); async function resolvePlatform(url) { const urlObj = new URL(url); if (domainCache.has(urlObj.hostname)) return domainCache.get(urlObj.hostname); const probe = await fetch(urlObj.origin + "/.well-known/nodeinfo", { headers: {"User-Agent": FRIENDLY_USERAGENT}, }).then((res) => res.json()); if (!probe?.links) { logger.error("fedimbed", `No nodeinfo for "${urlObj.hostname}"???`); domainCache.set(urlObj.hostname, null); return null; } const nodeinfo = await fetch(probe.links[probe.links.length - 1].href, { headers: {"User-Agent": FRIENDLY_USERAGENT}, }).then((res) => res.json()); if (!nodeinfo?.software?.name) { logger.error( "fedimbed", `Got nodeinfo for "${urlObj.hostname}", but missing software name.` ); domainCache.set(urlObj.hostname, null); return null; } domainCache.set(urlObj.hostname, nodeinfo.software.name); return nodeinfo.software.name; } const keyId = "https://hf.c7.pm/actor#main-key"; const privKey = fs.readFileSync( path.resolve(__dirname, "../../priv/private.pem") ); async function signedFetch(url, options) { const urlObj = new URL(url); const headers = { host: urlObj.host, date: new Date().toUTCString(), }; const headerNames = ["(request-target)", "host", "date"]; httpSignature.sign( { getHeader: (name) => headers[name.toLowerCase()], setHeader: (name, value) => (headers[name] = value), method: options.method ?? "GET", path: urlObj.pathname, }, { keyId, key: privKey, headers: headerNames, authorizationHeaderName: "signature", } ); options.headers = Object.assign(headers, options.headers ?? {}); return await fetch(url, options); } async function processUrl(msg, url, spoiler = false) { let urlObj = new URL(url); // some lemmy instances have old reddit frontend subdomains // but these frontends are just frontends and dont actually expose the API if (urlObj.hostname.startsWith("old.")) { urlObj.hostname = urlObj.hostname.replace("old.", ""); url = urlObj.href; } let platform = await resolvePlatform(url); let color = PLATFORM_COLORS[platform]; let platformName = platform .replace("gotosocial", "GoToSocial") .replace("birdsitelive", '"Twitter" (BirdsiteLive)') .replace(/^(.)/, (_, c) => c.toUpperCase()); const images = []; const videos = []; const audios = []; let content, cw, author, timestamp, title, emotes = []; // Fetch post let rawPostData; try { rawPostData = await signedFetch(url, { headers: { "User-Agent": FRIENDLY_USERAGENT, Accept: "application/activity+json", }, }).then((res) => res.text()); } catch (err) { logger.error( "fedimbed", `Failed to signed fetch "${url}", retrying unsigned: ${err}` ); } if (!rawPostData) { try { rawPostData = await fetch(url, { headers: { "User-Agent": FRIENDLY_USERAGENT, Accept: "application/activity+json", }, }).then((res) => res.text()); } catch (err) { logger.error("fedimbed", `Failed to fetch "${url}": ${err}`); } } let postData; if (rawPostData?.startsWith("{")) { postData = JSON.parse(rawPostData); } else { logger.warn("fedimbed", `Got non-JSON for "${url}": ${rawPostData}`); } if (postData?.error) { logger.error("fedimbed", `Received error for "${url}": ${postData.error}`); } if (!postData) { // We failed to get post. // Assume it was due to AFM or forced HTTP signatures and use MastoAPI // Follow redirect from /object since we need the ID from /notice if (PATH_REGEX.pleroma.test(urlObj.pathname)) { url = await signedFetch(url, { method: "HEAD", headers: { "User-Agent": FRIENDLY_USERAGENT, }, redirect: "manual", }).then((res) => res.headers.get("location")); if (url.startsWith("/")) { url = urlObj.origin + url; } urlObj = new URL(url); } let redirUrl; const options = {}; const headers = {}; if (PATH_REGEX.pleroma2.test(urlObj.pathname)) { redirUrl = url.replace("notice", "api/v1/statuses"); } else if (PATH_REGEX.mastodon.test(urlObj.pathname)) { const postId = urlObj.pathname.match(PATH_REGEX.mastodon)?.[2]; redirUrl = urlObj.origin + "/api/v1/statuses/" + postId; } else if (PATH_REGEX.mastodon2.test(urlObj.pathname)) { redirUrl = url.replace(/^\/(.+?)\/statuses/, "/api/v1/statuses"); } else if (PATH_REGEX.misskey.test(urlObj.pathname)) { let noteId = url.split("/notes/")[1]; if (noteId.indexOf("/") > -1) { noteId = noteId.split("/")[0]; } else if (noteId.indexOf("?") > -1) { noteId = noteId.split("?")[0]; } else if (noteId.indexOf("#") > -1) { noteId = noteId.split("#")[0]; } logger.verbose("fedimbed", "Misskey post ID: " + noteId); redirUrl = urlObj.origin + "/api/notes/show/"; options.method = "POST"; options.body = JSON.stringify({noteId}); headers["Content-Type"] = "application/json"; } else { logger.error( "fedimbed", `Missing MastoAPI replacement for "${platform}"` ); } if (redirUrl) { logger.verbose( "fedimbed", `Redirecting "${url}" to "${redirUrl}": ${JSON.stringify( options )}, ${JSON.stringify(headers)}` ); let rawPostData2; try { rawPostData2 = await signedFetch( redirUrl, Object.assign(options, { headers: Object.assign(headers, { "User-Agent": FRIENDLY_USERAGENT, }), }) ).then((res) => res.text()); } catch (err) { logger.error( "fedimbed", `Failed to signed fetch "${url}" via MastoAPI, retrying unsigned: ${err}` ); } if (!rawPostData2) { try { rawPostData2 = await signedFetch( redirUrl, Object.assign(options, { headers: Object.assign(headers, { "User-Agent": FRIENDLY_USERAGENT, }), }) ).then((res) => res.text()); } catch (err) { logger.error( "fedimbed", `Failed to fetch "${url}" via MastoAPI: ${err}` ); } } let postData2; if (rawPostData2?.startsWith("{")) { postData2 = JSON.parse(rawPostData2); } else { logger.warn( "fedimbed", `Got non-JSON for "${url}" via MastoAPI: ${rawPostData2}` ); } if (!postData2) { logger.warn( "fedimbed", `Bailing trying to re-embed "${url}": Failed to get post from normal and MastoAPI.` ); } else if (postData2.error) { logger.error( "fedimbed", `Bailing trying to re-embed "${url}", MastoAPI gave us error: ${JSON.stringify( postData2.error )}` ); } else { cw = postData2.spoiler_warning ?? postData2.spoiler_text ?? postData2.cw; content = postData2.akkoma?.source?.content ?? postData2.pleroma?.content?.["text/plain"] ?? postData2.text ?? postData2.content; author = { name: postData2.account?.display_name ?? postData2.account?.username ?? postData2.user?.name ?? postData2.user?.username, handle: postData2.account?.fqn ?? `${postData2.account?.username ?? postData2.user?.username}@${ urlObj.hostname }`, url: postData2.account?.url ?? `${urlObj.origin}/@${ postData2.account?.username ?? postData2.user?.username }`, avatar: postData2.account?.avatar ?? postData2.user?.avatarUrl, }; timestamp = postData2.created_at ?? postData2.createdAt; if (!spoiler && postData2.sensitive) { spoiler = true; } emotes = postData2.emojis .filter((x) => !x.name.endsWith("@.")) .map((x) => ({name: `:${x.name}:`, url: x.url})); const attachments = postData2.media_attachments ?? postData2.files; if (attachments) { for (const attachment of attachments) { const fileType = attachment.pleroma?.mime_type ?? attachment.type.indexOf("/") > -1 ? attachment.type : attachment.type + "/" + (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? attachment.type == "image" ? "png" : attachment.type == "video" ? "mp4" : "mpeg"); if (attachment.type.startsWith("image")) { images.push({ url: attachment.url, desc: attachment.description ?? attachment.comment, type: fileType, }); } else if (attachment.type.startsWith("video")) { videos.push({ url: attachment.url, desc: attachment.description ?? attachment.comment, type: fileType, }); } else if (attachment.type.startsWith("audio")) { audios.push({ url: attachment.url, desc: attachment.description ?? attachment.comment, type: fileType, }); } } } } } } else { if (postData.id) { const realUrlObj = new URL(postData.id); if (realUrlObj.origin != urlObj.origin) { platform = await resolvePlatform(postData.id); color = PLATFORM_COLORS[platform]; platformName = platform .replace("gotosocial", "GoToSocial") .replace(/^(.)/, (_, c) => c.toUpperCase()); url = postData.id; } } content = postData._misskey_content ?? postData.source?.content ?? postData.content; cw = postData.summary; timestamp = postData.published; if (!spoiler && postData.sensitive) { spoiler = true; } if (postData.tag) emotes = postData.tag .filter((x) => !!x.icon) .map((x) => ({name: x.name, url: x.icon.url})); // NB: gts doesnt send singular attachments as areay const attachments = Array.isArray(postData.attachment) ? postData.attachment : [postData.attachment]; for (const attachment of attachments) { if (attachment.mediaType) { if (attachment.mediaType.startsWith("video/")) { videos.push({ url: attachment.url, desc: attachment.name, type: attachment.mediaType, }); } else if (attachment.mediaType.startsWith("image/")) { images.push({ url: attachment.url, desc: attachment.name, type: attachment.mediaType, }); } else if (attachment.mediaType.startsWith("audio/")) { audios.push({ url: attachment.url, desc: attachment.name, type: attachment.mediaType, }); } } } if (postData.image?.url) { const imageUrl = new URL(postData.image?.url); images.push({ url: postData.image?.url, desc: "", type: "image/" + imageUrl.pathname.substring(imageUrl.pathname.lastIndexOf(".") + 1), }); } if (postData.name) title = postData.name; // Author data is not sent with the post with AS2 const authorData = await signedFetch( postData.actor ?? postData.attributedTo, { headers: { "User-Agent": FRIENDLY_USERAGENT, Accept: "application/activity+json", }, } ) .then((res) => res.json()) .catch((err) => { logger.error("fedimbed", `Failed to get author for "${url}": ${err}`); }); if (authorData) { const authorUrlObj = new URL(authorData.url ?? authorData.id); author = { name: authorData.name, handle: `${authorData.preferredUsername}@${authorUrlObj.hostname}`, url: authorData.url, avatar: authorData.icon?.url, }; } } // We could just continue without author but it'd look ugly and be confusing. if (!author) { logger.warn( "fedimbed", `Bailing trying to re-embed "${url}": Failed to get author.` ); return; } // Start constructing embed content = content ?? ""; cw = cw ?? ""; // TODO: convert certain HTML tags back to markdown content = content.replace(/<\/?\s*br\s*\/?>/g, "\n"); content = content.replace(/<\/p>

/g, "\n\n"); content = content.replace(/

    /g, "\n"); content = content.replace(/
  1. /g, "- "); content = content.replace(/<\/li>/g, "\n"); content = content.replace( /([^<]+?)<\/a>/g, "[$2]($1)" ); content = content.replace(/<\/?code>/g, "`"); content = content.replace(/<\/?em>/g, "*"); content = content.replace(/<\/?u>/g, "__"); content = content.replace(/<\/?s>/g, "~~"); content = content.replace(/(<([^>]+)>)/gi, ""); content = parseHtmlEntities(content); for (const emote of emotes) { content = content.replaceAll(emote.name, `[${emote.name}](${emote.url})`); } cw = cw.replace(/(<([^>]+)>)/gi, ""); cw = parseHtmlEntities(cw); let desc = ""; let MAX_LENGTH = 3999; if ( cw != "" && images.length == 0 && videos.length == 0 && audios.length == 0 ) { desc += "\u26a0 " + cw + "\n\n||" + content + "||"; MAX_LENGTH -= 8 - cw.length; } else { desc = content; } if (desc.length > MAX_LENGTH) { if (desc.endsWith("||")) { desc = desc.substring(0, MAX_LENGTH - 2); desc += "\u2026||"; } else { desc = desc.substring(0, MAX_LENGTH) + "\u2026"; } } const user = author.name ? `${author.name} (${author.handle})` : author.handle; const baseEmbed = { color, url, timestamp, description: desc, title: title ?? user, author: title ? { name: user, url: author.url, } : null, footer: { text: platformName, }, thumbnail: { url: author.avatar, }, fields: [], }; if (images.length > 0) { if (images.length > 1) { baseEmbed.fields.push({ name: "Images", value: images .map((attachment, index) => `[Image ${index + 1}](${attachment.url})`) .join(" | "), inline: true, }); } else { baseEmbed.fields.push({ name: "Image", value: `[Click for image](${images[0].url})`, inline: true, }); } } if (videos.length > 0) { if (videos.length > 1) { baseEmbed.fields.push({ name: "Videos", value: videos .map((attachment, index) => `[Video ${index + 1}](${attachment.url})`) .join(" | "), inline: true, }); } else { baseEmbed.fields.push({ name: "Video", value: `[Click for video](${videos[0].url})`, inline: true, }); } } if (audios.length > 0) { if (audios.length > 1) { baseEmbed.fields.push({ name: "Audios", value: audios .map((attachment, index) => `[Audio ${index + 1}](${attachment.url})`) .join(" | "), inline: true, }); } else { baseEmbed.fields.push({ name: "Audio", value: `[Click for audio](${audios[0].url})`, inline: true, }); } } const embeds = []; if (images.length > 0) { for (const attachment of images) { const embed = Object.assign({}, baseEmbed); embed.image = { url: attachment.url, }; embeds.push(embed); } } else { embeds.push(baseEmbed); } const files = []; if (videos.length > 0) { for (const attachment of videos) { const size = await fetch(attachment.url, { method: "HEAD", headers: { "User-Agent": FRIENDLY_USERAGENT, }, }).then((res) => Number(res.headers.get("Content-Length"))); if (size <= getUploadLimit(msg.channel.guild)) { const file = await fetch(attachment.url, { headers: { "User-Agent": FRIENDLY_USERAGENT, }, }) .then((res) => res.arrayBuffer()) .then((buf) => Buffer.from(buf)); files.push({ filename: (cw != "" || spoiler ? "SPOILER_" : "") + (attachment.type.indexOf("/") > -1 ? attachment.type.replace("/", ".") : attachment.type + "." + (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? "mp4")), file, }); } } } if (audios.length > 0) { for (const attachment of audios) { const size = await fetch(attachment.url, { method: "HEAD", headers: { "User-Agent": FRIENDLY_USERAGENT, }, }).then((res) => Number(res.headers.get("Content-Length"))); if (size <= getUploadLimit(msg.channel.guild)) { const file = await fetch(attachment.url, { headers: { "User-Agent": FRIENDLY_USERAGENT, }, }) .then((res) => res.arrayBuffer()) .then((buf) => Buffer.from(buf)); files.push({ filename: (cw != "" || spoiler ? "SPOILER_" : "") + (attachment.type.indexOf("/") > -1 ? attachment.type .replace("/", ".") .replace("mpeg", "mp3") .replace("vnd.wave", "wav") .replace("x-", "") : attachment.type + "." + (url.match(/\.([a-z0-9]{3,4})$/)?.[0] ?? "mp3")), file, }); } } } let sendWait = false; if (files.length > 0) { sendWait = true; await msg.addReaction("\uD83D\uDCE4"); } await msg.channel .createMessage({ content: cw != "" && (images.length > 0 || videos.length > 0 || audios.length > 0) ? `:warning: ${cw} || ${url} ||` : spoiler ? `|| ${url} ||` : "", embeds, attachments: files, allowedMentions: { repliedUser: false, }, messageReference: { messageID: msg.id, }, }) .then(() => { if (sendWait) { msg.removeReaction("\uD83D\uDCE4"); } if ((msg.flags & MessageFlags.SUPPRESS_EMBEDS) === 0) { msg.edit({flags: MessageFlags.SUPPRESS_EMBEDS}).catch(() => {}); } }); } events.add("messageCreate", "fedimbed", async function (msg) { if (msg.author.id == hf.bot.user.id) return; if (!msg.guildID) return; if (!(await hasFlag(msg.guildID, "fedimbed"))) return; if (!msg.content || msg.content == "") return; if (URLS_REGEX.test(msg.content)) { const urls = msg.content.match(URLS_REGEX); for (let url of urls) { const hasSpoiler = SPOILER_REGEX.test(url); url = url .replace(/\|/g, "") .trim() .replace("@\u200b", "@") .replace("@%E2%80%8B", "@"); for (const service of Object.keys(PATH_REGEX)) { const regex = PATH_REGEX[service]; const urlObj = new URL(url); if (regex.test(urlObj.pathname)) { logger.verbose( "fedimbed", `Hit "${service}" for "${url}", processing now.` ); await processUrl(msg, url, hasSpoiler).catch((err) => { logger.error( "fedimbed", `Error processing "${url}":\n` + err.stack ); }); break; } } } } });