From c6175e09f82290bc3624490e44e313c1a7ca4a7b Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Sep 2024 01:39:41 +1200 Subject: [PATCH] Generate public url for linked discord attachments --- src/d2m/converters/edit-to-changes.test.js | 4 +- src/d2m/converters/message-to-event.js | 9 +-- src/d2m/converters/message-to-event.test.js | 10 ++-- src/discord/utils.js | 10 ++++ src/web/routes/download-discord.js | 64 +++++++++++++++++++++ src/web/server.js | 1 + 6 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 src/web/routes/download-discord.js diff --git a/src/d2m/converters/edit-to-changes.test.js b/src/d2m/converters/edit-to-changes.test.js index b561bb13..9721a859 100644 --- a/src/d2m/converters/edit-to-changes.test.js +++ b/src/d2m/converters/edit-to-changes.test.js @@ -99,9 +99,9 @@ test("edit2changes: change file type", async t => { t.deepEqual(eventsToRedact, ["$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ"]) t.deepEqual(eventsToSend, [{ $type: "m.room.message", - body: "📝 Uploaded file: https://cdn.discordapp.com/attachments/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt (20 MB)", + body: "📝 Uploaded file: https://bridge.example.org/download/discordcdn/112760669178241024/1141501302497615912/gaze_into_my_dark_mind.txt (20 MB)", format: "org.matrix.custom.html", - formatted_body: "📝 Uploaded file: gaze_into_my_dark_mind.txt (20 MB)", + formatted_body: "📝 Uploaded file: gaze_into_my_dark_mind.txt (20 MB)", "m.mentions": {}, msgtype: "m.text" }]) diff --git a/src/d2m/converters/message-to-event.js b/src/d2m/converters/message-to-event.js index 4614082b..bd445378 100644 --- a/src/d2m/converters/message-to-event.js +++ b/src/d2m/converters/message-to-event.js @@ -103,6 +103,7 @@ const embedTitleParser = markdown.markdownEngine.parserFor({ * @param {DiscordTypes.APIAttachment} attachment */ async function attachmentToEvent(mentions, attachment) { + const publicURL = dUtils.getPublicUrlForCdn(attachment.url) const emoji = attachment.content_type?.startsWith("image/jp") ? "📸" : attachment.content_type?.startsWith("image/") ? "🖼️" @@ -116,9 +117,9 @@ async function attachmentToEvent(mentions, attachment) { $type: "m.room.message", "m.mentions": mentions, msgtype: "m.text", - body: `${emoji} Uploaded SPOILER file: ${attachment.url} (${pb(attachment.size)})`, + body: `${emoji} Uploaded SPOILER file: ${publicURL} (${pb(attachment.size)})`, format: "org.matrix.custom.html", - formatted_body: `
${emoji} Uploaded SPOILER file: ${attachment.url} (${pb(attachment.size)})
` + formatted_body: `
${emoji} Uploaded SPOILER file: ${publicURL} (${pb(attachment.size)})
` } } // for large files, always link them instead of uploading so I don't use up all the space in the content repo @@ -127,9 +128,9 @@ async function attachmentToEvent(mentions, attachment) { $type: "m.room.message", "m.mentions": mentions, msgtype: "m.text", - body: `${emoji} Uploaded file: ${attachment.url} (${pb(attachment.size)})`, + body: `${emoji} Uploaded file: ${publicURL} (${pb(attachment.size)})`, format: "org.matrix.custom.html", - formatted_body: `${emoji} Uploaded file: ${attachment.filename} (${pb(attachment.size)})` + formatted_body: `${emoji} Uploaded file: ${attachment.filename} (${pb(attachment.size)})` } } else if (attachment.content_type?.startsWith("image/") && attachment.width && attachment.height) { return { diff --git a/src/d2m/converters/message-to-event.test.js b/src/d2m/converters/message-to-event.test.js index 3df1130e..a76741f5 100644 --- a/src/d2m/converters/message-to-event.test.js +++ b/src/d2m/converters/message-to-event.test.js @@ -305,9 +305,9 @@ test("message2event: spoiler attachment", async t => { $type: "m.room.message", "m.mentions": {}, msgtype: "m.text", - body: "📄 Uploaded SPOILER file: https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)", + body: "📄 Uploaded SPOILER file: https://bridge.example.org/download/discordcdn/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)", format: "org.matrix.custom.html", - formatted_body: "
📄 Uploaded SPOILER file: https://cdn.discordapp.com/attachments/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)
" + formatted_body: "
📄 Uploaded SPOILER file: https://bridge.example.org/download/discordcdn/1100319550446252084/1147465564307079258/SPOILER_69-GNDP-CADENCE.nfs.gci (74 KB)
" }]) }) @@ -788,7 +788,7 @@ test("message2event: very large attachment is linked instead of being uploaded", content: "hey", attachments: [{ filename: "hey.jpg", - url: "https://discord.com/404/hey.jpg", + url: "https://cdn.discordapp.com/attachments/123/456/789.mega", content_type: "application/i-made-it-up", size: 100e6 }] @@ -802,9 +802,9 @@ test("message2event: very large attachment is linked instead of being uploaded", $type: "m.room.message", "m.mentions": {}, msgtype: "m.text", - body: "📄 Uploaded file: https://discord.com/404/hey.jpg (100 MB)", + body: "📄 Uploaded file: https://bridge.example.org/download/discordcdn/123/456/789.mega (100 MB)", format: "org.matrix.custom.html", - formatted_body: '📄 Uploaded file: hey.jpg (100 MB)' + formatted_body: '📄 Uploaded file: hey.jpg (100 MB)' }]) }) diff --git a/src/discord/utils.js b/src/discord/utils.js index 865b2e3b..85d86cce 100644 --- a/src/discord/utils.js +++ b/src/discord/utils.js @@ -3,6 +3,8 @@ const DiscordTypes = require("discord-api-types/v10") const assert = require("assert").strict +const {reg} = require("../matrix/read-registration") + const EPOCH = 1420070400000 /** @@ -117,6 +119,13 @@ function timestampToSnowflakeInexact(timestamp) { return String((timestamp - EPOCH) * 2**22) } +/** @param {string} url */ +function getPublicUrlForCdn(url) { + const match = url.match(`https://cdn.discordapp.com/attachments/([0-9]+)/([0-9]+)/([-A-Za-z0-9_.,]+)`) + if (!match) return url + return `${reg.ooye.bridge_origin}/download/discordcdn/${match[1]}/${match[2]}/${match[3]}` +} + module.exports.getPermissions = getPermissions module.exports.hasPermission = hasPermission module.exports.hasSomePermissions = hasSomePermissions @@ -125,3 +134,4 @@ module.exports.isWebhookMessage = isWebhookMessage module.exports.isEphemeralMessage = isEphemeralMessage module.exports.snowflakeToTimestampExact = snowflakeToTimestampExact module.exports.timestampToSnowflakeInexact = timestampToSnowflakeInexact +module.exports.getPublicUrlForCdn = getPublicUrlForCdn diff --git a/src/web/routes/download-discord.js b/src/web/routes/download-discord.js new file mode 100644 index 00000000..f5871d3c --- /dev/null +++ b/src/web/routes/download-discord.js @@ -0,0 +1,64 @@ +// @ts-check + +const assert = require("assert/strict") +const {defineEventHandler, getValidatedRouterParams, sendRedirect, createError} = require("h3") +const {z} = require("zod") + +const {discord, as, select} = require("../../passthrough") + +const schema = { + params: z.object({ + channel_id: z.string().regex(/^[0-9]+$/), + attachment_id: z.string().regex(/^[0-9]+$/), + file_name: z.string().regex(/^[-A-Za-z0-9_.,]+$/) + }) +} + +/** @type {Map>} */ +const cache = new Map() + +function hasExpired(url) { + const params = new URL(url).searchParams + const ex = params.get("ex") + assert(ex) // refreshed urls from the discord api always include this parameter + return parseInt(ex, 16) < Date.now() / 1000 +} + +// purge expired urls from cache every hour +setInterval(() => { + for (const entry of cache.entries()) { + if (hasExpired(entry[1])) cache.delete(entry[0]) + } + console.log(`purged discord media cache, it now has ${cache.size} urls`) +}, 60 * 60 * 1000).unref() + +as.router.get(`/download/discordcdn/:channel_id/:attachment_id/:file_name`, defineEventHandler(async event => { + const params = await getValidatedRouterParams(event, schema.params.parse) + + const row = select("channel_room", "channel_id", {channel_id: params.channel_id}).get() + if (row == null) { + throw createError({ + status: 403, + data: `The file you requested isn't permitted by this media proxy.` + }) + } + + const url = `https://cdn.discordapp.com/attachments/${params.channel_id}/${params.attachment_id}/${params.file_name}` + let promise = cache.get(url) + let refreshed + if (promise) { + console.log("using existing cache entry") + refreshed = await promise + if (hasExpired(refreshed)) promise = undefined + console.log(promise) + } + if (!promise) { + console.log("refreshing and storing") + promise = discord.snow.channel.refreshAttachmentURLs([url]).then(x => x.refreshed_urls[0].refreshed) + cache.set(url, promise) + refreshed = await promise + } + assert(refreshed) // will have been assigned by one of the above branches + + return sendRedirect(event, refreshed) +})) diff --git a/src/web/server.js b/src/web/server.js index 7d77e62b..967885ce 100644 --- a/src/web/server.js +++ b/src/web/server.js @@ -3,3 +3,4 @@ const {sync, as} = require("../passthrough") sync.require("./routes/download-matrix") +sync.require("./routes/download-discord")