Barebones matrix media proxy

package-lock.json

@@ -25,6 +25,7 @@
         "enquirer": "^2.4.1",
         "entities": "^5.0.0",
         "get-stream": "^6.0.1",
+        "h3": "^1.12.0",
         "heatsync": "^2.5.3",
         "js-yaml": "^4.1.0",
         "minimist": "^1.2.8",
@@ -34,7 +35,8 @@
         "snowtransfer": "^0.10.5",
         "stream-mime-type": "^1.0.2",
         "try-to-catch": "^3.0.1",
-        "xxhash-wasm": "^1.0.2"
+        "xxhash-wasm": "^1.0.2",
+        "zod": "^3.23.8"
       },
       "devDependencies": {
         "@cloudrac3r/tap-dot": "^2.0.2",

package.json

@@ -34,6 +34,7 @@
     "enquirer": "^2.4.1",
     "entities": "^5.0.0",
     "get-stream": "^6.0.1",
+    "h3": "^1.12.0",
     "heatsync": "^2.5.3",
     "js-yaml": "^4.1.0",
     "minimist": "^1.2.8",
@@ -43,7 +44,8 @@
     "snowtransfer": "^0.10.5",
     "stream-mime-type": "^1.0.2",
     "try-to-catch": "^3.0.1",
-    "xxhash-wasm": "^1.0.2"
+    "xxhash-wasm": "^1.0.2",
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "@cloudrac3r/tap-dot": "^2.0.2",

scripts/seed.js

@@ -3,48 +3,63 @@
 const assert = require("assert").strict
 const fs = require("fs")
 const sqlite = require("better-sqlite3")
-const {scheduler: {wait}} = require("timers/promises")
+const {scheduler} = require("timers/promises")
 const {isDeepStrictEqual} = require("util")
+const {createServer} = require("http")
 
 const {prompt} = require("enquirer")
 const Input = require("enquirer/lib/prompts/input")
 const fetch = require("node-fetch")
 const {magenta, bold, cyan} = require("ansi-colors")
 const HeatSync = require("heatsync")
+const {SnowTransfer} = require("snowtransfer")
+const {createApp, defineEventHandler, toNodeListener} = require("h3")
 
 const args = require("minimist")(process.argv.slice(2), {string: ["emoji-guild"]})
 
+// Move database file if it's still in the old location
+if (fs.existsSync("db")) {
+	if (fs.existsSync("db/ooye.db")) {
+		fs.renameSync("db/ooye.db", "src/db/ooye.db")
+	}
+	const files = fs.readdirSync("db")
+	if (files.length) {
+		console.error("You must manually move or delete the files in the db folder:")
+		for (const file of files) {
+			console.error(file)
+		}
+		process.exit(1)
+	}
+	fs.rmSync("db", {recursive: true})
+}
+
 const config = require("../config")
-const passthrough = require("../passthrough")
+const passthrough = require("../src/passthrough")
-const db = new sqlite("db/ooye.db")
+const db = new sqlite("src/db/ooye.db")
-const migrate = require("../db/migrate")
+const migrate = require("../src/db/migrate")
 
 const sync = new HeatSync({watchFS: false})
 
 Object.assign(passthrough, { sync, config, db })
 
-const orm = sync.require("../db/orm")
+const orm = sync.require("../src/db/orm")
 passthrough.from = orm.from
 passthrough.select = orm.select
 
-const DiscordClient = require("../d2m/discord-client")
+let registration = require("../src/matrix/read-registration")
-const discord = new DiscordClient(config.discordToken, "no")
+let {reg, getTemplateRegistration, writeRegistration, readRegistration, checkRegistration, registrationFilePath} = registration
-passthrough.discord = discord
-
-let registration = require("../matrix/read-registration")
-let {reg, getTemplateRegistration, writeRegistration, readRegistration, registrationFilePath} = registration
 
 function die(message) {
 	console.error(message)
 	process.exit(1)
 }
 
-async function uploadAutoEmoji(guild, name, filename) {
+async function uploadAutoEmoji(snow, guild, name, filename) {
 	let emoji = guild.emojis.find(e => e.name === name)
 	if (!emoji) {
 		console.log(`   Uploading ${name}...`)
 		const data = fs.readFileSync(filename, null)
-		emoji = await discord.snow.guildAssets.createEmoji(guild.id, {name, image: "data:image/png;base64," + data.toString("base64")})
+		emoji = await snow.guildAssets.createEmoji(guild.id, {name, image: "data:image/png;base64," + data.toString("base64")})
 	} else {
 		console.log(`   Reusing ${name}...`)
 	}
@@ -88,46 +103,90 @@ async function validateHomeserverOrigin(serverUrlPrompt, url) {
 			name: "server_name",
 			message: "Homeserver name"
 		})
+
 		console.log("What is the URL of your homeserver?")
-		const serverUrlPrompt = new Input({
+		const serverOriginPrompt = new Input({
 			type: "input",
 			name: "server_origin",
 			message: "Homeserver URL",
 			initial: () => `https://${serverNameResponse.server_name}`,
-			validate: url => validateHomeserverOrigin(serverUrlPrompt, url)
+			validate: url => validateHomeserverOrigin(serverOriginPrompt, url)
 		})
-		/** @type {{server_origin: string}} */ // @ts-ignore
+		/** @type {string} */ // @ts-ignore
-		const serverUrlResponse = await serverUrlPrompt.run()
+		const serverOrigin = await serverOriginPrompt.run()
-		console.log("Your Matrix homeserver needs to be able to send HTTP requests to OOYE.")
+
-		console.log("What URL should OOYE be reachable on? Usually, the default works fine,")
+		const app = createApp()
-		console.log("but you need to change this if you use multiple servers or containers.")
+		app.use(defineEventHandler(() => "Out Of Your Element is listening.\n"))
-		/** @type {{url: string}} */
+		const server = createServer(toNodeListener(app))
-		const urlResponse = await prompt({
+		await server.listen(6693)
+
+		console.log("OOYE has its own web server. It needs to be accessible on the public internet.")
+		console.log("You need to enter a public URL where you will be able to host this web server.")
+		console.log("OOYE listens on localhost:6693, so you will probably have to set up a reverse proxy.")
+		console.log("Now listening on port 6693. Feel free to send some test requests.")
+		/** @type {{bridge_origin: string}} */
+		const bridgeOriginResponse = await prompt({
 			type: "input",
-			name: "url",
+			name: "bridge_origin",
 			message: "URL to reach OOYE",
-			initial: "http://localhost:6693",
+			initial: () => `https://bridge.${serverNameResponse.server_name}`,
-			validate: url => !!url.match(/^https?:\/\//)
+			validate: async url => {
+				process.stdout.write(magenta(" checking, please wait..."))
+				try {
+					const res = await fetch(url)
+					if (res.status !== 200) return `Server returned status code ${res.status}`
+					const text = await res.text()
+					if (text !== "Out Of Your Element is listening.\n") return `Server does not point to OOYE`
+					return true
+				} catch (e) {
+					return e.message
+				}
+			}
+		})
+
+		await server.close()
+
+		console.log("What is your Discord bot token?")
+		/** @type {{discord_token: string}} */
+		const discordTokenResponse = await prompt({
+			type: "input",
+			name: "discord_token",
+			message: "Bot token",
+			validate: async token => {
+				process.stdout.write(magenta(" checking, please wait..."))
+				try {
+					const snow = new SnowTransfer(token)
+					await snow.user.getSelf()
+					return true
+				} catch (e) {
+					return e.message
+				}
+			}
 		})
 		const template = getTemplateRegistration()
-		reg = {...template, ...urlResponse, ooye: {...template.ooye, ...serverNameResponse, ...serverUrlResponse}}
+		reg = {...template, url: bridgeOriginResponse.bridge_origin, ooye: {...template.ooye, ...serverNameResponse, ...bridgeOriginResponse, server_origin: serverOrigin, ...discordTokenResponse}}
 		registration.reg = reg
+		checkRegistration(reg)
 		writeRegistration(reg)
-	}
-
-	// Done with user prompts, reg is now guaranteed to be valid
-	const api = require("../matrix/api")
-	const file = require("../matrix/file")
-	const utils = require("../m2d/converters/utils")
-
 		console.log(`✅ Registration file saved as ${registrationFilePath}`)
+	} else {
+		console.log(`✅ Valid registration file found at ${registrationFilePath}`)
+	}
 	console.log(`  In ${cyan("Synapse")}, you need to add it to homeserver.yaml and ${cyan("restart Synapse")}.`)
 	console.log("    https://element-hq.github.io/synapse/latest/application_services.html")
 	console.log(`  In ${cyan("Conduit")}, you need to send the file contents to the #admins room.`)
 	console.log("    https://docs.conduit.rs/appservices.html")
 	console.log()
 
-	const {as} = require("../matrix/appservice")
+	// Done with user prompts, reg is now guaranteed to be valid
+	const api = require("../src/matrix/api")
+	const file = require("../src/matrix/file")
+	const utils = require("../src/m2d/converters/utils")
+	const DiscordClient = require("../src/d2m/discord-client")
+	const discord = new DiscordClient(reg.ooye.discord_token, "no")
+	passthrough.discord = discord
+
+	const {as} = require("../src/matrix/appservice")
 	console.log("⏳ Waiting until homeserver registration works... (Ctrl+C to cancel)")
 
 	let itWorks = false
@@ -150,7 +209,7 @@ async function validateHomeserverOrigin(serverUrlPrompt, url) {
 			} else {
 				process.stderr.write(".")
 			}
-			await wait(5000)
+			await scheduler.wait(5000)
 		}
 	} while (!itWorks)
 	console.log("")
@@ -228,8 +287,8 @@ async function validateHomeserverOrigin(serverUrlPrompt, url) {
 		}
 		// Upload those emojis to the chosen location
 		db.prepare("REPLACE INTO auto_emoji (name, emoji_id, guild_id) VALUES ('_', '_', ?)").run(guild.id)
-		await uploadAutoEmoji(guild, "L1", "docs/img/L1.png")
+		await uploadAutoEmoji(discord.snow, guild, "L1", "docs/img/L1.png")
-		await uploadAutoEmoji(guild, "L2", "docs/img/L2.png")
+		await uploadAutoEmoji(discord.snow, guild, "L2", "docs/img/L2.png")
 	}
 	console.log("✅ Emojis are ready...")
 

src/db/orm-defs.d.ts

@@ -100,6 +100,10 @@ export type Models = {
 		emoji_id: string
 		guild_id: string
 	}
+
+	media_proxy: {
+		permitted_hash: number
+	}
 }
 
 export type Prepared<Row> = {

src/matrix/read-registration.js

@@ -28,9 +28,9 @@ function writeRegistration(reg) {
 /** @returns {import("../types").InitialAppServiceRegistrationConfig} reg */
 function getTemplateRegistration() {
 	return {
-		id: crypto.randomBytes(16).toString("hex"),
+		id: "ooye",
-		as_token: crypto.randomBytes(16).toString("hex"),
+		as_token: crypto.randomBytes(32).toString("hex"),
-		hs_token: crypto.randomBytes(16).toString("hex"),
+		hs_token: crypto.randomBytes(32).toString("hex"),
 		namespaces: {
 			users: [{
 				exclusive: true,
@@ -46,6 +46,7 @@ function getTemplateRegistration() {
 		],
 		sender_localpart: "_ooye_bot",
 		rate_limited: false,
+		socket: 6693,
 		ooye: {
 			namespace_prefix: "_ooye_",
 			max_file_size: 5000000,

src/types.d.ts

@@ -16,12 +16,14 @@ export type AppServiceRegistrationConfig = {
 	}
 	protocols: [string]
 	rate_limited: boolean
+	socket?: string | number,
 	ooye: {
 		namespace_prefix: string
 		max_file_size: number
 		server_name: string
 		server_origin: string
 		bridge_origin: string
+		discord_token: string
 		content_length_workaround: boolean
 		include_user_id_in_mxid: boolean
 		invite: string[]
@@ -49,6 +51,7 @@ export type InitialAppServiceRegistrationConfig = {
 	}
 	protocols: [string]
 	rate_limited: boolean
+	socket?: string | number,
 	ooye: {
 		namespace_prefix: string
 		max_file_size: number,

src/web/routes/download-matrix.js

@@ -0,0 +1,48 @@
+// @ts-check
+
+const {defineEventHandler, getValidatedRouterParams, setResponseStatus, setResponseHeader, sendStream, createError} = require("h3")
+const {z} = require("zod")
+const fetch = require("node-fetch")
+
+/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
+let hasher = null
+// @ts-ignore
+require("xxhash-wasm")().then(h => hasher = h)
+
+const {reg} = require("../../matrix/read-registration")
+const {as, select} = require("../../passthrough")
+
+const schema = {
+	params: z.object({
+		server_name: z.string(),
+		media_id: z.string()
+	})
+}
+
+as.router.get(`/download/matrix/:server_name/:media_id`, defineEventHandler(async event => {
+	const params = await getValidatedRouterParams(event, schema.params.parse)
+
+	const serverAndMediaID = `${params.server_name}/${params.media_id}`
+	const unsignedHash = hasher.h64(serverAndMediaID)
+	const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
+
+	const row = select("media_proxy", "permitted_hash", {permitted_hash: signedHash}).get()
+	if (row == null) {
+		throw createError({
+			status: 403,
+			data: `The file you requested isn't permitted by this media proxy.`
+		})
+	}
+
+	const res = await fetch(`${reg.ooye.server_origin}/_matrix/client/v1/media/download/${params.server_name}/${params.media_id}`, {
+		headers: {
+			Authorization: `Bearer ${reg.as_token}`
+		}
+	})
+
+	setResponseStatus(event, res.status)
+	setResponseHeader(event, "Content-Type", res.headers.get("content-type"))
+	setResponseHeader(event, "Transfer-Encoding", "chunked")
+
+	return sendStream(event, res.body)
+}))

src/web/server.js

@@ -0,0 +1,5 @@
+// @ts-check
+
+const {sync, as} = require("../passthrough")
+
+sync.require("./routes/download-matrix")

start.js

@@ -15,7 +15,7 @@ Object.assign(passthrough, {config, sync, db})
 
 const DiscordClient = require("./src/d2m/discord-client")
 
-const discord = new DiscordClient(config.discordToken, "full")
+const discord = new DiscordClient(config.discordToken, "no")
 passthrough.discord = discord
 
 const {as} = require("./src/matrix/appservice")
@@ -26,12 +26,13 @@ passthrough.from = orm.from
 passthrough.select = orm.select
 
 const power = require("./src/matrix/power.js")
-sync.require("./src/m2d/event-dispatcher")
+// sync.require("./src/m2d/event-dispatcher")
 
 ;(async () => {
 	await migrate.migrate(db)
 	await discord.cloud.connect()
 	console.log("Discord gateway started")
+	sync.require("./src/web/server")
 	await power.applyPower()
 
 	require("./src/stdin")