diff --git a/d2m/actions/register-user.js b/d2m/actions/register-user.js index 05f8518..64e0ceb 100644 --- a/d2m/actions/register-user.js +++ b/d2m/actions/register-user.js @@ -11,10 +11,6 @@ const api = sync.require("../../matrix/api") const file = sync.require("../../matrix/file") /** @type {import("../converters/user-to-mxid")} */ const userToMxid = sync.require("../converters/user-to-mxid") -/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore -let hasher = null -// @ts-ignore -require("xxhash-wasm")().then(h => hasher = h) /** * A sim is an account that is being simulated by the bridge to copy events from the other side. @@ -124,9 +120,7 @@ async function memberToStateContent(user, member, guildID) { } function hashProfileContent(content) { - const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`) - const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range - return signedHash + return `${content.displayname}\u0000${content.avatar_url}` } /** @@ -143,11 +137,11 @@ async function syncUser(user, member, guildID, roomID) { const mxid = await ensureSimJoined(user, roomID) const content = await memberToStateContent(user, member, guildID) const currentHash = hashProfileContent(content) - const existingHash = select("sim_member", "hashed_profile_content", "WHERE room_id = ? AND mxid = ?").safeIntegers().pluck().get(roomID, mxid) + const existingHash = select("sim_member", "profile_event_content_hash", "WHERE room_id = ? AND mxid = ?").pluck().get(roomID, mxid) // only do the actual sync if the hash has changed since we last looked if (existingHash !== currentHash) { await api.sendState(roomID, "m.room.member", mxid, content, mxid) - db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) + db.prepare("UPDATE sim_member SET profile_event_content_hash = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid) } return mxid } diff --git a/db/migrate.js b/db/migrate.js deleted file mode 100644 index 7c1faf9..0000000 --- a/db/migrate.js +++ /dev/null @@ -1,42 +0,0 @@ -// @ts-check - -const fs = require("fs") -const {join} = require("path") - -async function migrate(db) { - let files = fs.readdirSync(join(__dirname, "migrations")) - files = files.sort() - db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run() - let progress = db.prepare("SELECT * FROM migration").pluck().get() - if (!progress) { - progress = "" - db.prepare("INSERT INTO migration VALUES ('')").run() - } - - let migrationRan = false - - for (const filename of files) { - if (progress >= filename) continue - console.log(`Applying database migration ${filename}`) - if (filename.endsWith(".sql")) { - const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8") - db.exec(sql) - } else if (filename.endsWith(".js")) { - await require("./" + join("migrations", filename))(db) - } else { - continue - } - - migrationRan = true - db.transaction(() => { - db.prepare("DELETE FROM migration").run() - db.prepare("INSERT INTO migration VALUES (?)").run(filename) - })() - } - - if (migrationRan) { - console.log("Database migrations all done.") - } -} - -module.exports.migrate = migrate diff --git a/db/migrations/0001-schema.sql b/db/migrations/0001-schema.sql index b02fac3..13fbc1b 100644 --- a/db/migrations/0001-schema.sql +++ b/db/migrations/0001-schema.sql @@ -1,5 +1,3 @@ -BEGIN TRANSACTION; - CREATE TABLE IF NOT EXISTS "sim" ( "discord_id" TEXT NOT NULL, "sim_name" TEXT NOT NULL UNIQUE, @@ -88,5 +86,3 @@ CREATE TABLE IF NOT EXISTS "reaction" ( "encoded_emoji" TEXT NOT NULL, PRIMARY KEY ("hashed_event_id") ) WITHOUT ROWID; - -COMMIT; diff --git a/db/migrations/0002-optimise-profile-content.up.js b/db/migrations/0002-optimise-profile-content.js similarity index 60% rename from db/migrations/0002-optimise-profile-content.up.js rename to db/migrations/0002-optimise-profile-content.js index a8619cf..22acdfa 100644 --- a/db/migrations/0002-optimise-profile-content.up.js +++ b/db/migrations/0002-optimise-profile-content.js @@ -1,13 +1,13 @@ module.exports = async function(db) { const hasher = await require("xxhash-wasm")() - const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all() + const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member").pluck().all() const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?") db.transaction(() => { - for (let s of contents) { - let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s)) - const unsignedHash = hasher.h64Raw(b) + for (const s of contents) { + if (!Buffer.isBuffer(s)) s = Buffer.from(s) + const unsignedHash = hasher.h64(eventID) const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range - stmt.run(signedHash, s) + stmt.run(s, signedHash) } })() } diff --git a/db/migrations/0002-optimise-profile-content.sql b/db/migrations/0002-optimise-profile-content.sql index c648b24..6deeb9f 100644 --- a/db/migrations/0002-optimise-profile-content.sql +++ b/db/migrations/0002-optimise-profile-content.sql @@ -1,5 +1,3 @@ -BEGIN TRANSACTION; - -- Change hashed_profile_content column affinity to INTEGER CREATE TABLE "new_sim_member" ( @@ -15,6 +13,4 @@ DROP TABLE sim_member; ALTER TABLE new_sim_member RENAME TO sim_member; -COMMIT; - VACUUM; diff --git a/db/orm-utils.d.ts b/db/orm-utils.d.ts index e0c2cb4..58eeb28 100644 --- a/db/orm-utils.d.ts +++ b/db/orm-utils.d.ts @@ -54,7 +54,7 @@ export type Models = { sim_member: { mxid: string room_id: string - hashed_profile_content: number + profile_event_content_hash: any } webhook: { @@ -79,9 +79,8 @@ export type Models = { export type Prepared = { pluck: () => Prepared - safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}> all: (..._: any[]) => Row[] - get: (..._: any[]) => Row | null + get: (..._: any[]) => Row? } export type AllKeys = U extends any ? keyof U : never diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index b3938d1..b028dc5 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -457,19 +457,12 @@ async function eventToMessage(event, guild, di) { } } else if (event.type === "m.sticker") { content = "" + let filename = event.content.body + if (event.type === "m.sticker" && event.content.info.mimetype.includes("/")) { + filename += "." + event.content.info.mimetype.split("/")[1] + } const url = utils.getPublicUrlForMxc(event.content.url) assert(url) - let filename = event.content.body - if (event.type === "m.sticker") { - let mimetype - if (event.content.info?.mimetype?.includes("/")) { - mimetype = event.content.info.mimetype - } else { - const res = await fetch(url, {method: "HEAD"}) - mimetype = res.headers.get("content-type") || "image/webp" - } - filename += "." + mimetype.split("/")[1] - } attachments.push({id: "0", filename}) pendingFiles.push({name: filename, url}) } diff --git a/matrix/file.js b/matrix/file.js index 2dd64f1..bf5a588 100644 --- a/matrix/file.js +++ b/matrix/file.js @@ -13,13 +13,6 @@ const IMAGE_SIZE = 1024 /** @type {Map>} */ const inflight = new Map() -/** - * @param {string} url - */ -function _removeExpiryParams(url) { - return url.replace(/\?(?:(?:ex|is|sg|hm)=[a-f0-9]+&?)*$/, "") -} - /** * @param {string} path */ @@ -33,17 +26,14 @@ async function uploadDiscordFileToMxc(path) { url = DISCORD_IMAGES_BASE + path } - // Discord attachment content is always the same no matter what their ?ex parameter is. - const urlNoExpiry = _removeExpiryParams(url) - // Are we uploading this file RIGHT NOW? Return the same inflight promise with the same resolution - const existingInflight = inflight.get(urlNoExpiry) + const existingInflight = inflight.get(url) if (existingInflight) { return existingInflight } // Has this file already been uploaded in the past? Grab the existing copy from the database. - const existingFromDb = select("file", "mxc_url", "WHERE discord_url = ?").pluck().get(urlNoExpiry) + const existingFromDb = select("file", "mxc_url", "WHERE discord_url = ?").pluck().get(url) if (typeof existingFromDb === "string") { return existingFromDb } @@ -51,15 +41,15 @@ async function uploadDiscordFileToMxc(path) { // Download from Discord const promise = fetch(url, {}).then(/** @param {import("node-fetch").Response} res */ async res => { // Upload to Matrix - const root = await module.exports._actuallyUploadDiscordFileToMxc(urlNoExpiry, res) + const root = await module.exports._actuallyUploadDiscordFileToMxc(url, res) // Store relationship in database - db.prepare("INSERT INTO file (discord_url, mxc_url) VALUES (?, ?)").run(urlNoExpiry, root.content_uri) - inflight.delete(urlNoExpiry) + db.prepare("INSERT INTO file (discord_url, mxc_url) VALUES (?, ?)").run(url, root.content_uri) + inflight.delete(url) return root.content_uri }) - inflight.set(urlNoExpiry, promise) + inflight.set(url, promise) return promise } @@ -118,4 +108,3 @@ module.exports.stickerFormat = stickerFormat module.exports.sticker = sticker module.exports.uploadDiscordFileToMxc = uploadDiscordFileToMxc module.exports._actuallyUploadDiscordFileToMxc = _actuallyUploadDiscordFileToMxc -module.exports._removeExpiryParams = _removeExpiryParams diff --git a/matrix/file.test.js b/matrix/file.test.js deleted file mode 100644 index 2d59dbc..0000000 --- a/matrix/file.test.js +++ /dev/null @@ -1,22 +0,0 @@ -// @ts-check - -const {test} = require("supertape") -const file = require("./file") - -test("removeExpiryParams: url without params is unchanged", t => { - const url = "https://cdn.discordapp.com/attachments/1154455830591176734/1157034603496882267/59ce542f-bf66-4d9a-83b7-ad6d05a69bac.jpg" - const result = file._removeExpiryParams(url) - t.equal(result, url) -}) - -test("removeExpiryParams: params are removed", t => { - const url = "https://cdn.discordapp.com/attachments/112760669178241024/1157363960518029322/image.png?ex=651856ae&is=6517052e&hm=88353defb15cbd833e6977817e8f72f4ff28f4edfd26b8ad5f267a4f2b946e69&" - const result = file._removeExpiryParams(url) - t.equal(result, "https://cdn.discordapp.com/attachments/112760669178241024/1157363960518029322/image.png") -}) - -test("removeExpiryParams: rearranged params are removed", t => { - const url = "https://cdn.discordapp.com/attachments/112760669178241024/1157363960518029322/image.png?hm=88353defb15cbd833e6977817e8f72f4ff28f4edfd26b8ad5f267a4f2b946e69&ex=651856ae&is=6517052e" - const result = file._removeExpiryParams(url) - t.equal(result, "https://cdn.discordapp.com/attachments/112760669178241024/1157363960518029322/image.png") -}) diff --git a/scripts/migrate-from-old-bridge.js b/scripts/migrate-from-old-bridge.js index aec345d..84daad7 100644 --- a/scripts/migrate-from-old-bridge.js +++ b/scripts/migrate-from-old-bridge.js @@ -20,7 +20,7 @@ const newAT = reg.as_token const oldDB = new sqlite(reg.old_bridge.database) const db = new sqlite("db/ooye.db") -db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration ( +db.exec(`CREATE TABLE IF NOT EXISTS migration ( discord_channel TEXT NOT NULL, migrated INTEGER NOT NULL, PRIMARY KEY("discord_channel") @@ -69,7 +69,7 @@ async function migrateGuild(guild) { const spaceID = await createSpace.syncSpace(guild) let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id) - const migrated = db.prepare("SELECT discord_channel FROM half_shot_migration WHERE migrated = 1").pluck().all() + const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all() oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel)) console.log("Found these rooms which can be migrated:") console.log(oldRooms) @@ -128,7 +128,7 @@ async function migrateGuild(guild) { await createRoom.syncRoom(row.discord_channel) console.log(`-- -- Finished syncing`) - db.prepare("INSERT INTO half_shot_migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id) + db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id) } // Step 5: Call syncSpace to make sure everything is up to date diff --git a/start.js b/start.js index 819281f..49cfadb 100644 --- a/start.js +++ b/start.js @@ -1,7 +1,6 @@ // @ts-check const sqlite = require("better-sqlite3") -const migrate = require("./db/migrate") const HeatSync = require("heatsync") const config = require("./config") @@ -31,7 +30,6 @@ discord.snow.requestHandler.on("requestError", data => { }) ;(async () => { - await migrate.migrate(db) await discord.cloud.connect() console.log("Discord gateway started") diff --git a/test/data.js b/test/data.js index f82b4ae..8cfa717 100644 --- a/test/data.js +++ b/test/data.js @@ -93,28 +93,7 @@ module.exports = { afk_timeout: 300, id: "112760669178241024", icon: "a_f83622e09ead74f0c5c527fe241f8f8c", - emojis: [ - { - version: 0, - roles: [], - require_colons: true, - name: "hippo", - managed: false, - id: "230201364309868544", - available: true, - animated: false - }, - { - version: 0, - roles: [], - require_colons: true, - name: "hipposcope", - managed: false, - id: "393635038903926784", - available: true, - animated: true - } - ], + emojis: [], premium_subscription_count: 14, roles: [], discovery_splash: null, @@ -1154,139 +1133,6 @@ module.exports = { } }, webhook_id: "1109360903096369153" - }, - reply_with_only_embed: { - type: 19, - tts: false, - timestamp: "2023-09-29T20:44:42.606000+00:00", - referenced_message: { - type: 19, - tts: false, - timestamp: "2023-09-29T20:44:42.204000+00:00", - pinned: false, - message_reference: { - message_id: "1157413453921787924", - guild_id: "1150201337112449045", - channel_id: "1150208267285434429" - }, - mentions: [ - { - username: "goat_six", - public_flags: 64, - id: "334539029879980040", - global_name: "GoatSixx", - discriminator: "0", - avatar_decoration_data: null, - avatar: "fd87e077c6ebe4239ce573bae083ed66" - } - ], - mention_roles: [], - mention_everyone: false, - id: "1157417694728044624", - flags: 0, - embeds: [ - { - url: "https://twitter.com/dynastic/status/1707484191963648161", - type: "rich", - timestamp: "2023-09-28T19:55:29.543000+00:00", - reference_id: "1157417694728044624", - footer: { - text: "Twitter", - proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png", - icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png" - }, - fields: [ - { value: "119", name: "Retweets", inline: true }, - { value: "5581", name: "Likes", inline: true } - ], - description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?", - color: 1942002, - author: { - url: "https://twitter.com/dynastic", - proxy_icon_url: "https://images-ext-2.discordapp.net/external/06UZNFT37nepFbzmK2FN4q-9DO_UeSaOaZQICSiMexU/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg", - name: "dynastic (@dynastic)", - icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg" - } - } - ], - edited_timestamp: null, - content: "https://twitter.com/dynastic/status/1707484191963648161", - components: [], - channel_id: "1150208267285434429", - author: { - username: "pokemongod", - public_flags: 0, - id: "66255093481082800", - global_name: "PokemonGod", - discriminator: "0", - avatar_decoration_data: null, - avatar: "0cab06c4256499749cbdd4561c629f84" - }, - attachments: [] - }, - pinned: false, - message_reference: { - message_id: "1157417694728044624", - guild_id: "1150201337112449045", - channel_id: "1150208267285434429" - }, - mentions: [], - mention_roles: [], - mention_everyone: false, - member: { - roles: [ "1153875112832008212" ], - premium_since: null, - pending: false, - nick: null, - mute: false, - joined_at: "2023-09-20T02:07:44.874994+00:00", - flags: 0, - deaf: false, - communication_disabled_until: null, - avatar: null - }, - id: "1157417696414150778", - flags: 0, - embeds: [ - { - url: "https://twitter.com/i/status/1707484191963648161", - type: "rich", - timestamp: "2023-09-28T19:55:29+00:00", - footer: { - text: "Twitter", - proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png", - icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png" - }, - fields: [ - { value: "119", name: "Retweets", inline: true }, - { value: "5581", name: "Likes", inline: true } - ], - description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?", - color: 1942002, - author: { - url: "https://twitter.com/i/user/719631291747078145", - proxy_icon_url: "https://images-ext-1.discordapp.net/external/6LgXrIifZ-MhwPPiwqAomgoy93d932jZiJqLCAf79Fw/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg", - name: "dynastic (@dynastic)", - icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg" - } - } - ], - edited_timestamp: null, - content: "", - components: [], - channel_id: "1150208267285434429", - author: { - username: "Twitter Video Embeds", - public_flags: 65536, - id: "842601826674540574", - global_name: null, - discriminator: "4945", - bot: true, - avatar_decoration_data: null, - avatar: "6ed5bf10f953b22d47893b4655705b30" - }, - attachments: [], - guild_id: "1150201337112449045" } }, message_update: { diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 4724e30..2a56381 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -17,7 +17,7 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES ('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'), ('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');; -INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES +INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL); INSERT INTO message_channel (message_id, channel_id) VALUES diff --git a/test/test.js b/test/test.js index 76fe658..c2ec57b 100644 --- a/test/test.js +++ b/test/test.js @@ -1,12 +1,8 @@ // @ts-check const fs = require("fs") -const {join} = require("path") const sqlite = require("better-sqlite3") -const migrate = require("../db/migrate") const HeatSync = require("heatsync") -const {test} = require("supertape") -const data = require("./data") const config = require("../config") const passthrough = require("../passthrough") @@ -15,15 +11,12 @@ const db = new sqlite(":memory:") const reg = require("../matrix/read-registration") reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded +db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8")) +db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8")) + const sync = new HeatSync({watchFS: false}) -const discord = { - guilds: new Map([ - [data.guild.general.id, data.guild.general] - ]) -} - -Object.assign(passthrough, { discord, config, sync, db }) +Object.assign(passthrough, { config, sync, db }) const orm = sync.require("../db/orm") passthrough.from = orm.from @@ -32,27 +25,17 @@ passthrough.select = orm.select const file = sync.require("../matrix/file") file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) } -;(async () => { - const p = migrate.migrate(db) - test("migrate: migration works", async t => { - await p - t.pass("it did not throw an error") - }) - await p - db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8")) - require("../db/orm.test") - require("../matrix/kstate.test") - require("../matrix/api.test") - require("../matrix/file.test") - require("../matrix/read-registration.test") - require("../matrix/txnid.test") - require("../d2m/converters/message-to-event.test") - require("../d2m/converters/message-to-event.embeds.test") - require("../d2m/converters/edit-to-changes.test") - require("../d2m/converters/thread-to-announcement.test") - require("../d2m/actions/create-room.test") - require("../d2m/converters/user-to-mxid.test") - require("../d2m/actions/register-user.test") - require("../m2d/converters/event-to-message.test") - require("../m2d/converters/utils.test") -})() +require("../db/orm.test") +require("../matrix/kstate.test") +require("../matrix/api.test") +require("../matrix/read-registration.test") +require("../matrix/txnid.test") +require("../d2m/converters/message-to-event.test") +require("../d2m/converters/message-to-event.embeds.test") +require("../d2m/converters/edit-to-changes.test") +require("../d2m/converters/thread-to-announcement.test") +require("../d2m/actions/create-room.test") +require("../d2m/converters/user-to-mxid.test") +require("../d2m/actions/register-user.test") +require("../m2d/converters/event-to-message.test") +require("../m2d/converters/utils.test") diff --git a/types.d.ts b/types.d.ts index e93f379..3b4aacd 100644 --- a/types.d.ts +++ b/types.d.ts @@ -139,8 +139,8 @@ export namespace Event { export type M_Sticker = { body: string url: string - info?: { - mimetype?: string + info: { + mimetype: string w?: number h?: number size?: number @@ -217,7 +217,7 @@ export namespace R { } } -export type Pagination = { +export type Pagination { chunk: T[] next_batch?: string prev_match?: string