From e3b2c844d83b30179fef233b7277cc1955f9bd9c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 7 Oct 2023 20:58:46 +1300 Subject: [PATCH] Fix emoji_id references and add tests --- d2m/converters/emoji-to-key.js | 4 ++-- d2m/converters/emoji-to-key.test.js | 21 +++++++++++++++++ d2m/converters/expression.js | 2 +- d2m/converters/message-to-event.js | 20 +++++----------- d2m/converters/message-to-event.test.js | 12 ++++++++++ test/data.js | 31 +++++++++++++++++++++++++ test/ooye-test-data.sql | 3 ++- test/test.js | 3 ++- 8 files changed, 77 insertions(+), 19 deletions(-) create mode 100644 d2m/converters/emoji-to-key.test.js diff --git a/d2m/converters/emoji-to-key.js b/d2m/converters/emoji-to-key.js index bfd05c23..267664c8 100644 --- a/d2m/converters/emoji-to-key.js +++ b/d2m/converters/emoji-to-key.js @@ -14,7 +14,7 @@ async function emojiToKey(emoji) { let key if (emoji.id) { // Custom emoji - const mxc = select("emoji", "mxc_url", {emoji_id: emoji.id}).pluck().get(emoji.id) + const mxc = select("emoji", "mxc_url", {emoji_id: emoji.id}).pluck().get() if (mxc) { // The custom emoji is registered and we should send it key = mxc @@ -22,7 +22,7 @@ async function emojiToKey(emoji) { // The custom emoji is not registered. We will register it and then add it. assert(emoji.name) // The docs say: "name may be null when custom emoji data is not available, for example, if it was deleted from the guild" const mxc = await file.uploadDiscordFileToMxc(file.emoji(emoji.id, emoji.animated)) - db.prepare("INSERT OR IGNORE INTO emoji (id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, mxc) + db.prepare("INSERT OR IGNORE INTO emoji (emoji_id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, mxc) key = mxc // TODO: what happens if the matrix user also tries adding this reaction? the bridge bot isn't able to use that emoji... } diff --git a/d2m/converters/emoji-to-key.test.js b/d2m/converters/emoji-to-key.test.js new file mode 100644 index 00000000..5af046c0 --- /dev/null +++ b/d2m/converters/emoji-to-key.test.js @@ -0,0 +1,21 @@ +// @ts-check + +const {test} = require("supertape") +const {emojiToKey} = require("./emoji-to-key") +const data = require("../../test/data") +const Ty = require("../../types") + +test("emoji2key: unicode emoji works", async t => { + const result = await emojiToKey({id: null, name: "🐈"}) + t.equal(result, "🐈") +}) + +test("emoji2key: custom emoji works", async t => { + const result = await emojiToKey({id: "230201364309868544", name: "hippo", animated: false}) + t.equal(result, "mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC") +}) + +test("emoji2key: custom animated emoji works", async t => { + const result = await emojiToKey({id: "393635038903926784", name: "hipposcope", animated: true}) + t.equal(result, "mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc") +}) diff --git a/d2m/converters/expression.js b/d2m/converters/expression.js index 87f0d885..1c52c983 100644 --- a/d2m/converters/expression.js +++ b/d2m/converters/expression.js @@ -29,7 +29,7 @@ async function emojisToState(emojis) { }, url } - db.prepare("INSERT OR IGNORE INTO emoji (id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, url) + db.prepare("INSERT OR IGNORE INTO emoji (emoji_id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(emoji.id, emoji.name, +!!emoji.animated, url) }).catch(e => { if (e.data.errcode === "M_TOO_LARGE") { // Very unlikely to happen. Only possible for 3x-series emojis uploaded shortly after animated emojis were introduced, when there was no 256 KB size limit. return diff --git a/d2m/converters/message-to-event.js b/d2m/converters/message-to-event.js index e3b932f2..32415824 100644 --- a/d2m/converters/message-to-event.js +++ b/d2m/converters/message-to-event.js @@ -9,6 +9,8 @@ const passthrough = require("../../passthrough") const {sync, db, discord, select, from} = passthrough /** @type {import("../../matrix/file")} */ const file = sync.require("../../matrix/file") +/** @type {import("./emoji-to-key")} */ +const emojiToKey = sync.require("./emoji-to-key") /** @type {import("./lottie")} */ const lottie = sync.require("./lottie") const reg = require("../../matrix/read-registration") @@ -150,22 +152,12 @@ async function messageToEvent(message, guild, options = {}, di) { // Handling emojis that we don't know about. The emoji has to be present in the DB for it to be picked up in the emoji markdown converter. // So we scan the message ahead of time for all its emojis and ensure they are in the DB. const emojiMatches = [...content.matchAll(/<(a?):([^:>]{2,64}):([0-9]+)>/g)] - const emojiDownloads = [] - for (const match of emojiMatches) { + await Promise.all(emojiMatches.map(match => { const id = match[3] const name = match[2] - const animated = +!!match[1] - const exists = select("emoji", "emoji_id", {emoji_id: id}).pluck().get() - if (!exists) { - // The custom emoji is not registered. We will register it and then add it. - emojiDownloads.push( - file.uploadDiscordFileToMxc(file.emoji(id, animated)).then(mxc => { - db.prepare("INSERT OR IGNORE INTO emoji (id, name, animated, mxc_url) VALUES (?, ?, ?, ?)").run(id, name, animated, mxc) - }) - ) - } - } - await Promise.all(emojiDownloads) + const animated = match[1] + return emojiToKey.emojiToKey({id, name, animated}) // Register the custom emoji if needed + })) let html = markdown.toHTML(content, { discordCallback: getDiscordParseCallbacks(message, true) diff --git a/d2m/converters/message-to-event.test.js b/d2m/converters/message-to-event.test.js index f3699122..715aa5d4 100644 --- a/d2m/converters/message-to-event.test.js +++ b/d2m/converters/message-to-event.test.js @@ -464,6 +464,18 @@ test("message2event: mid-message small bridged emoji", async t => { }]) }) +test("message2event: emoji that hasn't been registered yet", async t => { + const events = await messageToEvent(data.message.not_been_registered_emoji, data.guild.general, {}) + t.deepEqual(events, [{ + $type: "m.room.message", + "m.mentions": {}, + msgtype: "m.text", + body: ":Yeah:", + format: "org.matrix.custom.html", + formatted_body: ':Yeah:' + }]) +}) + test("message2event: emoji triple long name", async t => { const events = await messageToEvent(data.message.emoji_triple_long_name, data.guild.general, {}) t.deepEqual(events, [{ diff --git a/test/data.js b/test/data.js index 9d1cdbca..bcaf9c4a 100644 --- a/test/data.js +++ b/test/data.js @@ -1089,6 +1089,37 @@ module.exports = { flags: 0, components: [] }, + not_been_registered_emoji: { + id: "1126733830494093453", + type: 0, + content: "<:Yeah:1125827250609201255>", + channel_id: "112760669178241024", + author: { + id: "111604486476181504", + username: "kyuugryphon", + avatar: "e4ce31267ca524d19be80e684d4cafa1", + discriminator: "0", + public_flags: 0, + flags: 0, + banner: null, + accent_color: null, + global_name: "KyuuGryphon", + avatar_decoration: null, + display_name: "KyuuGryphon", + banner_color: null + }, + attachments: [], + embeds: [], + mentions: [], + mention_roles: [], + pinned: false, + mention_everyone: false, + tts: false, + timestamp: "2023-07-07T04:37:58.892000+00:00", + edited_timestamp: null, + flags: 0, + components: [] + }, emoji_triple_long_name: { id: "1156394116540805170", type: 0, diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 88dabcd0..4ad318c7 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -71,7 +71,8 @@ INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/avatars/113340068197859328/b48302623a12bc7c59a71328f72ccb39.png?size=1024', 'mxc://cadence.moe/UpAeIqeclhKfeiZNdIWNcXXL'), ('https://cdn.discordapp.com/emojis/230201364309868544.png', 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), ('https://cdn.discordapp.com/emojis/393635038903926784.gif', 'mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc'), -('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'); +('https://cdn.discordapp.com/attachments/176333891320283136/1157854643037163610/Screenshot_20231001_034036.jpg', 'mxc://cadence.moe/zAXdQriaJuLZohDDmacwWWDR'), +('https://cdn.discordapp.com/emojis/1125827250609201255.png', 'mxc://cadence.moe/pgdGTxAyEltccRgZKxdqzHHP'); INSERT INTO emoji (emoji_id, name, animated, mxc_url) VALUES ('230201364309868544', 'hippo', 0, 'mxc://cadence.moe/qWmbXeRspZRLPcjseyLmeyXC'), diff --git a/test/test.js b/test/test.js index 76fe6580..98ecf1c2 100644 --- a/test/test.js +++ b/test/test.js @@ -50,8 +50,9 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../d2m/converters/message-to-event.embeds.test") require("../d2m/converters/edit-to-changes.test") require("../d2m/converters/thread-to-announcement.test") - require("../d2m/actions/create-room.test") require("../d2m/converters/user-to-mxid.test") + require("../d2m/converters/emoji-to-key.test") + require("../d2m/actions/create-room.test") require("../d2m/actions/register-user.test") require("../m2d/converters/event-to-message.test") require("../m2d/converters/utils.test")