From b7f90db20afd78d324216fe79033bccfbc431438 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Oct 2023 19:27:45 +1300 Subject: [PATCH 1/2] Fix reply preview "undefined" on embed description --- d2m/converters/user-to-mxid.test.js | 2 +- m2d/converters/event-to-message.js | 2 +- m2d/converters/event-to-message.test.js | 80 +++++++++++++++++++++++++ test/ooye-test-data.sql | 13 ++-- 4 files changed, 91 insertions(+), 6 deletions(-) diff --git a/d2m/converters/user-to-mxid.test.js b/d2m/converters/user-to-mxid.test.js index 1b31260..e709473 100644 --- a/d2m/converters/user-to-mxid.test.js +++ b/d2m/converters/user-to-mxid.test.js @@ -17,7 +17,7 @@ test("user2name: works on emojis", t => { }) test("user2name: works on single emoji at the end", t => { - t.equal(userToSimName({username: "Amanda 🎵", discriminator: "2192"}), "amanda") + t.equal(userToSimName({username: "Melody 🎵", discriminator: "2192"}), "melody") }) test("user2name: works on crazy name", t => { diff --git a/m2d/converters/event-to-message.js b/m2d/converters/event-to-message.js index e7bbda5..5f6f3e6 100644 --- a/m2d/converters/event-to-message.js +++ b/m2d/converters/event-to-message.js @@ -352,7 +352,7 @@ async function eventToMessage(event, guild, di) { const contentPreviewChunks = chunk( entities.decodeHTML5Strict( // Remove entities like & " repliedToContent.replace(/.*<\/mx-reply>/, "") // Remove everything before replies, so just use the actual message body - .replace(/
.*?<\/blockquote>/, "") // If the message starts with a blockquote, don't count it and use the message body afterwards + .replace(/^\s*
.*?<\/blockquote>(.....)/s, "$1") // If the message starts with a blockquote, don't count it and use the message body afterwards .replace(/(?:\n|
)+/g, " ") // Should all be on one line .replace(/]*data-mx-spoiler\b[^>]*>.*?<\/span>/g, "[spoiler]") // Good enough method of removing spoiler content. (I don't want to break out the HTML parser unless I have to.) .replace(/<[^>]+>/g, "") // Completely strip all HTML tags and formatting. diff --git a/m2d/converters/event-to-message.test.js b/m2d/converters/event-to-message.test.js index 1a1c3f0..074da1d 100644 --- a/m2d/converters/event-to-message.test.js +++ b/m2d/converters/event-to-message.test.js @@ -813,6 +813,86 @@ test("event2message: should include a reply preview when message ends with a blo ) }) +test("event2message: should include a reply preview when replying to a description-only bot embed", async t => { + t.deepEqual( + await eventToMessage({ + type: "m.room.message", + sender: "@cadence:cadence.moe", + content: { + msgtype: "m.text", + body: "> <@_ooye_amanda:cadence.moe> > It looks like this queue has ended.\n\nso you're saying on matrix side I would have to edit ^this^ to add \"Timed out\" before the blockquote?", + format: "org.matrix.custom.html", + formatted_body: "
In reply to @_ooye_amanda:cadence.moe
It looks like this queue has ended.
so you're saying on matrix side I would have to edit ^this^ to add "Timed out" before the blockquote?", + "m.relates_to": { + "m.in_reply_to": { + event_id: "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ" + } + } + }, + event_id: "$qCOlszCawu5hlnF2a2PGyXeGGvtoNJdXyRAEaTF0waA", + room_id: "!CzvdIdUQXgUjDVKxeU:cadence.moe" + }, data.guild.general, { + api: { + getEvent: mockGetEvent(t, "!CzvdIdUQXgUjDVKxeU:cadence.moe", "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ", { + type: "m.room.message", + room_id: "!edUxjVdzgUvXDUIQCK:cadence.moe", + sender: "@_ooye_amanda:cadence.moe", + content: { + "m.mentions": {}, + msgtype: "m.notice", + body: "> Now Playing: [**LOADING**](https://amanda.moe)\n" + + "> \n" + + "> `[​====[LOADING]=====]`", + format: "org.matrix.custom.html", + formatted_body: '
Now Playing: LOADING

[​====[LOADING]=====]
' + }, + unsigned: { + "m.relations": { + "m.replace": { + type: "m.room.message", + room_id: "!edUxjVdzgUvXDUIQCK:cadence.moe", + sender: "@_ooye_amanda:cadence.moe", + content: { + "m.mentions": {}, + msgtype: "m.notice", + body: "* > It looks like this queue has ended.", + format: "org.matrix.custom.html", + formatted_body: "*
It looks like this queue has ended.
", + "m.new_content": { + "m.mentions": {}, + msgtype: "m.notice", + body: "> It looks like this queue has ended.", + format: "org.matrix.custom.html", + formatted_body: "
It looks like this queue has ended.
" + }, + "m.relates_to": { + rel_type: "m.replace", + event_id: "$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ" + } + }, + event_id: "$nrLF310vALFIXPNk6MEIy0lYiGXi210Ok0DATSaF5jQ", + user_id: "@_ooye_amanda:cadence.moe", + } + }, + user_id: "@_ooye_amanda:cadence.moe", + } + }) + } + }), + { + messagesToDelete: [], + messagesToEdit: [], + messagesToSend: [{ + username: "cadence [they]", + content: "> <:L1:1144820033948762203><:L2:1144820084079087647>https://discord.com/channels/112760669178241024/497161350934560778/1162625810109317170 <@1109360903096369153>:" + + "\n> It looks like this queue has ended." + + `\nso you're saying on matrix side I would have to edit ^this^ to add "Timed out" before the blockquote?`, + avatar_url: "https://matrix.cadence.moe/_matrix/media/r0/download/cadence.moe/azCAhThKTojXSZJRoWwZmhvU" + }] + } + ) +}) + test("event2message: entities are not escaped in main message or reply preview", async t => { // Intended result: Testing? in italics, followed by the sequence "':.`[]&things t.deepEqual( diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index f8039cc..4fba480 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -18,7 +18,8 @@ INSERT INTO sim (user_id, sim_name, localpart, mxid) VALUES ('771520384671416320', 'bojack_horseman', '_ooye_bojack_horseman', '@_ooye_bojack_horseman:cadence.moe'), ('112890272819507200', '.wing.', '_ooye_.wing.', '@_ooye_.wing.:cadence.moe'), ('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'), -('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');; +('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe'), +('1109360903096369153', 'amanda', '_ooye_amanda', '@_ooye_amanda:cadence.moe'); INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES ('@_ooye_bojack_horseman:cadence.moe', '!hYnGGlPHlbujVVfktC:cadence.moe', NULL); @@ -38,7 +39,8 @@ INSERT INTO message_channel (message_id, channel_id) VALUES ('1145688633186193479', '1100319550446252084'), ('1145688633186193480', '1100319550446252084'), ('1145688633186193481', '1100319550446252084'), -('1162005526675193909', '1162005314908999790'); +('1162005526675193909', '1162005314908999790'), +('1162625810109317170', '497161350934560778'); INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES ('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1), @@ -60,7 +62,9 @@ INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part ('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0), ('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0), ('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0), -('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0); +('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0), +('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1), +('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), @@ -91,7 +95,8 @@ INSERT INTO member_cache (room_id, mxid, displayname, avatar_url) VALUES ('!BpMdOUkWWhFxmTrENV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'malformed mxc'), ('!fGgIymcYWOqjbSRUdV:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), ('!BnKuBPCvyfOkhcUjEu:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), -('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'); +('!maggESguZBqGBZtSnr:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'), +('!CzvdIdUQXgUjDVKxeU:cadence.moe', '@cadence:cadence.moe', 'cadence [they]', 'mxc://cadence.moe/azCAhThKTojXSZJRoWwZmhvU'); INSERT INTO "auto_emoji" ("name","emoji_id","guild_id") VALUES ('L1','1144820033948762203','529176156398682115'), From c24752625d28947ee35a769557f020141480bd98 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Sat, 14 Oct 2023 22:08:10 +1300 Subject: [PATCH 2/2] Split part and reaction_part Now, reactions should always end up on the bottom of a message group, instead of sometimes being in the middle. --- d2m/actions/add-reaction.js | 2 +- d2m/actions/edit-message.js | 24 ++++++---- d2m/actions/remove-reaction.js | 2 +- d2m/actions/send-message.js | 10 ++-- d2m/converters/edit-to-changes.js | 30 ++++++------ d2m/converters/edit-to-changes.test.js | 25 ++++------ .../0007-split-part-and-reaction-part.sql | 24 ++++++++++ db/orm-defs.d.ts | 1 + m2d/actions/add-reaction.js | 2 +- m2d/actions/send-event.js | 3 +- scripts/check-migrate.js | 15 ++++++ test/ooye-test-data.sql | 46 +++++++++---------- test/test.js | 8 ++-- 13 files changed, 121 insertions(+), 71 deletions(-) create mode 100644 db/migrations/0007-split-part-and-reaction-part.sql create mode 100644 scripts/check-migrate.js diff --git a/d2m/actions/add-reaction.js b/d2m/actions/add-reaction.js index f962a73..b131f13 100644 --- a/d2m/actions/add-reaction.js +++ b/d2m/actions/add-reaction.js @@ -21,7 +21,7 @@ async function addReaction(data) { const user = data.member?.user assert.ok(user && user.username) - const parentID = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get() // 0 = primary + const parentID = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get() if (!parentID) return // Nothing can be done if the parent message was never bridged. assert.equal(typeof parentID, "string") diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js index a1538b9..2a08526 100644 --- a/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -12,7 +12,7 @@ const api = sync.require("../../matrix/api") * @param {import("discord-api-types/v10").APIGuild} guild */ async function editMessage(message, guild) { - const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promoteEvent, promoteNextEvent} = await editToChanges.editToChanges(message, guild, api) + const {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api) // 1. Replace all the things. for (const {oldID, newContent} of eventsToReplace) { @@ -36,24 +36,30 @@ async function editMessage(message, guild) { } // 3. Consistency: Ensure there is exactly one part = 0 - let eventPart = 1 - if (promoteEvent) { - db.prepare("UPDATE event_message SET part = 0 WHERE event_id = ?").run(promoteEvent) - } else if (promoteNextEvent) { - eventPart = 0 + const sendNewEventParts = new Set() + for (const promotion of promotions) { + if ("eventID" in promotion) { + db.prepare(`UPDATE event_message SET ${promotion.column} = 0 WHERE event_id = ?`).run(promotion.eventID) + } else if ("nextEvent" in promotion) { + sendNewEventParts.add(promotion.column) + } } // 4. Send all the things. + if (eventsToSend.length) { + db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) + } for (const content of eventsToSend) { const eventType = content.$type /** @type {Pick> & { $type?: string }} */ const contentWithoutType = {...content} delete contentWithoutType.$type + delete contentWithoutType.$sender + const part = sendNewEventParts.has("part") && eventsToSend[0] === content ? 0 : 1 + const reactionPart = sendNewEventParts.has("reaction_part") && eventsToSend[eventsToSend.length - 1] === content ? 0 : 1 const eventID = await api.sendEvent(roomID, eventType, contentWithoutType, senderMxid) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, eventPart) // part 1 = supporting; source 1 = discord - - eventPart = 1 + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, content.msgtype || null, message.id, part, reactionPart) // source 1 = discord } } diff --git a/d2m/actions/remove-reaction.js b/d2m/actions/remove-reaction.js index 3047c32..b2dba0f 100644 --- a/d2m/actions/remove-reaction.js +++ b/d2m/actions/remove-reaction.js @@ -20,7 +20,7 @@ const converter = sync.require("../converters/remove-reaction") async function removeSomeReactions(data) { const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get() if (!roomID) return - const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, part: 0}).pluck().get() + const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get() if (!eventIDForMessage) return /** @type {Ty.Pagination>} */ diff --git a/d2m/actions/send-message.js b/d2m/actions/send-message.js index 082cce4..b59fc7f 100644 --- a/d2m/actions/send-message.js +++ b/d2m/actions/send-message.js @@ -33,12 +33,14 @@ async function sendMessage(message, guild) { const events = await messageToEvent.messageToEvent(message, guild, {}, {api}) const eventIDs = [] - let eventPart = 0 // 0 is primary, 1 is supporting if (events.length) { db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id) if (senderMxid) api.sendTyping(roomID, false, senderMxid) } for (const event of events) { + const part = event === events[0] ? 0 : 1 + const reactionPart = event === events[events.length - 1] ? 0 : 1 + const eventType = event.$type if ("$sender" in event) senderMxid = event.$sender /** @type {Pick> & { $type?: string, $sender?: string }} */ @@ -48,12 +50,14 @@ async function sendMessage(message, guild) { const useTimestamp = message["backfill"] ? new Date(message.timestamp).getTime() : undefined const eventID = await api.sendEvent(roomID, eventType, eventWithoutType, senderMxid, useTimestamp) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, eventPart) // source 1 = discord + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 1)").run(eventID, eventType, event.msgtype || null, message.id, part, reactionPart) // source 1 = discord // The primary event is part = 0 and has the most important and distinct information. It is used to provide reply previews, be pinned, and possibly future uses. // The first event is chosen to be the primary part because it is usually the message text content and is more likely to be distinct. // For example, "Reply to 'this meme made me think of you'" is more useful than "Replied to image". - eventPart = 1 + + // The last event gets reaction_part = 0. Reactions are managed there because reactions are supposed to appear at the bottom. + eventIDs.push(eventID) } diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js index dc42708..f86921c 100644 --- a/d2m/converters/edit-to-changes.js +++ b/d2m/converters/edit-to-changes.js @@ -26,7 +26,7 @@ async function editToChanges(message, guild, api) { /** @type {string?} Null if we don't have a sender in the room, which will happen if it's a webhook's message. The bridge bot will do the edit instead. */ const senderMxid = from("sim").join("sim_member", "mxid").where({user_id: message.author.id, room_id: roomID}).pluck("mxid").get() || null - const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part"], {message_id: message.id}).all() + const oldEventRows = select("event_message", ["event_id", "event_type", "event_subtype", "part", "reaction_part"], {message_id: message.id}).all() // Figure out what we will be replacing them with @@ -83,17 +83,21 @@ async function editToChanges(message, guild, api) { // Anything remaining in oldEventRows is present in the old version only and should be redacted. eventsToRedact = oldEventRows - // If events are being deleted, we might be deleting the part = 0. But we want to have a part = 0 at all times. In this case we choose an existing event to promote. - let promoteEvent = null, promoteNextEvent = false - if (eventsToRedact.some(e => e.part === 0)) { - if (eventsToReplace.length) { - // We can choose an existing event to promote. Bigger order is better. - const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text") - eventsToReplace.sort((a, b) => order(b) - order(a)) - promoteEvent = eventsToReplace[0].old.event_id - } else { - // Everything is being deleted. Whatever gets sent in their place will be the new part = 0. - promoteNextEvent = true + // We want to maintain exactly one part = 0 and one reaction_part = 0 database row at all times. + /** @type {({column: string, eventID: string} | {column: string, nextEvent: true})[]} */ + const promotions = [] + for (const column of ["part", "reaction_part"]) { + // If no events with part = 0 exist (or will exist), we need to do some management. + if (!eventsToReplace.some(e => e.old[column] === 0)) { + if (eventsToReplace.length) { + // We can choose an existing event to promote. Bigger order is better. + const order = e => 2*+(e.event_type === "m.room.message") + 1*+(e.event_subtype === "m.text") + eventsToReplace.sort((a, b) => order(b) - order(a)) + promotions.push({column, eventID: eventsToReplace[0].old.event_id}) + } else { + // No existing events to promote, but new events are being sent. Whatever gets sent will be the next part = 0. + promotions.push({column, nextEvent: true}) + } } } @@ -117,7 +121,7 @@ async function editToChanges(message, guild, api) { eventsToRedact = eventsToRedact.map(e => e.event_id) eventsToReplace = eventsToReplace.map(e => ({oldID: e.old.event_id, newContent: makeReplacementEventContent(e.old.event_id, e.newFallbackContent, e.newInnerContent)})) - return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promoteEvent, promoteNextEvent} + return {roomID, eventsToReplace, eventsToRedact, eventsToSend, senderMxid, promotions} } /** diff --git a/d2m/converters/edit-to-changes.test.js b/d2m/converters/edit-to-changes.test.js index 8c67f6a..7c29787 100644 --- a/d2m/converters/edit-to-changes.test.js +++ b/d2m/converters/edit-to-changes.test.js @@ -4,7 +4,7 @@ const data = require("../../test/data") const Ty = require("../../types") test("edit2changes: edit by webhook", async t => { - const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {}) + const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.edit_by_webhook, data.guild.general, {}) t.deepEqual(eventsToRedact, []) t.deepEqual(eventsToSend, []) t.deepEqual(eventsToReplace, [{ @@ -27,12 +27,11 @@ test("edit2changes: edit by webhook", async t => { } }]) t.equal(senderMxid, null) - t.equal(promoteEvent, null) - t.equal(promoteNextEvent, false) + t.deepEqual(promotions, []) }) test("edit2changes: bot response", async t => { - const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.bot_response, data.guild.general, { + const {senderMxid, eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.bot_response, data.guild.general, { async getJoinedMembers(roomID) { t.equal(roomID, "!hYnGGlPHlbujVVfktC:cadence.moe") return new Promise(resolve => { @@ -84,21 +83,19 @@ test("edit2changes: bot response", async t => { } }]) t.equal(senderMxid, "@_ooye_bojack_horseman:cadence.moe") - t.equal(promoteEvent, null) - t.equal(promoteNextEvent, false) + t.deepEqual(promotions, []) }) test("edit2changes: remove caption from image", async t => { - const {eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.removed_caption_from_image, data.guild.general, {}) + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.removed_caption_from_image, data.guild.general, {}) t.deepEqual(eventsToRedact, ["$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA"]) t.deepEqual(eventsToSend, []) t.deepEqual(eventsToReplace, []) - t.equal(promoteEvent, "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI") - t.equal(promoteNextEvent, false) + t.deepEqual(promotions, [{column: "part", eventID: "$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI"}]) }) test("edit2changes: change file type", async t => { - const {eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.changed_file_type, data.guild.general, {}) + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.changed_file_type, data.guild.general, {}) t.deepEqual(eventsToRedact, ["$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ"]) t.deepEqual(eventsToSend, [{ $type: "m.room.message", @@ -109,12 +106,11 @@ test("edit2changes: change file type", async t => { msgtype: "m.text" }]) t.deepEqual(eventsToReplace, []) - t.equal(promoteEvent, null) - t.equal(promoteNextEvent, true) + t.deepEqual(promotions, [{column: "part", nextEvent: true}, {column: "reaction_part", nextEvent: true}]) }) test("edit2changes: add caption back to that image", async t => { - const {eventsToRedact, eventsToReplace, eventsToSend, promoteEvent, promoteNextEvent} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {}) + const {eventsToRedact, eventsToReplace, eventsToSend, promotions} = await editToChanges(data.message_update.added_caption_to_image, data.guild.general, {}) t.deepEqual(eventsToRedact, []) t.deepEqual(eventsToSend, [{ $type: "m.room.message", @@ -123,8 +119,7 @@ test("edit2changes: add caption back to that image", async t => { "m.mentions": {} }]) t.deepEqual(eventsToReplace, []) - t.equal(promoteEvent, null) - t.equal(promoteNextEvent, false) + t.deepEqual(promotions, []) }) test("edit2changes: stickers and attachments are not changed, only the content can be edited", async t => { diff --git a/db/migrations/0007-split-part-and-reaction-part.sql b/db/migrations/0007-split-part-and-reaction-part.sql new file mode 100644 index 0000000..4aad6c1 --- /dev/null +++ b/db/migrations/0007-split-part-and-reaction-part.sql @@ -0,0 +1,24 @@ +BEGIN TRANSACTION; + +-- Add column reaction_part to event_message, copying the existing value from part + +CREATE TABLE "new_event_message" ( + "event_id" TEXT NOT NULL, + "event_type" TEXT, + "event_subtype" TEXT, + "message_id" TEXT NOT NULL, + "part" INTEGER NOT NULL, + "reaction_part" INTEGER NOT NULL, + "source" INTEGER NOT NULL, + PRIMARY KEY("message_id","event_id") +) WITHOUT ROWID; + +INSERT INTO new_event_message SELECT event_id, event_type, event_subtype, message_id, part, part, source FROM event_message; + +DROP TABLE event_message; + +ALTER TABLE new_event_message RENAME TO event_message; + +COMMIT; + +VACUUM; diff --git a/db/orm-defs.d.ts b/db/orm-defs.d.ts index 0714e0b..ec8d498 100644 --- a/db/orm-defs.d.ts +++ b/db/orm-defs.d.ts @@ -14,6 +14,7 @@ export type Models = { event_type: string | null event_subtype: string | null part: number + reaction_part: number source: number } diff --git a/m2d/actions/add-reaction.js b/m2d/actions/add-reaction.js index e6a94d9..cfd471b 100644 --- a/m2d/actions/add-reaction.js +++ b/m2d/actions/add-reaction.js @@ -16,7 +16,7 @@ const emoji = sync.require("../converters/emoji") async function addReaction(event) { const channelID = select("channel_room", "channel_id", {room_id: event.room_id}).pluck().get() if (!channelID) return // We just assume the bridge has already been created - const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id, part: 0}).pluck().get() // 0 = primary + const messageID = select("event_message", "message_id", {event_id: event.content["m.relates_to"].event_id}, "ORDER BY reaction_part").pluck().get() if (!messageID) return // Nothing can be done if the parent message was never bridged. const key = event.content["m.relates_to"].key diff --git a/m2d/actions/send-event.js b/m2d/actions/send-event.js index c1e3ba3..13b5650 100644 --- a/m2d/actions/send-event.js +++ b/m2d/actions/send-event.js @@ -98,9 +98,10 @@ async function sendEvent(event) { } for (const message of messagesToSend) { + const reactionPart = messagesToEdit.length === 0 && message === messagesToSend[messagesToSend.length - 1] ? 0 : 1 const messageResponse = await channelWebhook.sendMessageWithWebhook(channelID, message, threadID) db.prepare("REPLACE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(messageResponse.id, threadID || channelID) - db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES (?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart) // source 0 = matrix + db.prepare("INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES (?, ?, ?, ?, ?, ?, 0)").run(event.event_id, event.type, event.content["msgtype"] || null, messageResponse.id, eventPart, reactionPart) // source 0 = matrix eventPart = 1 messageResponses.push(messageResponse) diff --git a/scripts/check-migrate.js b/scripts/check-migrate.js new file mode 100644 index 0000000..308ea87 --- /dev/null +++ b/scripts/check-migrate.js @@ -0,0 +1,15 @@ +// @ts-check + +// Trigger the database migration flow and exit after committing. +// You can use this to run migrations locally and check the result using sqlitebrowser. + +const sqlite = require("better-sqlite3") + +const config = require("../config") +const passthrough = require("../passthrough") +const db = new sqlite("db/ooye.db") +const migrate = require("../db/migrate") + +Object.assign(passthrough, {config, db }) + +migrate.migrate(db) diff --git a/test/ooye-test-data.sql b/test/ooye-test-data.sql index 4fba480..0627d08 100644 --- a/test/ooye-test-data.sql +++ b/test/ooye-test-data.sql @@ -42,29 +42,29 @@ INSERT INTO message_channel (message_id, channel_id) VALUES ('1162005526675193909', '1162005314908999790'), ('1162625810109317170', '497161350934560778'); -INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, source) VALUES -('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 1), -('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', 0, 0), -('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', 0, 1), -('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', 0, 1), -('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1), -('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 1), -('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', 0, 1), -('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', 0, 1), -('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', 1, 1), -('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 1), -('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 1), -('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1), -('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 0), -('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0), -('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 1), -('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0), -('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0), -('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0), -('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0), -('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0), -('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1), -('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 1); +INSERT INTO event_message (event_id, event_type, event_subtype, message_id, part, reaction_part, source) VALUES +('$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg', 'm.room.message', 'm.text', '1126786462646550579', 0, 0, 1), +('$Ij3qo7NxMA4VPexlAiIx2CB9JbsiGhJeyt-2OvkAUe4', 'm.room.message', 'm.text', '1128118177155526666', 0, 0, 0), +('$zXSlyI78DQqQwwfPUSzZ1b-nXzbUrCDljJgnGDdoI10', 'm.room.message', 'm.text', '1141619794500649020', 0, 0, 1), +('$fdD9OZ55xg3EAsfvLZza5tMhtjUO91Wg3Otuo96TplY', 'm.room.message', 'm.text', '1141206225632112650', 0, 0, 1), +('$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA', 'm.room.message', 'm.text', '1141501302736695316', 0, 1, 1), +('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCI', 'm.room.message', 'm.image', '1141501302736695316', 1, 0, 1), +('$51f4yqHinwnSbPEQ9dCgoyy4qiIJSX0QYYVUnvwyTCJ', 'm.room.message', 'm.image', '1141501302736695317', 0, 0, 1), +('$vgTKOR5ZTYNMKaS7XvgEIDaOWZtVCEyzLLi5Pc5Gz4M', 'm.room.message', 'm.text', '1128084851279536279', 0, 1, 1), +('$YUJFa5j0ZJe7PUvD2DykRt9g51RoadUEYmuJLdSEbJ0', 'm.room.message', 'm.image', '1128084851279536279', 1, 0, 1), +('$oLyUTyZ_7e_SUzGNWZKz880ll9amLZvXGbArJCKai2Q', 'm.room.message', 'm.text', '1128084748338741392', 0, 0, 1), +('$FchUVylsOfmmbj-VwEs5Z9kY49_dt2zd0vWfylzy5Yo', 'm.room.message', 'm.text', '1143121514925928541', 0, 0, 1), +('$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4', 'm.room.message', 'm.text', '1106366167788044450', 0, 1, 1), +('$Ijf1MFCD39ktrNHxrA-i2aKoRWNYdAV2ZXYQeiZIgEU', 'm.room.message', 'm.image', '1106366167788044450', 1, 1, 0), +('$f9cjKiacXI9qPF_nUAckzbiKnJEi0LM399kOkhdd8f8', 'm.sticker', NULL, '1106366167788044450', 1, 0, 0), +('$Fxy8SMoJuTduwReVkHZ1uHif9EuvNx36Hg79cltiA04', 'm.room.message', 'm.text', '1144865310588014633', 0, 0, 1), +('$v_Gtr-bzv9IVlSLBO5DstzwmiDd-GSFaNfHX66IupV8', 'm.room.message', 'm.text', '1144874214311067708', 0, 0, 0), +('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSs', 'm.room.message', 'm.text', '1145688633186193479', 0, 0, 0), +('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193480', 0, 0, 0), +('$7LIdiJCEqjcWUrpzWzS8TELOlFfBEe4ytgS7zn2lbSt', 'm.room.message', 'm.text', '1145688633186193481', 1, 0, 0), +('$nUM-ABBF8KdnvrhXwLlYAE9dgDl_tskOvvcNIBrtsVo', 'm.room.message', 'm.text', '1162005526675193909', 0, 0, 0), +('$0wEdIP8fhTq-P68xwo_gyUw-Zv0KA2aS2tfhdFSrLZc', 'm.room.message', 'm.text', '1162625810109317170', 1, 1, 1), +('$zJFjTvNn1w_YqpR4o4ISKUFisNRgZcu1KSMI_LADPVQ', 'm.room.message', 'm.notice', '1162625810109317170', 1, 0, 1); INSERT INTO file (discord_url, mxc_url) VALUES ('https://cdn.discordapp.com/attachments/497161332244742154/1124628646431297546/image.png', 'mxc://cadence.moe/qXoZktDqNtEGuOCZEADAMvhM'), diff --git a/test/test.js b/test/test.js index 5e15a14..90e3f5a 100644 --- a/test/test.js +++ b/test/test.js @@ -50,16 +50,16 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not require("../matrix/file.test") require("../matrix/read-registration.test") require("../matrix/txnid.test") + require("../d2m/actions/create-room.test") + require("../d2m/actions/register-user.test") + require("../d2m/converters/edit-to-changes.test") + require("../d2m/converters/emoji-to-key.test") require("../d2m/converters/message-to-event.test") require("../d2m/converters/message-to-event.embeds.test") - require("../d2m/converters/edit-to-changes.test") require("../d2m/converters/pins-to-list.test") require("../d2m/converters/remove-reaction.test") require("../d2m/converters/thread-to-announcement.test") require("../d2m/converters/user-to-mxid.test") - require("../d2m/converters/emoji-to-key.test") - require("../d2m/actions/create-room.test") - require("../d2m/actions/register-user.test") require("../m2d/converters/event-to-message.test") require("../m2d/converters/utils.test") })()