bbedward/out-of-your-element
1
0
Fork 0
forked from cadence/out-of-your-element
Code Pull requests Activity

Add database migrations

Browse source
This commit is contained in:
Cadence Ember 2023-10-01 01:24:05 +13:00
parent 7a218f1254
commit 1d99b91ef7
11 changed files with 263 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

12
d2m/actions/register-user.js
View file

@ -11,6 +11,10 @@ const api = sync.require("../../matrix/api")
const file = sync.require("../../matrix/file")
/** @type {import("../converters/user-to-mxid")} */
const userToMxid = sync.require("../converters/user-to-mxid")
/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
let hasher = null
// @ts-ignore
require("xxhash-wasm")().then(h => hasher = h)
/**
* A sim is an account that is being simulated by the bridge to copy events from the other side.
@ -120,7 +124,9 @@ async function memberToStateContent(user, member, guildID) {
}
function hashProfileContent(content) {
return `${content.displayname}\u0000${content.avatar_url}`
const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
return signedHash
}
/**
@ -137,11 +143,11 @@ async function syncUser(user, member, guildID, roomID) {
const mxid = await ensureSimJoined(user, roomID)
const content = await memberToStateContent(user, member, guildID)
const currentHash = hashProfileContent(content)
const existingHash = select("sim_member", "profile_event_content_hash", "WHERE room_id = ? AND mxid = ?").pluck().get(roomID, mxid)
const existingHash = select("sim_member", "hashed_profile_content", "WHERE room_id = ? AND mxid = ?").safeIntegers().pluck().get(roomID, mxid)
// only do the actual sync if the hash has changed since we last looked
if (existingHash !== currentHash) {
await api.sendState(roomID, "m.room.member", mxid, content, mxid)
db.prepare("UPDATE sim_member SET profile_event_content_hash = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
}
return mxid
}

42
db/migrate.js Normal file
View file

@ -0,0 +1,42 @@
// @ts-check
const fs = require("fs")
const {join} = require("path")
async function migrate(db) {
let files = fs.readdirSync(join(__dirname, "migrations"))
files = files.sort()
db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
let progress = db.prepare("SELECT * FROM migration").pluck().get()
if (!progress) {
progress = ""
db.prepare("INSERT INTO migration VALUES ('')").run()
}
let migrationRan = false
for (const filename of files) {
if (progress >= filename) continue
console.log(`Applying database migration ${filename}`)
if (filename.endsWith(".sql")) {
const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8")
db.exec(sql)
} else if (filename.endsWith(".js")) {
await require("./" + join("migrations", filename))(db)
} else {
continue
}
migrationRan = true
db.transaction(() => {
db.prepare("DELETE FROM migration").run()
db.prepare("INSERT INTO migration VALUES (?)").run(filename)
})()
}
if (migrationRan) {
console.log("Database migrations all done.")
}
}
module.exports.migrate = migrate

4
db/migrations/0001-schema.sql
View file

@ -1,3 +1,5 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "sim" (
"discord_id" TEXT NOT NULL,
"sim_name" TEXT NOT NULL UNIQUE,
@ -86,3 +88,5 @@ CREATE TABLE IF NOT EXISTS "reaction" (
"encoded_emoji" TEXT NOT NULL,
PRIMARY KEY ("hashed_event_id")
) WITHOUT ROWID;
COMMIT;

4
db/migrations/0002-optimise-profile-content.sql
View file

@ -1,3 +1,5 @@
BEGIN TRANSACTION;
-- Change hashed_profile_content column affinity to INTEGER
CREATE TABLE "new_sim_member" (
@ -13,4 +15,6 @@ DROP TABLE sim_member;
ALTER TABLE new_sim_member RENAME TO sim_member;
COMMIT;
VACUUM;

10
db/migrations/0002-optimise-profile-content.js → db/migrations/0002-optimise-profile-content.up.js
View file

@ -1,13 +1,13 @@
module.exports = async function(db) {
const hasher = await require("xxhash-wasm")()
const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member").pluck().all()
const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
db.transaction(() => {
for (const s of contents) {
if (!Buffer.isBuffer(s)) s = Buffer.from(s)
const unsignedHash = hasher.h64(eventID)
for (let s of contents) {
let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
const unsignedHash = hasher.h64Raw(b)
const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
stmt.run(s, signedHash)
stmt.run(signedHash, s)
}
})()
}

5
db/orm-utils.d.ts vendored
View file

@ -54,7 +54,7 @@ export type Models = {
sim_member: {
mxid: string
room_id: string
profile_event_content_hash: any
hashed_profile_content: number
}
webhook: {
@ -79,8 +79,9 @@ export type Models = {
export type Prepared<Row> = {
pluck: () => Prepared<Row[keyof Row]>
safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
all: (..._: any[]) => Row[]
get: (..._: any[]) => Row?
get: (..._: any[]) => Row | null
}
export type AllKeys<U> = U extends any ? keyof U : never

6
scripts/migrate-from-old-bridge.js
View file

@ -20,7 +20,7 @@ const newAT = reg.as_token
const oldDB = new sqlite(reg.old_bridge.database)
const db = new sqlite("db/ooye.db")
db.exec(`CREATE TABLE IF NOT EXISTS migration (
db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration (
discord_channel TEXT NOT NULL,
migrated INTEGER NOT NULL,
PRIMARY KEY("discord_channel")
@ -69,7 +69,7 @@ async function migrateGuild(guild) {
const spaceID = await createSpace.syncSpace(guild)
let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id)
const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all()
const migrated = db.prepare("SELECT discord_channel FROM half_shot_migration WHERE migrated = 1").pluck().all()
oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel))
console.log("Found these rooms which can be migrated:")
console.log(oldRooms)
@ -128,7 +128,7 @@ async function migrateGuild(guild) {
await createRoom.syncRoom(row.discord_channel)
console.log(`-- -- Finished syncing`)
db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
db.prepare("INSERT INTO half_shot_migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
}
// Step 5: Call syncSpace to make sure everything is up to date

2
start.js
View file

@ -1,6 +1,7 @@
// @ts-check
const sqlite = require("better-sqlite3")
const migrate = require("./db/migrate")
const HeatSync = require("heatsync")
const config = require("./config")
@ -30,6 +31,7 @@ discord.snow.requestHandler.on("requestError", data => {
})
;(async () => {
await migrate.migrate(db)
await discord.cloud.connect()
console.log("Discord gateway started")

156
test/data.js
View file

@ -93,7 +93,28 @@ module.exports = {
afk_timeout: 300,
id: "112760669178241024",
icon: "a_f83622e09ead74f0c5c527fe241f8f8c",
emojis: [],
emojis: [
{
version: 0,
roles: [],
require_colons: true,
name: "hippo",
managed: false,
id: "230201364309868544",
available: true,
animated: false
},
{
version: 0,
roles: [],
require_colons: true,
name: "hipposcope",
managed: false,
id: "393635038903926784",
available: true,
animated: true
}
],
premium_subscription_count: 14,
roles: [],
discovery_splash: null,
@ -1133,6 +1154,139 @@ module.exports = {
}
},
webhook_id: "1109360903096369153"
},
reply_with_only_embed: {
type: 19,
tts: false,
timestamp: "2023-09-29T20:44:42.606000+00:00",
referenced_message: {
type: 19,
tts: false,
timestamp: "2023-09-29T20:44:42.204000+00:00",
pinned: false,
message_reference: {
message_id: "1157413453921787924",
guild_id: "1150201337112449045",
channel_id: "1150208267285434429"
},
mentions: [
{
username: "goat_six",
public_flags: 64,
id: "334539029879980040",
global_name: "GoatSixx",
discriminator: "0",
avatar_decoration_data: null,
avatar: "fd87e077c6ebe4239ce573bae083ed66"
}
],
mention_roles: [],
mention_everyone: false,
id: "1157417694728044624",
flags: 0,
embeds: [
{
url: "https://twitter.com/dynastic/status/1707484191963648161",
type: "rich",
timestamp: "2023-09-28T19:55:29.543000+00:00",
reference_id: "1157417694728044624",
footer: {
text: "Twitter",
proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png",
icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
},
fields: [
{ value: "119", name: "Retweets", inline: true },
{ value: "5581", name: "Likes", inline: true }
],
description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?",
color: 1942002,
author: {
url: "https://twitter.com/dynastic",
proxy_icon_url: "https://images-ext-2.discordapp.net/external/06UZNFT37nepFbzmK2FN4q-9DO_UeSaOaZQICSiMexU/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg",
name: "dynastic (@dynastic)",
icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg"
}
}
],
edited_timestamp: null,
content: "https://twitter.com/dynastic/status/1707484191963648161",
components: [],
channel_id: "1150208267285434429",
author: {
username: "pokemongod",
public_flags: 0,
id: "66255093481082800",
global_name: "PokemonGod",
discriminator: "0",
avatar_decoration_data: null,
avatar: "0cab06c4256499749cbdd4561c629f84"
},
attachments: []
},
pinned: false,
message_reference: {
message_id: "1157417694728044624",
guild_id: "1150201337112449045",
channel_id: "1150208267285434429"
},
mentions: [],
mention_roles: [],
mention_everyone: false,
member: {
roles: [ "1153875112832008212" ],
premium_since: null,
pending: false,
nick: null,
mute: false,
joined_at: "2023-09-20T02:07:44.874994+00:00",
flags: 0,
deaf: false,
communication_disabled_until: null,
avatar: null
},
id: "1157417696414150778",
flags: 0,
embeds: [
{
url: "https://twitter.com/i/status/1707484191963648161",
type: "rich",
timestamp: "2023-09-28T19:55:29+00:00",
footer: {
text: "Twitter",
proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png",
icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
},
fields: [
{ value: "119", name: "Retweets", inline: true },
{ value: "5581", name: "Likes", inline: true }
],
description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?",
color: 1942002,
author: {
url: "https://twitter.com/i/user/719631291747078145",
proxy_icon_url: "https://images-ext-1.discordapp.net/external/6LgXrIifZ-MhwPPiwqAomgoy93d932jZiJqLCAf79Fw/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg",
name: "dynastic (@dynastic)",
icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg"
}
}
],
edited_timestamp: null,
content: "",
components: [],
channel_id: "1150208267285434429",
author: {
username: "Twitter Video Embeds",
public_flags: 65536,
id: "842601826674540574",
global_name: null,
discriminator: "4945",
bot: true,
avatar_decoration_data: null,
avatar: "6ed5bf10f953b22d47893b4655705b30"
},
attachments: [],
guild_id: "1150201337112449045"
}
},
message_update: {

2
test/ooye-test-data.sql
View file

@ -17,7 +17,7 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES
('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'),
('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');;
INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES
INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL);
INSERT INTO message_channel (message_id, channel_id) VALUES

53
test/test.js
View file

@ -1,8 +1,12 @@
// @ts-check
const fs = require("fs")
const {join} = require("path")
const sqlite = require("better-sqlite3")
const migrate = require("../db/migrate")
const HeatSync = require("heatsync")
const {test} = require("supertape")
const data = require("./data")
const config = require("../config")
const passthrough = require("../passthrough")
@ -11,12 +15,15 @@ const db = new sqlite(":memory:")
const reg = require("../matrix/read-registration")
reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded
db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8"))
db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8"))
const sync = new HeatSync({watchFS: false})
Object.assign(passthrough, { config, sync, db })
const discord = {
guilds: new Map([
[data.guild.general.id, data.guild.general]
])
}
Object.assign(passthrough, { discord, config, sync, db })
const orm = sync.require("../db/orm")
passthrough.from = orm.from
@ -25,17 +32,27 @@ passthrough.select = orm.select
const file = sync.require("../matrix/file")
file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) }
require("../db/orm.test")
require("../matrix/kstate.test")
require("../matrix/api.test")
require("../matrix/read-registration.test")
require("../matrix/txnid.test")
require("../d2m/converters/message-to-event.test")
require("../d2m/converters/message-to-event.embeds.test")
require("../d2m/converters/edit-to-changes.test")
require("../d2m/converters/thread-to-announcement.test")
require("../d2m/actions/create-room.test")
require("../d2m/converters/user-to-mxid.test")
require("../d2m/actions/register-user.test")
require("../m2d/converters/event-to-message.test")
require("../m2d/converters/utils.test")
;(async () => {
const p = migrate.migrate(db)
test("migrate: migration works", async t => {
await p
t.pass("it did not throw an error")
})
await p
db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8"))
require("../db/orm.test")
require("../matrix/kstate.test")
require("../matrix/api.test")
require("../matrix/file.test")
require("../matrix/read-registration.test")
require("../matrix/txnid.test")
require("../d2m/converters/message-to-event.test")
require("../d2m/converters/message-to-event.embeds.test")
require("../d2m/converters/edit-to-changes.test")
require("../d2m/converters/thread-to-announcement.test")
require("../d2m/actions/create-room.test")
require("../d2m/converters/user-to-mxid.test")
require("../d2m/actions/register-user.test")
require("../m2d/converters/event-to-message.test")
require("../m2d/converters/utils.test")
})()