Add database migrations
This commit is contained in:
parent
7a218f1254
commit
1d99b91ef7
11 changed files with 263 additions and 33 deletions
|
@ -11,6 +11,10 @@ const api = sync.require("../../matrix/api")
|
|||
const file = sync.require("../../matrix/file")
|
||||
/** @type {import("../converters/user-to-mxid")} */
|
||||
const userToMxid = sync.require("../converters/user-to-mxid")
|
||||
/** @type {import("xxhash-wasm").XXHashAPI} */ // @ts-ignore
|
||||
let hasher = null
|
||||
// @ts-ignore
|
||||
require("xxhash-wasm")().then(h => hasher = h)
|
||||
|
||||
/**
|
||||
* A sim is an account that is being simulated by the bridge to copy events from the other side.
|
||||
|
@ -120,7 +124,9 @@ async function memberToStateContent(user, member, guildID) {
|
|||
}
|
||||
|
||||
function hashProfileContent(content) {
|
||||
return `${content.displayname}\u0000${content.avatar_url}`
|
||||
const unsignedHash = hasher.h64(`${content.displayname}\u0000${content.avatar_url}`)
|
||||
const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
|
||||
return signedHash
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -137,11 +143,11 @@ async function syncUser(user, member, guildID, roomID) {
|
|||
const mxid = await ensureSimJoined(user, roomID)
|
||||
const content = await memberToStateContent(user, member, guildID)
|
||||
const currentHash = hashProfileContent(content)
|
||||
const existingHash = select("sim_member", "profile_event_content_hash", "WHERE room_id = ? AND mxid = ?").pluck().get(roomID, mxid)
|
||||
const existingHash = select("sim_member", "hashed_profile_content", "WHERE room_id = ? AND mxid = ?").safeIntegers().pluck().get(roomID, mxid)
|
||||
// only do the actual sync if the hash has changed since we last looked
|
||||
if (existingHash !== currentHash) {
|
||||
await api.sendState(roomID, "m.room.member", mxid, content, mxid)
|
||||
db.prepare("UPDATE sim_member SET profile_event_content_hash = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
|
||||
db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
|
||||
}
|
||||
return mxid
|
||||
}
|
||||
|
|
42
db/migrate.js
Normal file
42
db/migrate.js
Normal file
|
@ -0,0 +1,42 @@
|
|||
// @ts-check
|
||||
|
||||
const fs = require("fs")
|
||||
const {join} = require("path")
|
||||
|
||||
async function migrate(db) {
|
||||
let files = fs.readdirSync(join(__dirname, "migrations"))
|
||||
files = files.sort()
|
||||
db.prepare("CREATE TABLE IF NOT EXISTS migration (filename TEXT NOT NULL)").run()
|
||||
let progress = db.prepare("SELECT * FROM migration").pluck().get()
|
||||
if (!progress) {
|
||||
progress = ""
|
||||
db.prepare("INSERT INTO migration VALUES ('')").run()
|
||||
}
|
||||
|
||||
let migrationRan = false
|
||||
|
||||
for (const filename of files) {
|
||||
if (progress >= filename) continue
|
||||
console.log(`Applying database migration ${filename}`)
|
||||
if (filename.endsWith(".sql")) {
|
||||
const sql = fs.readFileSync(join(__dirname, "migrations", filename), "utf8")
|
||||
db.exec(sql)
|
||||
} else if (filename.endsWith(".js")) {
|
||||
await require("./" + join("migrations", filename))(db)
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
migrationRan = true
|
||||
db.transaction(() => {
|
||||
db.prepare("DELETE FROM migration").run()
|
||||
db.prepare("INSERT INTO migration VALUES (?)").run(filename)
|
||||
})()
|
||||
}
|
||||
|
||||
if (migrationRan) {
|
||||
console.log("Database migrations all done.")
|
||||
}
|
||||
}
|
||||
|
||||
module.exports.migrate = migrate
|
|
@ -1,3 +1,5 @@
|
|||
BEGIN TRANSACTION;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "sim" (
|
||||
"discord_id" TEXT NOT NULL,
|
||||
"sim_name" TEXT NOT NULL UNIQUE,
|
||||
|
@ -86,3 +88,5 @@ CREATE TABLE IF NOT EXISTS "reaction" (
|
|||
"encoded_emoji" TEXT NOT NULL,
|
||||
PRIMARY KEY ("hashed_event_id")
|
||||
) WITHOUT ROWID;
|
||||
|
||||
COMMIT;
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
BEGIN TRANSACTION;
|
||||
|
||||
-- Change hashed_profile_content column affinity to INTEGER
|
||||
|
||||
CREATE TABLE "new_sim_member" (
|
||||
|
@ -13,4 +15,6 @@ DROP TABLE sim_member;
|
|||
|
||||
ALTER TABLE new_sim_member RENAME TO sim_member;
|
||||
|
||||
COMMIT;
|
||||
|
||||
VACUUM;
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
module.exports = async function(db) {
|
||||
const hasher = await require("xxhash-wasm")()
|
||||
const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member").pluck().all()
|
||||
const contents = db.prepare("SELECT distinct hashed_profile_content FROM sim_member WHERE hashed_profile_content IS NOT NULL").pluck().all()
|
||||
const stmt = db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE hashed_profile_content = ?")
|
||||
db.transaction(() => {
|
||||
for (const s of contents) {
|
||||
if (!Buffer.isBuffer(s)) s = Buffer.from(s)
|
||||
const unsignedHash = hasher.h64(eventID)
|
||||
for (let s of contents) {
|
||||
let b = Buffer.isBuffer(s) ? Uint8Array.from(s) : Uint8Array.from(Buffer.from(s))
|
||||
const unsignedHash = hasher.h64Raw(b)
|
||||
const signedHash = unsignedHash - 0x8000000000000000n // shifting down to signed 64-bit range
|
||||
stmt.run(s, signedHash)
|
||||
stmt.run(signedHash, s)
|
||||
}
|
||||
})()
|
||||
}
|
5
db/orm-utils.d.ts
vendored
5
db/orm-utils.d.ts
vendored
|
@ -54,7 +54,7 @@ export type Models = {
|
|||
sim_member: {
|
||||
mxid: string
|
||||
room_id: string
|
||||
profile_event_content_hash: any
|
||||
hashed_profile_content: number
|
||||
}
|
||||
|
||||
webhook: {
|
||||
|
@ -79,8 +79,9 @@ export type Models = {
|
|||
|
||||
export type Prepared<Row> = {
|
||||
pluck: () => Prepared<Row[keyof Row]>
|
||||
safeIntegers: () => Prepared<{[K in keyof Row]: Row[K] extends number ? BigInt : Row[K]}>
|
||||
all: (..._: any[]) => Row[]
|
||||
get: (..._: any[]) => Row?
|
||||
get: (..._: any[]) => Row | null
|
||||
}
|
||||
|
||||
export type AllKeys<U> = U extends any ? keyof U : never
|
||||
|
|
|
@ -20,7 +20,7 @@ const newAT = reg.as_token
|
|||
const oldDB = new sqlite(reg.old_bridge.database)
|
||||
const db = new sqlite("db/ooye.db")
|
||||
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS migration (
|
||||
db.exec(`CREATE TABLE IF NOT EXISTS half_shot_migration (
|
||||
discord_channel TEXT NOT NULL,
|
||||
migrated INTEGER NOT NULL,
|
||||
PRIMARY KEY("discord_channel")
|
||||
|
@ -69,7 +69,7 @@ async function migrateGuild(guild) {
|
|||
const spaceID = await createSpace.syncSpace(guild)
|
||||
|
||||
let oldRooms = oldDB.prepare("SELECT matrix_id, discord_guild, discord_channel FROM room_entries INNER JOIN remote_room_data ON remote_id = room_id WHERE discord_guild = ?").all(guild.id)
|
||||
const migrated = db.prepare("SELECT discord_channel FROM migration WHERE migrated = 1").pluck().all()
|
||||
const migrated = db.prepare("SELECT discord_channel FROM half_shot_migration WHERE migrated = 1").pluck().all()
|
||||
oldRooms = oldRooms.filter(row => discord.channels.has(row.discord_channel) && !migrated.includes(row.discord_channel))
|
||||
console.log("Found these rooms which can be migrated:")
|
||||
console.log(oldRooms)
|
||||
|
@ -128,7 +128,7 @@ async function migrateGuild(guild) {
|
|||
await createRoom.syncRoom(row.discord_channel)
|
||||
console.log(`-- -- Finished syncing`)
|
||||
|
||||
db.prepare("INSERT INTO migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
|
||||
db.prepare("INSERT INTO half_shot_migration (discord_channel, migrated) VALUES (?, 1)").run(channel.id)
|
||||
}
|
||||
|
||||
// Step 5: Call syncSpace to make sure everything is up to date
|
||||
|
|
2
start.js
2
start.js
|
@ -1,6 +1,7 @@
|
|||
// @ts-check
|
||||
|
||||
const sqlite = require("better-sqlite3")
|
||||
const migrate = require("./db/migrate")
|
||||
const HeatSync = require("heatsync")
|
||||
|
||||
const config = require("./config")
|
||||
|
@ -30,6 +31,7 @@ discord.snow.requestHandler.on("requestError", data => {
|
|||
})
|
||||
|
||||
;(async () => {
|
||||
await migrate.migrate(db)
|
||||
await discord.cloud.connect()
|
||||
console.log("Discord gateway started")
|
||||
|
||||
|
|
156
test/data.js
156
test/data.js
|
@ -93,7 +93,28 @@ module.exports = {
|
|||
afk_timeout: 300,
|
||||
id: "112760669178241024",
|
||||
icon: "a_f83622e09ead74f0c5c527fe241f8f8c",
|
||||
emojis: [],
|
||||
emojis: [
|
||||
{
|
||||
version: 0,
|
||||
roles: [],
|
||||
require_colons: true,
|
||||
name: "hippo",
|
||||
managed: false,
|
||||
id: "230201364309868544",
|
||||
available: true,
|
||||
animated: false
|
||||
},
|
||||
{
|
||||
version: 0,
|
||||
roles: [],
|
||||
require_colons: true,
|
||||
name: "hipposcope",
|
||||
managed: false,
|
||||
id: "393635038903926784",
|
||||
available: true,
|
||||
animated: true
|
||||
}
|
||||
],
|
||||
premium_subscription_count: 14,
|
||||
roles: [],
|
||||
discovery_splash: null,
|
||||
|
@ -1133,6 +1154,139 @@ module.exports = {
|
|||
}
|
||||
},
|
||||
webhook_id: "1109360903096369153"
|
||||
},
|
||||
reply_with_only_embed: {
|
||||
type: 19,
|
||||
tts: false,
|
||||
timestamp: "2023-09-29T20:44:42.606000+00:00",
|
||||
referenced_message: {
|
||||
type: 19,
|
||||
tts: false,
|
||||
timestamp: "2023-09-29T20:44:42.204000+00:00",
|
||||
pinned: false,
|
||||
message_reference: {
|
||||
message_id: "1157413453921787924",
|
||||
guild_id: "1150201337112449045",
|
||||
channel_id: "1150208267285434429"
|
||||
},
|
||||
mentions: [
|
||||
{
|
||||
username: "goat_six",
|
||||
public_flags: 64,
|
||||
id: "334539029879980040",
|
||||
global_name: "GoatSixx",
|
||||
discriminator: "0",
|
||||
avatar_decoration_data: null,
|
||||
avatar: "fd87e077c6ebe4239ce573bae083ed66"
|
||||
}
|
||||
],
|
||||
mention_roles: [],
|
||||
mention_everyone: false,
|
||||
id: "1157417694728044624",
|
||||
flags: 0,
|
||||
embeds: [
|
||||
{
|
||||
url: "https://twitter.com/dynastic/status/1707484191963648161",
|
||||
type: "rich",
|
||||
timestamp: "2023-09-28T19:55:29.543000+00:00",
|
||||
reference_id: "1157417694728044624",
|
||||
footer: {
|
||||
text: "Twitter",
|
||||
proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png",
|
||||
icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
|
||||
},
|
||||
fields: [
|
||||
{ value: "119", name: "Retweets", inline: true },
|
||||
{ value: "5581", name: "Likes", inline: true }
|
||||
],
|
||||
description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?",
|
||||
color: 1942002,
|
||||
author: {
|
||||
url: "https://twitter.com/dynastic",
|
||||
proxy_icon_url: "https://images-ext-2.discordapp.net/external/06UZNFT37nepFbzmK2FN4q-9DO_UeSaOaZQICSiMexU/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg",
|
||||
name: "dynastic (@dynastic)",
|
||||
icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_400x400.jpg"
|
||||
}
|
||||
}
|
||||
],
|
||||
edited_timestamp: null,
|
||||
content: "https://twitter.com/dynastic/status/1707484191963648161",
|
||||
components: [],
|
||||
channel_id: "1150208267285434429",
|
||||
author: {
|
||||
username: "pokemongod",
|
||||
public_flags: 0,
|
||||
id: "66255093481082800",
|
||||
global_name: "PokemonGod",
|
||||
discriminator: "0",
|
||||
avatar_decoration_data: null,
|
||||
avatar: "0cab06c4256499749cbdd4561c629f84"
|
||||
},
|
||||
attachments: []
|
||||
},
|
||||
pinned: false,
|
||||
message_reference: {
|
||||
message_id: "1157417694728044624",
|
||||
guild_id: "1150201337112449045",
|
||||
channel_id: "1150208267285434429"
|
||||
},
|
||||
mentions: [],
|
||||
mention_roles: [],
|
||||
mention_everyone: false,
|
||||
member: {
|
||||
roles: [ "1153875112832008212" ],
|
||||
premium_since: null,
|
||||
pending: false,
|
||||
nick: null,
|
||||
mute: false,
|
||||
joined_at: "2023-09-20T02:07:44.874994+00:00",
|
||||
flags: 0,
|
||||
deaf: false,
|
||||
communication_disabled_until: null,
|
||||
avatar: null
|
||||
},
|
||||
id: "1157417696414150778",
|
||||
flags: 0,
|
||||
embeds: [
|
||||
{
|
||||
url: "https://twitter.com/i/status/1707484191963648161",
|
||||
type: "rich",
|
||||
timestamp: "2023-09-28T19:55:29+00:00",
|
||||
footer: {
|
||||
text: "Twitter",
|
||||
proxy_icon_url: "https://images-ext-1.discordapp.net/external/bXJWV2Y_F3XSra_kEqIYXAAsI3m1meckfLhYuWzxIfI/https/abs.twimg.com/icons/apple-touch-icon-192x192.png",
|
||||
icon_url: "https://abs.twimg.com/icons/apple-touch-icon-192x192.png"
|
||||
},
|
||||
fields: [
|
||||
{ value: "119", name: "Retweets", inline: true },
|
||||
{ value: "5581", name: "Likes", inline: true }
|
||||
],
|
||||
description: "does anyone know where to find that one video of the really mysterious yam-like object being held up to a bunch of random objects, like clocks, and they have unexplained impossible reactions to it?",
|
||||
color: 1942002,
|
||||
author: {
|
||||
url: "https://twitter.com/i/user/719631291747078145",
|
||||
proxy_icon_url: "https://images-ext-1.discordapp.net/external/6LgXrIifZ-MhwPPiwqAomgoy93d932jZiJqLCAf79Fw/https/pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg",
|
||||
name: "dynastic (@dynastic)",
|
||||
icon_url: "https://pbs.twimg.com/profile_images/1682417899162730499/q7dQMwLq_normal.jpg"
|
||||
}
|
||||
}
|
||||
],
|
||||
edited_timestamp: null,
|
||||
content: "",
|
||||
components: [],
|
||||
channel_id: "1150208267285434429",
|
||||
author: {
|
||||
username: "Twitter Video Embeds",
|
||||
public_flags: 65536,
|
||||
id: "842601826674540574",
|
||||
global_name: null,
|
||||
discriminator: "4945",
|
||||
bot: true,
|
||||
avatar_decoration_data: null,
|
||||
avatar: "6ed5bf10f953b22d47893b4655705b30"
|
||||
},
|
||||
attachments: [],
|
||||
guild_id: "1150201337112449045"
|
||||
}
|
||||
},
|
||||
message_update: {
|
||||
|
|
|
@ -17,7 +17,7 @@ INSERT INTO sim (discord_id, sim_name, localpart, mxid) VALUES
|
|||
('114147806469554185', 'extremity', '_ooye_extremity', '@_ooye_extremity:cadence.moe'),
|
||||
('111604486476181504', 'kyuugryphon', '_ooye_kyuugryphon', '@_ooye_kyuugryphon:cadence.moe');;
|
||||
|
||||
INSERT INTO sim_member (mxid, room_id, profile_event_content_hash) VALUES
|
||||
INSERT INTO sim_member (mxid, room_id, hashed_profile_content) VALUES
|
||||
('@_ooye_bojack_horseman:cadence.moe', '!uCtjHhfGlYbVnPVlkG:cadence.moe', NULL);
|
||||
|
||||
INSERT INTO message_channel (message_id, channel_id) VALUES
|
||||
|
|
53
test/test.js
53
test/test.js
|
@ -1,8 +1,12 @@
|
|||
// @ts-check
|
||||
|
||||
const fs = require("fs")
|
||||
const {join} = require("path")
|
||||
const sqlite = require("better-sqlite3")
|
||||
const migrate = require("../db/migrate")
|
||||
const HeatSync = require("heatsync")
|
||||
const {test} = require("supertape")
|
||||
const data = require("./data")
|
||||
|
||||
const config = require("../config")
|
||||
const passthrough = require("../passthrough")
|
||||
|
@ -11,12 +15,15 @@ const db = new sqlite(":memory:")
|
|||
const reg = require("../matrix/read-registration")
|
||||
reg.ooye.server_origin = "https://matrix.cadence.moe" // so that tests will pass even when hard-coded
|
||||
|
||||
db.exec(fs.readFileSync("db/ooye-schema.sql", "utf8"))
|
||||
db.exec(fs.readFileSync("db/ooye-test-data.sql", "utf8"))
|
||||
|
||||
const sync = new HeatSync({watchFS: false})
|
||||
|
||||
Object.assign(passthrough, { config, sync, db })
|
||||
const discord = {
|
||||
guilds: new Map([
|
||||
[data.guild.general.id, data.guild.general]
|
||||
])
|
||||
}
|
||||
|
||||
Object.assign(passthrough, { discord, config, sync, db })
|
||||
|
||||
const orm = sync.require("../db/orm")
|
||||
passthrough.from = orm.from
|
||||
|
@ -25,17 +32,27 @@ passthrough.select = orm.select
|
|||
const file = sync.require("../matrix/file")
|
||||
file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not allowed to upload files during testing.\nURL: ${url}`) }
|
||||
|
||||
require("../db/orm.test")
|
||||
require("../matrix/kstate.test")
|
||||
require("../matrix/api.test")
|
||||
require("../matrix/read-registration.test")
|
||||
require("../matrix/txnid.test")
|
||||
require("../d2m/converters/message-to-event.test")
|
||||
require("../d2m/converters/message-to-event.embeds.test")
|
||||
require("../d2m/converters/edit-to-changes.test")
|
||||
require("../d2m/converters/thread-to-announcement.test")
|
||||
require("../d2m/actions/create-room.test")
|
||||
require("../d2m/converters/user-to-mxid.test")
|
||||
require("../d2m/actions/register-user.test")
|
||||
require("../m2d/converters/event-to-message.test")
|
||||
require("../m2d/converters/utils.test")
|
||||
;(async () => {
|
||||
const p = migrate.migrate(db)
|
||||
test("migrate: migration works", async t => {
|
||||
await p
|
||||
t.pass("it did not throw an error")
|
||||
})
|
||||
await p
|
||||
db.exec(fs.readFileSync(join(__dirname, "ooye-test-data.sql"), "utf8"))
|
||||
require("../db/orm.test")
|
||||
require("../matrix/kstate.test")
|
||||
require("../matrix/api.test")
|
||||
require("../matrix/file.test")
|
||||
require("../matrix/read-registration.test")
|
||||
require("../matrix/txnid.test")
|
||||
require("../d2m/converters/message-to-event.test")
|
||||
require("../d2m/converters/message-to-event.embeds.test")
|
||||
require("../d2m/converters/edit-to-changes.test")
|
||||
require("../d2m/converters/thread-to-announcement.test")
|
||||
require("../d2m/actions/create-room.test")
|
||||
require("../d2m/converters/user-to-mxid.test")
|
||||
require("../d2m/actions/register-user.test")
|
||||
require("../m2d/converters/event-to-message.test")
|
||||
require("../m2d/converters/utils.test")
|
||||
})()
|
||||
|
|
Loading…
Reference in a new issue