Compare commits

...

12 commits
v3.4 ... main

22 changed files with 291 additions and 88 deletions

1
.gitignore vendored
View file

@ -4,6 +4,7 @@ registration.yaml
ooye.db*
events.db*
backfill.db*
custom-webroot
# Automatically generated
node_modules

View file

@ -89,7 +89,7 @@ Whether you read those or not, I'm more than happy to help you 1-on-1 with codin
# Dependency justification
Total transitive production dependencies: 137
Total transitive production dependencies: 134
### <font size="+2">🦕</font>
@ -119,8 +119,8 @@ Total transitive production dependencies: 137
* (0) entities: Looks fine. No dependencies.
* (0) get-relative-path: Looks fine. No dependencies.
* (1) heatsync: Module hot-reloader that I trust.
* (1) js-yaml: Will be removed in the future after registration.yaml is converted to JSON.
* (0) lru-cache: For holding unused nonce in memory and letting them be overwritten later if never used.
* (0) mime-type: File extension to mime type mapping that's already pulled in by stream-mime-type.
* (0) prettier-bytes: It does what I want and has no dependencies.
* (0) snowtransfer: Discord API library with bring-your-own-caching that I trust.
* (0) try-to-catch: Not strictly necessary, but it's already pulled in by supertape, so I may as well.

2
package-lock.json generated
View file

@ -33,6 +33,7 @@
"heatsync": "^2.7.2",
"htmx.org": "^2.0.4",
"lru-cache": "^11.0.2",
"mime-types": "^2.1.35",
"prettier-bytes": "^1.0.4",
"sharp": "^0.34.5",
"snowtransfer": "^0.17.1",
@ -2073,6 +2074,7 @@
"version": "2.1.35",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"license": "MIT",
"dependencies": {
"mime-db": "1.52.0"
},

View file

@ -42,6 +42,7 @@
"heatsync": "^2.7.2",
"htmx.org": "^2.0.4",
"lru-cache": "^11.0.2",
"mime-types": "^2.1.35",
"prettier-bytes": "^1.0.4",
"sharp": "^0.34.5",
"snowtransfer": "^0.17.1",

View file

@ -38,6 +38,7 @@ For more information about features, [see the user guide.](https://gitdab.com/ca
* This bridge is not designed for puppetting.
* Direct Messaging is not supported until I figure out a good way of doing it.
* Encrypted messages are not supported. Decryption is often unreliable on Matrix, and your messages end up in plaintext on Discord anyway, so there's not much advantage.
## Get started!

65
scripts/estimate-size.js Normal file
View file

@ -0,0 +1,65 @@
// @ts-check
const pb = require("prettier-bytes")
const sqlite = require("better-sqlite3")
const HeatSync = require("heatsync")
const {reg} = require("../src/matrix/read-registration")
const passthrough = require("../src/passthrough")
const sync = new HeatSync({watchFS: false})
Object.assign(passthrough, {reg, sync})
const DiscordClient = require("../src/d2m/discord-client")
const discord = new DiscordClient(reg.ooye.discord_token, "no")
passthrough.discord = discord
const db = new sqlite("ooye.db")
passthrough.db = db
const api = require("../src/matrix/api")
const {room: roomID} = require("minimist")(process.argv.slice(2), {string: ["room"]})
if (!roomID) {
console.error("Usage: ./scripts/estimate-size.js --room=<!room id here>")
process.exit(1)
}
const {channel_id, guild_id} = db.prepare("SELECT channel_id, guild_id FROM channel_room WHERE room_id = ?").get(roomID)
const max = 1000
;(async () => {
let total = 0
let size = 0
let from
while (total < max) {
const events = await api.getEvents(roomID, "b", {limit: 1000, from})
total += events.chunk.length
from = events.end
console.log(`Fetched ${total} events so far`)
for (const e of events.chunk) {
if (e.content?.info?.size) {
size += e.content.info.size
}
}
if (events.chunk.length === 0 || !events.end) break
}
console.log(`Total size of uploads: ${pb(size)}`)
const searchResults = await discord.snow.requestHandler.request(`/guilds/${guild_id}/messages/search`, {
channel_id,
offset: "0",
limit: "1"
}, "get", "json")
const totalAllTime = searchResults.total_results
const fractionCounted = total / totalAllTime
console.log(`That counts for ${(fractionCounted*100).toFixed(2)}% of the history on Discord (${totalAllTime.toLocaleString()} messages)`)
console.log(`The size of uploads for the whole history would be approx: ${pb(Math.floor(size/total*totalAllTime))}`)
})()

View file

@ -1,6 +1,7 @@
#!/usr/bin/env node
// @ts-check
const Ty = require("../src/types")
const assert = require("assert").strict
const fs = require("fs")
const sqlite = require("better-sqlite3")
@ -17,22 +18,6 @@ const {SnowTransfer} = require("snowtransfer")
const DiscordTypes = require("discord-api-types/v10")
const {createApp, defineEventHandler, toNodeListener} = require("h3")
// Move database file if it's still in the old location
if (fs.existsSync("db")) {
if (fs.existsSync("db/ooye.db")) {
fs.renameSync("db/ooye.db", "ooye.db")
}
const files = fs.readdirSync("db")
if (files.length) {
console.error("The db folder is deprecated and must be removed. Your ooye.db database file has already been moved to the root of the repo. You must manually move or delete the remaining files:")
for (const file of files) {
console.error(file)
}
process.exit(1)
}
fs.rmSync("db", {recursive: true})
}
const passthrough = require("../src/passthrough")
const db = new sqlite("ooye.db")
const migrate = require("../src/db/migrate")
@ -102,7 +87,7 @@ function defineEchoHandler() {
type: "input",
name: "server_name",
message: "Homeserver name",
validate: serverName => !!serverName.match(/[a-z][a-z.]+[a-z]/)
validate: serverName => !!serverName.match(/[a-z0-9][.a-z0-9-]+[a-z]/)
})
console.log("What is the URL of your homeserver?")
@ -301,8 +286,8 @@ function defineEchoHandler() {
console.log()
// Done with user prompts, reg is now guaranteed to be valid
const mreq = require("../src/matrix/mreq")
const api = require("../src/matrix/api")
const file = require("../src/matrix/file")
const DiscordClient = require("../src/d2m/discord-client")
const discord = new DiscordClient(reg.ooye.discord_token, "no")
passthrough.discord = discord
@ -359,7 +344,13 @@ function defineEchoHandler() {
await api.register(reg.sender_localpart)
// upload initial images...
const avatarUrl = await file.uploadDiscordFileToMxc("https://cadence.moe/friends/out_of_your_element.png")
const avatarBuffer = await fs.promises.readFile(join(__dirname, "..", "docs", "img", "icon.png"), null)
/** @type {Ty.R.FileUploaded} */
const root = await mreq.mreq("POST", "/media/v3/upload", avatarBuffer, {
headers: {"Content-Type": "image/png"}
})
const avatarUrl = root.content_uri
assert(avatarUrl)
console.log("✅ Matrix appservice login works...")
@ -368,8 +359,7 @@ function defineEchoHandler() {
console.log("✅ Emojis are ready...")
// set profile data on discord...
const avatarImageBuffer = await fetch("https://cadence.moe/friends/out_of_your_element.png").then(res => res.arrayBuffer())
await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + Buffer.from(avatarImageBuffer).toString("base64")})
await discord.snow.user.updateSelf({avatar: "data:image/png;base64," + avatarBuffer.toString("base64")})
console.log("✅ Discord profile updated...")
// set profile data on homeserver...

View file

@ -153,7 +153,7 @@ async function editToChanges(message, guild, api) {
const embedsEnabled = select("guild_space", "url_preview", {guild_id: guild?.id}).pluck().get() ?? 1
if (messageReallyOld) {
eventsToSend = [] // Only allow edits to change and delete, but not send new.
} else if ((messageQuiteOld || !embedsEnabled) && !message.author.bot) {
} else if ((messageQuiteOld || !embedsEnabled) && !message.author?.bot) {
eventsToSend = eventsToSend.filter(e => e.msgtype !== "m.notice") // Only send events that aren't embeds.
}

View file

@ -20,10 +20,10 @@ const SPECIAL_USER_MAPPINGS = new Map([
function downcaseUsername(user) {
// First, try to convert the username to the set of allowed characters
let downcased = user.username.toLowerCase()
// spaces to underscores...
.replace(/ /g, "_")
// spaces and slashes to underscores...
.replace(/[ /]/g, "_")
// remove disallowed characters...
.replace(/[^a-z0-9._=/-]*/g, "")
.replace(/[^a-z0-9._=-]*/g, "")
// remove leading and trailing dashes and underscores...
.replace(/(?:^[_-]*|[_-]*$)/g, "")
// If requested, also make the Discord user ID part of the username

View file

@ -21,8 +21,12 @@ test("user2name: works on single emoji at the end", t => {
t.equal(userToSimName({username: "Melody 🎵", discriminator: "2192"}), "melody")
})
test("user2name: works on crazy name", t => {
t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7//")
test("user2name: works on really weird name", t => {
t.equal(userToSimName({username: "*** D3 &W (89) _7//-", discriminator: "0001"}), "d3_w_89__7")
})
test("user2name: treats slashes", t => {
t.equal(userToSimName({username: "Evil Lillith (she/her)", discriminator: "5892"}), "evil_lillith_she_her")
})
test("user2name: adds discriminator if name is unavailable (old tag format)", t => {

View file

@ -0,0 +1,5 @@
BEGIN TRANSACTION;
DELETE FROM sim WHERE sim_name like '%/%';
COMMIT;

View file

@ -0,0 +1,40 @@
// @ts-check
const {Readable} = require("stream")
const {ReadableStream} = require("stream/web")
const {sync} = require("../../passthrough")
const sharp = require("sharp")
/** @type {import("../../matrix/api")} */
const api = sync.require("../../matrix/api")
/** @type {import("../../matrix/mreq")} */
const mreq = sync.require("../../matrix/mreq")
const streamMimeType = require("stream-mime-type")
const WIDTH = 160
const HEIGHT = 160
/**
* Downloads the sticker from the web and converts to webp data.
* @param {string} mxc a single mxc:// URL
* @returns {Promise<ReadableStream>} sticker webp data, or undefined if the downloaded sticker is not valid
*/
async function getAndResizeSticker(mxc) {
const res = await api.getMedia(mxc)
if (res.status !== 200) {
const root = await res.json()
throw new mreq.MatrixServerError(root, {mxc})
}
const streamIn = Readable.fromWeb(res.body)
const { stream, mime } = await streamMimeType.getMimeType(streamIn)
const animated = ["image/gif", "image/webp"].includes(mime)
const transformer = sharp({animated: animated})
.resize(WIDTH, HEIGHT, {fit: "inside", background: {r: 0, g: 0, b: 0, alpha: 0}})
.webp()
stream.pipe(transformer)
return Readable.toWeb(transformer)
}
module.exports.getAndResizeSticker = getAndResizeSticker

View file

@ -631,23 +631,10 @@ async function eventToMessage(event, guild, channel, di) {
}
if (event.type === "m.sticker") {
content = ""
let filename = event.content.body
if (event.type === "m.sticker") {
let mimetype
if (event.content.info?.mimetype?.includes("/")) {
mimetype = event.content.info.mimetype
} else {
const res = await di.api.getMedia(event.content.url, {method: "HEAD"})
if (res.status === 200) {
mimetype = res.headers.get("content-type")
}
if (!mimetype) throw new Error(`Server error ${res.status} or missing content-type while detecting sticker mimetype`)
}
filename += "." + mimetype.split("/")[1]
}
attachments.push({id: "0", filename})
pendingFiles.push({name: filename, mxc: event.content.url})
const withoutMxc = mxUtils.makeMxcPublic(event.content.url)
assert(withoutMxc)
const url = `${reg.ooye.bridge_origin}/download/sticker/${withoutMxc}/_.webp`
content = `[${event.content.body || "\u2800"}](${url})`
} else if (event.type === "org.matrix.msc3381.poll.start") {
const pollContent = event.content["org.matrix.msc3381.poll.start"] // just for convenience

View file

@ -136,7 +136,7 @@ async function getEventForTimestamp(roomID, ts) {
*/
async function getEvents(roomID, dir, pagination = {}, filter) {
filter = filter && JSON.stringify(filter)
/** @type {Ty.Pagination<Ty.Event.Outer<any>>} */
/** @type {Ty.MessagesPagination<Ty.Event.Outer<any>>} */
const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/messages`, null, {...pagination, dir, filter}))
return root
}

View file

@ -11,7 +11,7 @@ const registrationFilePath = path.join(process.cwd(), "registration.yaml")
function checkRegistration(reg) {
reg["ooye"].invite = reg.ooye.invite.filter(mxid => mxid.endsWith(`:${reg.ooye.server_name}`)) // one day I will understand why typescript disagrees with dot notation on this line
assert(reg.ooye?.max_file_size)
assert(reg.ooye?.namespace_prefix)
assert(reg.ooye?.namespace_prefix != null)
assert(reg.ooye?.server_name)
assert(reg.sender_localpart?.startsWith(reg.ooye.namespace_prefix), "appservice's localpart must be in the namespace it controls")
assert(reg.ooye?.server_origin.match(/^https?:\/\//), "server origin must start with http or https")

8
src/types.d.ts vendored
View file

@ -498,7 +498,13 @@ export type Membership = "invite" | "knock" | "join" | "leave" | "ban"
export type Pagination<T> = {
chunk: T[]
next_batch?: string
prev_match?: string
prev_batch?: string
}
export type MessagesPagination<T> = {
chunk: T[]
start: string
end?: string
}
export type HierarchyPagination<T> = {

View file

@ -31,7 +31,15 @@ function addGlobals(obj) {
*/
function render(event, filename, locals) {
const path = join(__dirname, "pug", filename)
return renderPath(event, path, locals)
}
/**
* @param {import("h3").H3Event} event
* @param {string} path
* @param {Record<string, any>} locals
*/
function renderPath(event, path, locals) {
function compile() {
try {
const template = compileFile(path, {pretty})
@ -89,4 +97,5 @@ function createRoute(router, url, filename) {
module.exports.addGlobals = addGlobals
module.exports.render = render
module.exports.renderPath = renderPath
module.exports.createRoute = createRoute

View file

@ -41,6 +41,7 @@ block body
= ` Set up self-service`
.s-prose
block bridge-info
h2 What is this?
p #[a(href="https://gitdab.com/cadence/out-of-your-element") Out Of Your Element] is a bridge between the Discord and Matrix chat apps. It lets people on both platforms chat with each other without needing to get everyone on the same app.
p Just chat like usual, and the bridge will forward messages back and forth between the two platforms, so everyone sees the whole conversation.
@ -48,6 +49,7 @@ block body
p It's really easy to set up, even if you only have Discord. Just add the bot to your server, and it'll make everything available on Matrix automatically.
if locked
block locked-info
h2 This is a private instance
p Anybody can run their own instance of the Out Of Your Element software. The person running this instance has made it private, so you can't add it to your server just yet. If you know who's in charge of #{reg.ooye.server_name}, ask them for the password.

View file

@ -1,4 +1,10 @@
mixin guild(guild)
mixin guild-menuitem(guild)
- let bridgedRoomCount = from("channel_room").selectUnsafe("count(*) as count").where({guild_id: guild.id}).and("AND thread_parent IS NULL").get().count
li(role="menuitem")
a.s-topbar--item.s-user-card.d-flex.p4(href=rel(`/guild?guild_id=${guild.id}`) class={"bg-purple-200": bridgedRoomCount === 0, "h:bg-purple-300": bridgedRoomCount === 0})
+guild(guild, bridgedRoomCount)
mixin guild(guild, bridgedRoomCount)
span.s-avatar.s-avatar__32.s-user-card--avatar
if guild.icon
img.s-avatar--image(src=`https://cdn.discordapp.com/icons/${guild.id}/${guild.icon}.png?size=32` alt="")
@ -6,8 +12,12 @@ mixin guild(guild)
.s-avatar--letter.bg-silver-400.bar-md(aria-hidden="true")= guild.name[0]
.s-user-card--info.ai-start
strong= guild.name
if bridgedRoomCount != null
ul.s-user-card--awards
li #{discord.guildChannelMap.get(guild.id).filter(c => [0, 5, 15, 16].includes(discord.channels.get(c).type)).length} channels
if bridgedRoomCount
li #{bridgedRoomCount} bridged rooms
else
li.fc-purple Not yet linked
mixin define-theme(name, h, s, l)
style.
@ -58,6 +68,8 @@ html(lang="en")
title Out Of Your Element
<meta name="viewport" content="width=device-width, initial-scale=1">
link(rel="stylesheet" type="text/css" href=rel("/static/stacks.min.css"))
//- Please use responsibly!!!!!
link(rel="stylesheet" type="text/css" href=rel("/custom.css"))
<link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 80%22><text y=%22.83em%22 font-size=%2283%22>💬</text></svg>">
meta(name="htmx-config" content='{"requestClass":"is-loading"}')
style.
@ -79,6 +91,14 @@ html(lang="en")
.s-btn__dropdown:has(+ :popover-open) {
background-color: var(--theme-topbar-item-background-hover, var(--black-200)) !important;
}
@media (prefers-color-scheme: dark) {
body.theme-system .s-popover {
--_po-bg: var(--black-100);
--_po-bc: var(--bc-light);
--_po-bs: var(--bs-lg);
--_po-arrow-fc: var(--black-100);
}
}
+define-themed-button("matrix", "black")
body.themed.theme-system
header.s-topbar
@ -114,9 +134,7 @@ html(lang="en")
.s-popover--content.overflow-y-auto.overflow-x-hidden
ul.s-menu(role="menu")
each guild in [...managed].map(id => discord.guilds.get(id)).filter(g => g).sort((a, b) => a.name.toLowerCase() < b.name.toLowerCase() ? -1 : 1)
li(role="menuitem")
a.s-topbar--item.s-user-card.d-flex.p4(href=rel(`/guild?guild_id=${guild.id}`))
+guild(guild)
+guild-menuitem(guild)
//- Body
.mx-auto.w100.wmx9.py24.px8.fs-body1#content
block body

View file

@ -16,6 +16,9 @@ const emojiSheet = sync.require("../../m2d/actions/emoji-sheet")
/** @type {import("../../m2d/converters/emoji-sheet")} */
const emojiSheetConverter = sync.require("../../m2d/converters/emoji-sheet")
/** @type {import("../../m2d/actions/sticker")} */
const sticker = sync.require("../../m2d/actions/sticker")
const schema = {
params: z.object({
server_name: z.string(),
@ -23,6 +26,10 @@ const schema = {
}),
sheet: z.object({
e: z.array(z.string()).or(z.string())
}),
sticker: z.object({
server_name: z.string().regex(/^[^/]+$/),
media_id: z.string().regex(/^[A-Za-z0-9_-]+$/)
})
}
@ -90,3 +97,14 @@ as.router.get(`/download/sheet`, defineEventHandler(async event => {
setResponseHeader(event, "Content-Type", "image/png")
return buffer
}))
as.router.get(`/download/sticker/:server_name/:media_id/_.webp`, defineEventHandler(async event => {
const {server_name, media_id} = await getValidatedRouterParams(event, schema.sticker.parse)
/** remember that this has no mxc:// protocol in the string */
const mxc = server_name + "/" + media_id
verifyMediaHash(mxc)
const stream = await sticker.getAndResizeSticker(`mxc://${mxc}`)
setResponseHeader(event, "Content-Type", "image/webp")
return stream
}))

View file

@ -115,7 +115,7 @@ function getChannelRoomsLinks(guild, rooms, roles) {
let removedWrongTypeChannels = dUtils.filterTo(unlinkedChannels, c => c && [0, 5].includes(c.type))
let removedPrivateChannels = dUtils.filterTo(unlinkedChannels, c => {
const permissions = dUtils.getPermissions(guild.id, roles, guild.roles, botID, c["permission_overwrites"])
return dUtils.hasPermission(permissions, DiscordTypes.PermissionFlagsBits.ViewChannel)
return dUtils.hasSomePermissions(permissions, ["Administrator", "ViewChannel"])
})
unlinkedChannels.sort((a, b) => getPosition(a, discord.channels) - getPosition(b, discord.channels))

View file

@ -4,13 +4,14 @@ const assert = require("assert")
const fs = require("fs")
const {join} = require("path")
const h3 = require("h3")
const {defineEventHandler, defaultContentType, getRequestHeader, setResponseHeader, handleCacheHeaders} = h3
const mimeTypes = require("mime-types")
const {defineEventHandler, defaultContentType, getRequestHeader, setResponseHeader, handleCacheHeaders, serveStatic} = h3
const icons = require("@stackoverflow/stacks-icons")
const DiscordTypes = require("discord-api-types/v10")
const dUtils = require("../discord/utils")
const reg = require("../matrix/read-registration")
const {sync, discord, as, select} = require("../passthrough")
const {sync, discord, as, select, from} = require("../passthrough")
/** @type {import("./pug-sync")} */
const pugSync = sync.require("./pug-sync")
/** @type {import("../matrix/utils")} */
@ -19,21 +20,7 @@ const {id} = require("../../addbot")
// Pug
pugSync.addGlobals({id, h3, discord, select, DiscordTypes, dUtils, mUtils, icons, reg: reg.reg})
pugSync.createRoute(as.router, "/", "home.pug")
pugSync.createRoute(as.router, "/ok", "ok.pug")
// Routes
sync.require("./routes/download-matrix")
sync.require("./routes/download-discord")
sync.require("./routes/guild-settings")
sync.require("./routes/guild")
sync.require("./routes/info")
sync.require("./routes/link")
sync.require("./routes/log-in-with-matrix")
sync.require("./routes/oauth")
sync.require("./routes/password")
pugSync.addGlobals({id, h3, discord, select, from, DiscordTypes, dUtils, mUtils, icons, reg: reg.reg})
// Files
@ -65,12 +52,79 @@ as.router.get("/static/htmx.js", defineEventHandler({
}
}))
as.router.get("/icon.png", defineEventHandler(event => {
handleCacheHeaders(event, {maxAge: 86400})
return fs.promises.readFile(join(__dirname, "../../docs/img/icon.png"))
}))
as.router.get("/download/file/poll-star-avatar.png", defineEventHandler(event => {
handleCacheHeaders(event, {maxAge: 86400})
return fs.promises.readFile(join(__dirname, "../../docs/img/poll-star-avatar.png"))
}))
// Custom files
const publicDir = "custom-webroot"
/**
* @param {h3.H3Event} event
* @param {boolean} fallthrough
*/
function tryStatic(event, fallthrough) {
return serveStatic(event, {
indexNames: ["/index.html", "/index.pug"],
fallthrough,
getMeta: async id => {
// Check
const stats = await fs.promises.stat(join(publicDir, id)).catch(() => {});
if (!stats || !stats.isFile()) {
return
}
// Pug
if (id.match(/\.pug$/)) {
defaultContentType(event, "text/html; charset=utf-8")
return {}
}
// Everything else
else {
const mime = mimeTypes.lookup(id)
if (typeof mime === "string") defaultContentType(event, mime)
return {
size: stats.size
}
}
},
getContents: id => {
if (id.match(/\.pug$/)) {
const path = join(publicDir, id)
return pugSync.renderPath(event, path, {})
} else {
return fs.promises.readFile(join(publicDir, id))
}
}
})
}
as.router.get("/**", defineEventHandler(event => {
return tryStatic(event, false)
}))
as.router.get("/", defineEventHandler(async event => {
return (await tryStatic(event, true)) || pugSync.render(event, "home.pug", {})
}))
as.router.get("/icon.png", defineEventHandler(async event => {
const s = await tryStatic(event, true)
if (s) return s
handleCacheHeaders(event, {maxAge: 86400})
return fs.promises.readFile(join(__dirname, "../../docs/img/icon.png"))
}))
// Routes
pugSync.createRoute(as.router, "/ok", "ok.pug")
sync.require("./routes/download-matrix")
sync.require("./routes/download-discord")
sync.require("./routes/guild-settings")
sync.require("./routes/guild")
sync.require("./routes/info")
sync.require("./routes/link")
sync.require("./routes/log-in-with-matrix")
sync.require("./routes/oauth")
sync.require("./routes/password")