Upload files to Discord as streams for speed

m2d/actions/channel-webhook.js

@@ -2,6 +2,7 @@
 
 const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
 const passthrough = require("../../passthrough")
 const {discord, db, select} = passthrough
 
@@ -51,7 +52,7 @@ async function withWebhook(channelID, callback) {
 
 /**
  * @param {string} channelID
- * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]}} data
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
  * @param {string} [threadID]
  */
 async function sendMessageWithWebhook(channelID, data, threadID) {
@@ -64,7 +65,7 @@ async function sendMessageWithWebhook(channelID, data, threadID) {
 /**
  * @param {string} channelID
  * @param {string} messageID
- * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer}[]}} data
+ * @param {DiscordTypes.RESTPatchAPIWebhookWithTokenMessageJSONBody & {files?: {name: string, file: Buffer | Readable}[]}} data
  * @param {string} [threadID]
  */
 async function editMessageWithWebhook(channelID, messageID, data, threadID) {

m2d/actions/send-event.js

@@ -1,11 +1,11 @@
 // @ts-check
 
-const assert = require("assert").strict
-const crypto = require("crypto")
-const {pipeline} = require("stream")
-const {promisify} = require("util")
 const Ty = require("../../types")
 const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
+const assert = require("assert").strict
+const crypto = require("crypto")
+const fetch = require("node-fetch").default
 const passthrough = require("../../passthrough")
 const {sync, discord, db, select} = passthrough
 
@@ -17,13 +17,12 @@ const eventToMessage = sync.require("../converters/event-to-message")
 const api = sync.require("../../matrix/api")
 
 /**
- * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer})[]}} message
+ * @param {DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[], pendingFiles?: ({name: string, url: string} | {name: string, url: string, key: string, iv: string} | {name: string, buffer: Buffer | Readable})[]}} message
- * @returns {Promise<DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]}>}
+ * @returns {Promise<DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]}>}
  */
 async function resolvePendingFiles(message) {
 	if (!message.pendingFiles) return message
 	const files = await Promise.all(message.pendingFiles.map(async p => {
-		let fileBuffer
 		if ("buffer" in p) {
 			return {
 				name: p.name,
@@ -31,21 +30,22 @@ async function resolvePendingFiles(message) {
 			}
 		}
 		if ("key" in p) {
-			// Encrypted
+			// Encrypted file
 			const d = crypto.createDecipheriv("aes-256-ctr", Buffer.from(p.key, "base64url"), Buffer.from(p.iv, "base64url"))
-			fileBuffer = await fetch(p.url).then(res => res.arrayBuffer()).then(x => {
+			// @ts-ignore
-				return Buffer.concat([
+			fetch(p.url).then(res => res.body.pipe(d))
-					d.update(Buffer.from(x)),
+			return {
-					d.final()
+				name: p.name,
-				])
+				file: d
-			})
+			}
 		} else {
-			// Unencrypted
+			// Unencrypted file
-			fileBuffer = await fetch(p.url).then(res => res.arrayBuffer()).then(x => Buffer.from(x))
+			/** @type {Readable} */ // @ts-ignore
-		}
+			const body = await fetch(p.url).then(res => res.body)
-		return {
+			return {
-			name: p.name,
+				name: p.name,
-			file: fileBuffer // TODO: Once SnowTransfer supports ReadableStreams for attachment uploads, pass in those instead of Buffers
+				file: body
+			}
 		}
 	}))
 	const newMessage = {

m2d/converters/event-to-message.js

@@ -2,6 +2,7 @@
 
 const Ty = require("../../types")
 const DiscordTypes = require("discord-api-types/v10")
+const {Readable} = require("stream")
 const chunk = require("chunk-text")
 const TurndownService = require("turndown")
 const assert = require("assert").strict
@@ -9,8 +10,6 @@ const entities = require("entities")
 
 const passthrough = require("../../passthrough")
 const {sync, db, discord, select, from} = passthrough
-/** @type {import("../../matrix/file")} */
-const file = sync.require("../../matrix/file")
 /** @type {import("../converters/utils")} */
 const utils = sync.require("../converters/utils")
 /** @type {import("./emoji-sheet")} */
@@ -245,7 +244,7 @@ async function uploadEndOfMessageSpriteSheet(content, attachments, pendingFiles)
  * @param {{api: import("../../matrix/api")}} di simple-as-nails dependency injection for the matrix API
  */
 async function eventToMessage(event, guild, di) {
-	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer}[]})[]} */
+	/** @type {(DiscordTypes.RESTPostAPIWebhookWithTokenJSONBody & {files?: {name: string, file: Buffer | Readable}[]})[]} */
 	let messages = []
 
 	let displayName = event.sender