Compare commits
2 commits
408070f329
...
a2787f7b0b
| Author | SHA1 | Date | |
|---|---|---|---|
| a2787f7b0b | |||
| 1338e6ba88 |
5 changed files with 78 additions and 13 deletions
18
package-lock.json
generated
18
package-lock.json
generated
|
|
@ -225,9 +225,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@cloudrac3r/discord-markdown": {
|
"node_modules/@cloudrac3r/discord-markdown": {
|
||||||
"version": "2.6.7",
|
"version": "2.6.8",
|
||||||
"resolved": "https://registry.npmjs.org/@cloudrac3r/discord-markdown/-/discord-markdown-2.6.7.tgz",
|
"resolved": "https://registry.npmjs.org/@cloudrac3r/discord-markdown/-/discord-markdown-2.6.8.tgz",
|
||||||
"integrity": "sha512-bWLmBYWaNEDcQfZHDz4jaAxLKA9161ruEnHo3ms6kfRw8uYku/Uz7U1xTmQ2dQF/q1PiuBvM9I37pLiotlQj8A==",
|
"integrity": "sha512-ZrSimHqmLqXR+W3U1n6ge6poAjmQaMzXyWrTkT36znrgKhfuQAYxLBtKTf7m+cmr3VlaDVM2P+iPdSeTeaM0qg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"simple-markdown": "^0.7.3"
|
"simple-markdown": "^0.7.3"
|
||||||
|
|
@ -1210,9 +1210,9 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"node_modules/@types/node": {
|
"node_modules/@types/node": {
|
||||||
"version": "22.19.0",
|
"version": "22.19.1",
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.1.tgz",
|
||||||
"integrity": "sha512-xpr/lmLPQEj+TUnHmR+Ab91/glhJvsqcjB+yY0Ix9GO70H6Lb4FHH5GeqdOE5btAx7eIMwuHkp4H2MSkLcqWbA==",
|
"integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==",
|
||||||
"dev": true,
|
"dev": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
|
@ -1713,9 +1713,9 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/discord-api-types": {
|
"node_modules/discord-api-types": {
|
||||||
"version": "0.38.32",
|
"version": "0.38.33",
|
||||||
"resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.32.tgz",
|
"resolved": "https://registry.npmjs.org/discord-api-types/-/discord-api-types-0.38.33.tgz",
|
||||||
"integrity": "sha512-UhIqkFuUVwBzejLPPWF18qixYPucMf718RnGh1NxZYNS7czXUmcUsWWkzWR7lRWj5pjfj4LwrnN9McvpfLvGqQ==",
|
"integrity": "sha512-oau1V7OzrNX8yNi+DfQpoLZCNCv7cTFmvPKwHfMrA/tewsO6iQKrMTzA7pa3iBSj0fED6NlklJ/1B/cC1kI08Q==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"workspaces": [
|
"workspaces": [
|
||||||
"scripts/actions/documentation"
|
"scripts/actions/documentation"
|
||||||
|
|
|
||||||
65
scripts/text-probability.js
Normal file
65
scripts/text-probability.js
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
// @ts-check
|
||||||
|
|
||||||
|
const Ty = require("../src/types")
|
||||||
|
const fs = require("fs")
|
||||||
|
const domino = require("domino")
|
||||||
|
const repl = require("repl")
|
||||||
|
|
||||||
|
const pres = (() => {
|
||||||
|
const pres = []
|
||||||
|
for (const file of process.argv.slice(2)) {
|
||||||
|
const data = JSON.parse(fs.readFileSync(file, "utf8"))
|
||||||
|
/** @type {Ty.Event.Outer<{msgtype?: string}>[]} */
|
||||||
|
const events = data.messages
|
||||||
|
for (const event of events) {
|
||||||
|
if (event.type !== "m.room.message" || event.content.msgtype !== "m.text") continue
|
||||||
|
/** @type {Ty.Event.M_Room_Message} */ // @ts-ignore
|
||||||
|
const content = event.content
|
||||||
|
if (content.format !== "org.matrix.custom.html") continue
|
||||||
|
if (!content.formatted_body) continue
|
||||||
|
|
||||||
|
const document = domino.createDocument(content.formatted_body)
|
||||||
|
// @ts-ignore
|
||||||
|
for (const pre of document.querySelectorAll("pre").cache) {
|
||||||
|
const content = pre.textContent
|
||||||
|
if (content.length < 100) continue
|
||||||
|
pres.push(content)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pres
|
||||||
|
})()
|
||||||
|
|
||||||
|
// @ts-ignore
|
||||||
|
global.gc()
|
||||||
|
|
||||||
|
/** @param {string} text */
|
||||||
|
function probablyFixedWidthIntended(text) {
|
||||||
|
// if internal spaces are used, seems like they want a fixed-width font
|
||||||
|
if (text.match(/[^ ] {3,}[^ ]/)) return true
|
||||||
|
// if characters from Unicode General_Category "Symbol, other" are used, seems like they're doing ascii art and they want a fixed-width font
|
||||||
|
if (text.match(/\p{So}/v)) return true
|
||||||
|
// check start of line indentation
|
||||||
|
let indents = new Set()
|
||||||
|
for (const line of text.trimEnd().split("\n")) {
|
||||||
|
indents.add(line.match(/^ */)?.[0].length || 0)
|
||||||
|
// if there are more than 3 different indents (counting 0) then it's code
|
||||||
|
if (indents.size >= 3) return true
|
||||||
|
}
|
||||||
|
// if everything is indented then it's code
|
||||||
|
if (!indents.has(0)) return true
|
||||||
|
// if there is a high proportion of symbols then it's code (this filter works remarkably well on its own)
|
||||||
|
if ([...text.matchAll(/[\\`~;+|<>%$@*&"'=(){}[\]_^]|\.[a-zA-Z]|[a-z][A-Z]/g)].length / text.length >= 0.04) return true
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
Object.assign(repl.start().context, {pres, probablyFixedWidthIntended})
|
||||||
|
|
||||||
|
/*
|
||||||
|
if it has a lot of symbols then it's code
|
||||||
|
if it has >=3 levels of indentation then it's code
|
||||||
|
if it is all indented then it's code
|
||||||
|
if it has many spaces in a row in the middle then it's ascii art
|
||||||
|
if it has many non-latin characters then it's language
|
||||||
|
-> except if they are ascii art characters e.g. ⣿⣿⡇⢸⣿⠃ then it's ascii art
|
||||||
|
*/
|
||||||
|
|
@ -358,7 +358,7 @@ function assertExistsOrAutocreatable(channel, guildID) {
|
||||||
* @returns {Promise<string>} room ID
|
* @returns {Promise<string>} room ID
|
||||||
*/
|
*/
|
||||||
async function _syncRoom(channelID, shouldActuallySync) {
|
async function _syncRoom(channelID, shouldActuallySync) {
|
||||||
/** @ts-ignore @type {DiscordTypes.APIGuildChannel} */
|
/** @ts-ignore @type {DiscordTypes.APIGuildTextChannel} */
|
||||||
const channel = discord.channels.get(channelID)
|
const channel = discord.channels.get(channelID)
|
||||||
assert.ok(channel)
|
assert.ok(channel)
|
||||||
const guild = channelToGuild(channel)
|
const guild = channelToGuild(channel)
|
||||||
|
|
|
||||||
|
|
@ -396,7 +396,7 @@ async function getMedia(mxc, init = {}) {
|
||||||
...init
|
...init
|
||||||
})
|
})
|
||||||
if (res.status !== 200) {
|
if (res.status !== 200) {
|
||||||
throw await mreq.makeMatrixServerError(res, init)
|
throw await mreq.makeMatrixServerError(res, {...init, url})
|
||||||
}
|
}
|
||||||
if (init.method !== "HEAD") {
|
if (init.method !== "HEAD") {
|
||||||
assert(res.body)
|
assert(res.body)
|
||||||
|
|
|
||||||
|
|
@ -27,9 +27,9 @@ async function makeMatrixServerError(res, opts = {}) {
|
||||||
if (res.headers.get("content-type") === "application/json") {
|
if (res.headers.get("content-type") === "application/json") {
|
||||||
return new MatrixServerError(await res.json(), opts)
|
return new MatrixServerError(await res.json(), opts)
|
||||||
} else if (res.headers.get("content-type")?.startsWith("text/")) {
|
} else if (res.headers.get("content-type")?.startsWith("text/")) {
|
||||||
return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()})
|
return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, message: await res.text()}, opts)
|
||||||
} else {
|
} else {
|
||||||
return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")})
|
return new MatrixServerError({errcode: "CX_SERVER_ERROR", error: `Server returned HTTP status ${res.status}`, content_type: res.headers.get("content-type")}, opts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue