From fc7d4bc4206a54f8b4da6cad5e9427a924a41e02 Mon Sep 17 00:00:00 2001 From: Marie Date: Tue, 7 Nov 2023 19:39:18 +0100 Subject: [PATCH 01/20] chore: set release version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43f3e0525..726666fdf 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2023.11.0.beta4", + "version": "2023.11.0", "codename": "shonk", "repository": { "type": "git", From 1d411bb885b457b6d5660e07e1f1af44f30351bc Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Sun, 26 Nov 2023 18:47:20 +0100 Subject: [PATCH 02/20] chore: fix locales --- locales/en-US.yml | 4 ++-- locales/ja-JP.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index 2726641a6..a3e4fed1e 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -745,8 +745,8 @@ thisIsExperimentalFeature: "This is an experimental feature. Its functionality i developer: "Developer" makeExplorable: "Make account visible in \"Explore\"" makeExplorableDescription: "If you turn this off, your account will not show up in the \"Explore\" section." -makeIndexable: "Make public notes indexable" -makeIndexableDescription: "Allow note search to index your public notes." +makeIndexable: "Make public notes not indexable" +makeIndexableDescription: "Stop note search from indexing your public notes." showGapBetweenNotesInTimeline: "Show a gap between posts on the timeline" duplicate: "Duplicate" left: "Left" diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index fb2a64b39..6eb3ce2f9 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -745,8 +745,8 @@ thisIsExperimentalFeature: "これは実験的な機能です。仕様が変更 developer: "開発者" makeExplorable: "アカウントを見つけやすくする" makeExplorableDescription: "オフにすると、「みつける」にアカウントが載らなくなります。" -makeIndexable: "公開ノートをインデックス化" -makeIndexableDescription: "ノート検索で公開ノートにインデックスを付けられるようにする。" +makeIndexable: "公開ノートをインデックス不可にする" +makeIndexableDescription: "ノート検索があなたの公開ノートをインデックス化しないようにします。" showGapBetweenNotesInTimeline: "タイムラインのノートを離して表示" duplicate: "複製" left: "左" From 021d3924e6061b6a26b3ec49463b2199cc2217de Mon Sep 17 00:00:00 2001 From: Marie Date: Thu, 30 Nov 2023 23:57:04 +0100 Subject: [PATCH 03/20] chore: change version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1207e1802..071812e47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2023.11.2.beta1", + "version": "2023.11.2", "codename": "shonk", "repository": { "type": "git", From 7ba8fde9b942ce8f24dd926be8fb432e5bf68401 Mon Sep 17 00:00:00 2001 From: Marie Date: Sun, 31 Dec 2023 22:49:43 +0100 Subject: [PATCH 04/20] chore: change version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ec9e98175..3aae34b8b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2023.12.0.beta3", + "version": "2023.12.0", "codename": "shonk", "repository": { "type": "git", From 134d2895f00ae9b1de6f1b862ce6a6309ccb8c8e Mon Sep 17 00:00:00 2001 From: Marie Date: Sun, 31 Dec 2023 23:11:15 +0100 Subject: [PATCH 05/20] fix: merge conflict --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 071812e47..01bc4804b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2023.11.2", + "version": "2023.12.0", "codename": "shonk", "repository": { "type": "git", From 4c354fff2d518e84dbd2bb4751c44876a8eedea7 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 10:23:23 +0000 Subject: [PATCH 06/20] Merge branch 'gitlab-ci' into 'develop' --- .config/ci.yml | 216 +++++++++++++++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 104 ++++++++++++++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 .config/ci.yml create mode 100644 .gitlab-ci.yml diff --git a/.config/ci.yml b/.config/ci.yml new file mode 100644 index 000000000..c48fca49b --- /dev/null +++ b/.config/ci.yml @@ -0,0 +1,216 @@ +#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# Misskey configuration +#━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +# ┌─────┐ +#───┘ URL └───────────────────────────────────────────────────── + +# Final accessible URL seen by a user. +url: https://example.tld/ + +# ONCE YOU HAVE STARTED THE INSTANCE, DO NOT CHANGE THE +# URL SETTINGS AFTER THAT! + +# ┌───────────────────────┐ +#───┘ Port and TLS settings └─────────────────────────────────── + +# +# Misskey requires a reverse proxy to support HTTPS connections. +# +# +----- https://example.tld/ ------------+ +# +------+ |+-------------+ +----------------+| +# | User | ---> || Proxy (443) | ---> | Misskey (3000) || +# +------+ |+-------------+ +----------------+| +# +---------------------------------------+ +# +# You need to set up a reverse proxy. (e.g. nginx) +# An encrypted connection with HTTPS is highly recommended +# because tokens may be transferred in GET requests. + +# The port that your Misskey server should listen on. +port: 3000 + +# ┌──────────────────────────┐ +#───┘ PostgreSQL configuration └──────────────────────────────── + +db: + host: postgres + port: 5432 + + # Database name + db: postgres + + # Auth + user: postgres + pass: ci + + # Whether disable Caching queries + #disableCache: true + + # Extra Connection options + #extra: + # ssl: true + +dbReplications: false + +# You can configure any number of replicas here +#dbSlaves: +# - +# host: +# port: +# db: +# user: +# pass: +# - +# host: +# port: +# db: +# user: +# pass: + +# ┌─────────────────────┐ +#───┘ Redis configuration └───────────────────────────────────── + +redis: + host: redis + port: 6379 + #family: 0 # 0=Both, 4=IPv4, 6=IPv6 + #pass: example-pass + #prefix: example-prefix + #db: 1 + +#redisForPubsub: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + +#redisForJobQueue: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + +#redisForTimelines: +# host: redis +# port: 6379 +# #family: 0 # 0=Both, 4=IPv4, 6=IPv6 +# #pass: example-pass +# #prefix: example-prefix +# #db: 1 + +# ┌───────────────────────────┐ +#───┘ MeiliSearch configuration └───────────────────────────── + +# You can set scope to local (default value) or global +# (include notes from remote). + +#meilisearch: +# host: meilisearch +# port: 7700 +# apiKey: '' +# ssl: true +# index: '' +# scope: global + +# ┌───────────────┐ +#───┘ ID generation └─────────────────────────────────────────── + +# You can select the ID generation method. +# You don't usually need to change this setting, but you can +# change it according to your preferences. + +# Available methods: +# aid ... Short, Millisecond accuracy +# aidx ... Millisecond accuracy +# meid ... Similar to ObjectID, Millisecond accuracy +# ulid ... Millisecond accuracy +# objectid ... This is left for backward compatibility + +# ONCE YOU HAVE STARTED THE INSTANCE, DO NOT CHANGE THE +# ID SETTINGS AFTER THAT! + +id: 'aidx' + +# ┌─────────────────────┐ +#───┘ Other configuration └───────────────────────────────────── + +# Whether disable HSTS +#disableHsts: true + +# Number of worker processes +#clusterLimit: 1 + +# Job concurrency per worker +# deliverJobConcurrency: 128 +# inboxJobConcurrency: 16 +# relashionshipJobConcurrency: 16 +# What's relashionshipJob?: +# Follow, unfollow, block and unblock(ings) while following-imports, etc. or account migrations. + +# Job rate limiter +# deliverJobPerSec: 128 +# inboxJobPerSec: 32 +# relashionshipJobPerSec: 64 + +# Job attempts +# deliverJobMaxAttempts: 12 +# inboxJobMaxAttempts: 8 + +# Local address used for outgoing requests +#outgoingAddress: 127.0.0.1 + +# IP address family used for outgoing request (ipv4, ipv6 or dual) +#outgoingAddressFamily: ipv4 + +# Amount of characters that can be used when writing notes (maximum: 8192, minimum: 1) +maxNoteLength: 3000 + +# Proxy for HTTP/HTTPS +#proxy: http://127.0.0.1:3128 + +proxyBypassHosts: + - api.deepl.com + - api-free.deepl.com + - www.recaptcha.net + - hcaptcha.com + - challenges.cloudflare.com + +# Proxy for SMTP/SMTPS +#proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT +#proxySmtp: socks4://127.0.0.1:1080 # use SOCKS4 +#proxySmtp: socks5://127.0.0.1:1080 # use SOCKS5 + +# Media Proxy +#mediaProxy: https://example.com/proxy + +# Proxy remote files (default: true) +# Proxy remote files by this instance or mediaProxy to prevent remote files from running in remote domains. +proxyRemoteFiles: true + +# Movie Thumbnail Generation URL +# There is no reference implementation. +# For example, Misskey will point to the following URL: +# https://example.com/thumbnail.webp?thumbnail=1&url=https%3A%2F%2Fstorage.example.com%2Fpath%2Fto%2Fvideo.mp4 +#videoThumbnailGenerator: https://example.com + +# Sign to ActivityPub GET request (default: true) +signToActivityPubGet: true +# check that inbound ActivityPub GET requests are signed ("authorized fetch") +checkActivityPubGetSignature: false + +# For security reasons, uploading attachments from the intranet is prohibited, +# but exceptions can be made from the following settings. Default value is "undefined". +# Read changelog to learn more (Improvements of 12.90.0 (2021/09/04)). +#allowedPrivateNetworks: [ +# '127.0.0.1/32' +#] + +#customMOTD: ['Hello World', 'The sharks rule all', 'Shonks'] + +# Upload or download file size limits (bytes) +#maxFileSize: 262144000 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..f07b4c9ab --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,104 @@ +stages: + - test + - deploy + +testCommit: + stage: test + image: node:latest + services: + - postgres:15 + - redis + variables: + POSTGRES_PASSWORD: ci + script: + - apt-get update && apt-get install -y git wget curl build-essential python3 + - cp .config/ci.yml .config/default.yml + - corepack enable + - corepack prepare pnpm@latest --activate + - git submodule update --init + - pnpm install --frozen-lockfile + - pnpm run build + - pnpm run migrate + cache: + key: test + policy: pull-push + when: on_success + paths: + - node_modules/ + - packages/*/node_modules/ + only: + - develop + - stable + +getImageTag: + stage: deploy + image: ubuntu:latest + script: + - apt-get update && apt-get install -y jq + - | + if test -n "$CI_COMMIT_TAG"; then + tag="$CI_COMMIT_TAG" + elif test "$CI_COMMIT_BRANCH" == "stable"; then + tag="latest" + elif test "$CI_COMMIT_BRANCH" == "develop"; then + tag="develop" + else + tag="$CI_COMMIT_BRANCH" + fi + version=$(cat package.json | jq -r '.version') + - echo "REGISTRY_PUSH_TAG=$tag" >> build.env + - echo "REGISTRY_PUSH_VERSION=$version" >> build.env + artifacts: + reports: + dotenv: build.env + only: + - stable + - develop + - tags +buildDocker: + stage: deploy + parallel: + matrix: + - ARCH: amd64 + - ARCH: arm64 + tags: + - ${ARCH} + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - >- + /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/Dockerfile" + --destination "${CI_REGISTRY_IMAGE}:${ARCH}" + only: + - stable + - develop + - tags +mergeManifests: + stage: deploy + needs: + - job: buildDocker + artifacts: false + - job: getImageTag + artifacts: true + tags: + - docker + image: + name: mplatform/manifest-tool:alpine + entrypoint: [""] + script: + - >- + manifest-tool + --username=${CI_REGISTRY_USER} + --password=${CI_REGISTRY_PASSWORD} + push from-args + --platforms linux/amd64,linux/arm64 + --tags ${REGISTRY_PUSH_VERSION} + --template ${CI_REGISTRY_IMAGE}:ARCH + --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} + only: + - stable + - develop + - tags \ No newline at end of file From ad8818508f3941c33fab182ee8274ac26ff003a1 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 13:35:30 +0000 Subject: [PATCH 07/20] Update file .gitlab-ci.yml (cherry picked from commit 3b2d47b1e39f9ff5a3bb47ca5f4abbc85fee7944) --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f07b4c9ab..525d6ec65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,14 +40,17 @@ getImageTag: tag="$CI_COMMIT_TAG" elif test "$CI_COMMIT_BRANCH" == "stable"; then tag="latest" + version="$(cat package.json | jq -r '.version'), stable" elif test "$CI_COMMIT_BRANCH" == "develop"; then tag="develop" + version=$(cat package.json | jq -r '.version') else tag="$CI_COMMIT_BRANCH" - fi - version=$(cat package.json | jq -r '.version') + version=$(cat package.json | jq -r '.version') + fi - echo "REGISTRY_PUSH_TAG=$tag" >> build.env - echo "REGISTRY_PUSH_VERSION=$version" >> build.env + artifacts: reports: dotenv: build.env From 4c8116859c0f2f17e46cf986fe7b95c223340fc6 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 13:55:44 +0000 Subject: [PATCH 08/20] Revert "Merge branch 'cherry-pick-3b2d47b1' into 'stable'" This reverts merge request !386 --- .gitlab-ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 525d6ec65..f07b4c9ab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,17 +40,14 @@ getImageTag: tag="$CI_COMMIT_TAG" elif test "$CI_COMMIT_BRANCH" == "stable"; then tag="latest" - version="$(cat package.json | jq -r '.version'), stable" elif test "$CI_COMMIT_BRANCH" == "develop"; then tag="develop" - version=$(cat package.json | jq -r '.version') else tag="$CI_COMMIT_BRANCH" - version=$(cat package.json | jq -r '.version') - fi + fi + version=$(cat package.json | jq -r '.version') - echo "REGISTRY_PUSH_TAG=$tag" >> build.env - echo "REGISTRY_PUSH_VERSION=$version" >> build.env - artifacts: reports: dotenv: build.env From acf3e3460fbafd3b60ac0cee27758be345738584 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 14:00:56 +0000 Subject: [PATCH 09/20] build stable with stable tag --- .gitlab-ci.yml | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f07b4c9ab..4d04825b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,15 +90,26 @@ mergeManifests: entrypoint: [""] script: - >- - manifest-tool - --username=${CI_REGISTRY_USER} - --password=${CI_REGISTRY_PASSWORD} - push from-args - --platforms linux/amd64,linux/arm64 - --tags ${REGISTRY_PUSH_VERSION} - --template ${CI_REGISTRY_IMAGE}:ARCH - --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} + if test "$REGISTRY_PUSH_TAG" == "stable"; then + manifest-tool + --username=${CI_REGISTRY_USER} + --password=${CI_REGISTRY_PASSWORD} + push from-args + --platforms linux/amd64,linux/arm64 + --tags ${REGISTRY_PUSH_VERSION},stable + --template ${CI_REGISTRY_IMAGE}:ARCH + --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} + else + manifest-tool + --username=${CI_REGISTRY_USER} + --password=${CI_REGISTRY_PASSWORD} + push from-args + --platforms linux/amd64,linux/arm64 + --tags ${REGISTRY_PUSH_VERSION} + --template ${CI_REGISTRY_IMAGE}:ARCH + --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} + fi only: - stable - develop - - tags \ No newline at end of file + - tags From 01d695428ad61a1ab520518feae1964973be1c10 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 14:15:10 +0000 Subject: [PATCH 10/20] Revert "build stable with stable tag" This reverts commit acf3e3460fbafd3b60ac0cee27758be345738584 --- .gitlab-ci.yml | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4d04825b8..f07b4c9ab 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,26 +90,15 @@ mergeManifests: entrypoint: [""] script: - >- - if test "$REGISTRY_PUSH_TAG" == "stable"; then - manifest-tool - --username=${CI_REGISTRY_USER} - --password=${CI_REGISTRY_PASSWORD} - push from-args - --platforms linux/amd64,linux/arm64 - --tags ${REGISTRY_PUSH_VERSION},stable - --template ${CI_REGISTRY_IMAGE}:ARCH - --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} - else - manifest-tool - --username=${CI_REGISTRY_USER} - --password=${CI_REGISTRY_PASSWORD} - push from-args - --platforms linux/amd64,linux/arm64 - --tags ${REGISTRY_PUSH_VERSION} - --template ${CI_REGISTRY_IMAGE}:ARCH - --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} - fi + manifest-tool + --username=${CI_REGISTRY_USER} + --password=${CI_REGISTRY_PASSWORD} + push from-args + --platforms linux/amd64,linux/arm64 + --tags ${REGISTRY_PUSH_VERSION} + --template ${CI_REGISTRY_IMAGE}:ARCH + --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} only: - stable - develop - - tags + - tags \ No newline at end of file From 15f3c046d14c1bab5106dcfed051aeef03daf560 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 14:42:19 +0000 Subject: [PATCH 11/20] Update docker-compose_example.yml --- docker-compose_example.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose_example.yml b/docker-compose_example.yml index 5a8560bb4..c967f331a 100644 --- a/docker-compose_example.yml +++ b/docker-compose_example.yml @@ -2,8 +2,7 @@ version: "3" services: web: -# replace image below with git.joinsharkey.org/sharkey/sharkey:stable on next release -# image: ghcr.io/transfem-org/sharkey:stable +# image: registry.activitypub.software/transfem-org/sharkey:latest build: . restart: always links: From a6e257f502296495cf2c1bdc7565e9ce34848506 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 15:06:21 +0000 Subject: [PATCH 12/20] Merge branch 'feture/code-injection-fix' into 'develop' CVE: Fixed code injection from twitter import See merge request TransFem-org/Sharkey!390 (cherry picked from commit 127f8556d409a1082f0050a7ebf57ba846263f6f) 2a8e93e4 Fixed code injection from twitter import --- .../processors/ImportNotesProcessorService.ts | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 03a0e951b..d64a861b0 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -130,6 +130,17 @@ export class ImportNotesProcessorService { return typeof obj[Symbol.iterator] === 'function'; } + private parseTwitterFile(str : string) : null | [{ tweet: any }] { + const removed = str.replace(new RegExp('window\\.YTD\\.tweets\\.part0 = ', 'g'), ''); + + try { + return JSON.parse(removed); + } catch (error) { + //The format is not what we expected. Either this file was tampered with or twitters exports changed + return null; + } + } + @bindThis public async process(job: Bull.Job): Promise { this.logger.info(`Starting note import of ${job.data.user.id} ...`); @@ -175,23 +186,20 @@ export class ImportNotesProcessorService { try { this.logger.succ(`Unzipping to ${outputPath}`); ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); - const fakeWindow: any = { - window: { - YTD: { - tweets: { - part0: {}, - }, - }, - }, - }; - const script = new vm.Script(fs.readFileSync(outputPath + '/data/tweets.js', 'utf-8')); - const context = vm.createContext(fakeWindow); - script.runInContext(context); - const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => { - return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet); - }, []); - const processedTweets = await this.recreateChain(['id_str'], ['in_reply_to_status_id_str'], tweets, false); - this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null); + + const unprocessedTweetJson = this.parseTwitterFile(fs.readFileSync(outputPath + '/data/tweets.js', 'utf-8')); + + //Make sure that it isnt null (because if something went wrong in parseTwitterFile it returns null) + if (unprocessedTweetJson) { + const tweets = Object.keys(unprocessedTweetJson).reduce((m, key, i, obj) => { + return m.concat(unprocessedTweetJson[i].tweet); + }, []); + + const processedTweets = await this.recreateChain(['id_str'], ['in_reply_to_status_id_str'], tweets, false); + this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null); + } else { + this.logger.warn('Failed to import twitter notes due to malformed file'); + } } finally { cleanup(); } From 9c4353ee79b4ceb687c81361acdd4eb63c62084c Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 15:45:42 +0000 Subject: [PATCH 13/20] Update .gitlab-ci.yml (cherry picked from commit 8c5818acf03c0116ee0c7180f1ff5e94927aa27a) --- .gitlab-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f07b4c9ab..9960be505 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,6 +57,9 @@ getImageTag: - tags buildDocker: stage: deploy + needs: + - job: getImageTag + artifacts: true parallel: matrix: - ARCH: amd64 @@ -71,7 +74,7 @@ buildDocker: /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/Dockerfile" - --destination "${CI_REGISTRY_IMAGE}:${ARCH}" + --destination "${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG}-${ARCH}" only: - stable - develop @@ -96,9 +99,9 @@ mergeManifests: push from-args --platforms linux/amd64,linux/arm64 --tags ${REGISTRY_PUSH_VERSION} - --template ${CI_REGISTRY_IMAGE}:ARCH + --template ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG}-ARCH --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} only: - stable - develop - - tags \ No newline at end of file + - tags From 848e1f9a566a968d2a83791533abb0701da347d1 Mon Sep 17 00:00:00 2001 From: Amelia Yukii Date: Thu, 1 Feb 2024 15:59:14 +0000 Subject: [PATCH 14/20] version is better (cherry picked from commit fb455e4fd9fc086203b95b677114ce6a49898743) --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9960be505..aea1307ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -74,7 +74,7 @@ buildDocker: /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/Dockerfile" - --destination "${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG}-${ARCH}" + --destination "${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_VERSION}-${ARCH}" only: - stable - develop @@ -99,7 +99,7 @@ mergeManifests: push from-args --platforms linux/amd64,linux/arm64 --tags ${REGISTRY_PUSH_VERSION} - --template ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG}-ARCH + --template ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_VERSION}-ARCH --target ${CI_REGISTRY_IMAGE}:${REGISTRY_PUSH_TAG} only: - stable From 1948ca9aa8aff9bd36011d8217e6aba2547759b5 Mon Sep 17 00:00:00 2001 From: tamaina Date: Sat, 17 Feb 2024 12:41:19 +0900 Subject: [PATCH 15/20] Merge pull request from GHSA-qqrm-9grj-6v32 --- .../backend/src/core/HttpRequestService.ts | 55 ++++++++++++--- .../src/core/activitypub/ApRequestService.ts | 6 +- .../src/core/activitypub/ApResolverService.ts | 2 +- .../core/activitypub/LdSignatureService.ts | 6 +- .../src/core/activitypub/misc/validator.ts | 39 +++++++++++ .../test/e2e/fetch-validate-ap-deny.ts | 40 +++++++++++ packages/backend/test/unit/activitypub.ts | 2 +- packages/backend/test/utils.ts | 67 +++++++++++++++++-- 8 files changed, 195 insertions(+), 22 deletions(-) create mode 100644 packages/backend/src/core/activitypub/misc/validator.ts create mode 100644 packages/backend/test/e2e/fetch-validate-ap-deny.ts diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 73bb3dc7e..1352e137c 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -14,9 +14,16 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; +import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; +import type { IObject } from '@/core/activitypub/type.js'; import type { Response } from 'node-fetch'; import type { URL } from 'node:url'; +export type HttpRequestSendOptions = { + throwErrorWhenResponseNotOk: boolean; + validators?: ((res: Response) => void)[]; +}; + @Injectable() export class HttpRequestService { /** @@ -104,6 +111,23 @@ export class HttpRequestService { } } + @bindThis + public async getActivityJson(url: string): Promise { + const res = await this.send(url, { + method: 'GET', + headers: { + Accept: 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + }, + timeout: 5000, + size: 1024 * 256, + }, { + throwErrorWhenResponseNotOk: true, + validators: [validateContentTypeSetAsActivityPub], + }); + + return await res.json() as IObject; + } + @bindThis public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { const res = await this.send(url, { @@ -132,17 +156,20 @@ export class HttpRequestService { } @bindThis - public async send(url: string, args: { - method?: string, - body?: string, - headers?: Record, - timeout?: number, - size?: number, - } = {}, extra: { - throwErrorWhenResponseNotOk: boolean; - } = { - throwErrorWhenResponseNotOk: true, - }): Promise { + public async send( + url: string, + args: { + method?: string, + body?: string, + headers?: Record, + timeout?: number, + size?: number, + } = {}, + extra: HttpRequestSendOptions = { + throwErrorWhenResponseNotOk: true, + validators: [], + }, + ): Promise { const timeout = args.timeout ?? 5000; const controller = new AbortController(); @@ -166,6 +193,12 @@ export class HttpRequestService { throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); } + if (res.ok) { + for (const validator of (extra.validators ?? [])) { + validator(res); + } + } + return res; } } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index b59ce5241..bd7b9bdf0 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -14,6 +14,7 @@ import { HttpRequestService } from '@/core/HttpRequestService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; +import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; type Request = { url: string; @@ -66,7 +67,7 @@ export class ApRequestCreator { url: u.href, method: 'GET', headers: this.#objectAssignWithLcKey({ - 'Accept': 'application/activity+json, application/ld+json', + 'Accept': 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"', 'Date': new Date().toUTCString(), 'Host': new URL(args.url).host, }, args.additionalHeaders), @@ -190,6 +191,9 @@ export class ApRequestService { const res = await this.httpRequestService.send(url, { method: req.request.method, headers: req.request.headers, + }, { + throwErrorWhenResponseNotOk: true, + validators: [validateContentTypeSetAsActivityPub], }); return await res.json(); diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index 9ca63c9ec..870cf6372 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -105,7 +105,7 @@ export class Resolver { const object = (this.user ? await this.apRequestService.signedGet(value, this.user) as IObject - : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject; + : await this.httpRequestService.getActivityJson(value)) as IObject; if ( Array.isArray(object['@context']) ? diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index 39b5ff8ab..d8464b383 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -8,6 +8,7 @@ import { Injectable } from '@nestjs/common'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { CONTEXTS } from './misc/contexts.js'; +import { validateContentTypeSetAsJsonLD } from './misc/validator.js'; import type { JsonLdDocument } from 'jsonld'; import type { JsonLd, RemoteDocument } from 'jsonld/jsonld-spec.js'; @@ -133,7 +134,10 @@ class LdSignature { }, timeout: this.loderTimeout, }, - { throwErrorWhenResponseNotOk: false }, + { + throwErrorWhenResponseNotOk: false, + validators: [validateContentTypeSetAsJsonLD], + }, ).then(res => { if (!res.ok) { throw new Error(`${res.status} ${res.statusText}`); diff --git a/packages/backend/src/core/activitypub/misc/validator.ts b/packages/backend/src/core/activitypub/misc/validator.ts new file mode 100644 index 000000000..6ba14a222 --- /dev/null +++ b/packages/backend/src/core/activitypub/misc/validator.ts @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import type { Response } from 'node-fetch'; + +export function validateContentTypeSetAsActivityPub(response: Response): void { + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + + if (contentType === '') { + throw new Error('Validate content type of AP response: No content-type header'); + } + if ( + contentType.startsWith('application/activity+json') || + (contentType.startsWith('application/ld+json;') && contentType.includes('https://www.w3.org/ns/activitystreams')) + ) { + return; + } + throw new Error('Validate content type of AP response: Content type is not application/activity+json or application/ld+json'); +} + +const plusJsonSuffixRegex = /(application|text)\/[a-zA-Z0-9\.\-\+]+\+json/; + +export function validateContentTypeSetAsJsonLD(response: Response): void { + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + + if (contentType === '') { + throw new Error('Validate content type of JSON LD: No content-type header'); + } + if ( + contentType.startsWith('application/ld+json') || + contentType.startsWith('application/json') || + plusJsonSuffixRegex.test(contentType) + ) { + return; + } + throw new Error('Validate content type of JSON LD: Content type is not application/ld+json or application/json'); +} diff --git a/packages/backend/test/e2e/fetch-validate-ap-deny.ts b/packages/backend/test/e2e/fetch-validate-ap-deny.ts new file mode 100644 index 000000000..434a9fe20 --- /dev/null +++ b/packages/backend/test/e2e/fetch-validate-ap-deny.ts @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +process.env.NODE_ENV = 'test'; + +import { validateContentTypeSetAsActivityPub, validateContentTypeSetAsJsonLD } from '@/core/activitypub/misc/validator.js'; +import { signup, uploadFile, relativeFetch } from '../utils.js'; +import type * as misskey from 'misskey-js'; + +describe('validateContentTypeSetAsActivityPub/JsonLD (deny case)', () => { + let alice: misskey.entities.SignupResponse; + let aliceUploadedFile: any; + + beforeAll(async () => { + alice = await signup({ username: 'alice' }); + aliceUploadedFile = await uploadFile(alice); + }, 1000 * 60 * 2); + + test('ActivityStreams: ファイルはエラーになる', async () => { + const res = await relativeFetch(aliceUploadedFile.webpublicUrl); + + function doValidate() { + validateContentTypeSetAsActivityPub(res); + } + + expect(doValidate).toThrow('Content type is not'); + }); + + test('JSON-LD: ファイルはエラーになる', async () => { + const res = await relativeFetch(aliceUploadedFile.webpublicUrl); + + function doValidate() { + validateContentTypeSetAsJsonLD(res); + } + + expect(doValidate).toThrow('Content type is not'); + }); +}); diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 4b9b0cd3d..014e862f9 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -202,7 +202,7 @@ describe('ActivityPub', () => { describe('Renderer', () => { test('Render an announce with visibility: followers', () => { - rendererService.renderAnnounce(null, { + rendererService.renderAnnounce('https://example.com/notes/00example', { id: genAidx(Date.now()), visibility: 'followers', } as MiNote); diff --git a/packages/backend/test/utils.ts b/packages/backend/test/utils.ts index 46b8ea9cd..8c5fa0006 100644 --- a/packages/backend/test/utils.ts +++ b/packages/backend/test/utils.ts @@ -13,6 +13,8 @@ import fetch, { File, RequestInit } from 'node-fetch'; import { DataSource } from 'typeorm'; import { JSDOM } from 'jsdom'; import { DEFAULT_POLICIES } from '@/core/RoleService.js'; +import { Packed } from '@/misc/json-schema.js'; +import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; import { entities } from '../src/postgres.js'; import { loadConfig } from '../src/config.js'; import type * as misskey from 'misskey-js'; @@ -110,6 +112,20 @@ export function randomString(chars = 'abcdefghijklmnopqrstuvwxyz0123456789', len return randomString; } +/** + * @brief プロミスにタイムアウト追加 + * @param p 待ち対象プロミス + * @param timeout 待機ミリ秒 + */ +function timeoutPromise(p: Promise, timeout: number): Promise { + return Promise.race([ + p, + new Promise((reject) => { + setTimeout(() => { reject(new Error('timed out')); }, timeout); + }) as never, + ]); +} + export const signup = async (params?: Partial): Promise> => { const q = Object.assign({ username: randomString(), @@ -304,7 +320,6 @@ export const uploadFile = async (user?: UserToken, { path, name, blob }: UploadO }); const body = res.status !== 204 ? await res.json() as misskey.Endpoints['drive/files/create']['res'] : null; - return { status: res.status, headers: res.headers, @@ -317,12 +332,13 @@ export const uploadUrl = async (user: UserToken, url: string) => { const file = new Promise(ok => resolve = ok); const marker = Math.random().toString(); - const ws = await connectStream(user, 'main', (msg) => { - if (msg.type === 'urlUploadFinished' && msg.body.marker === marker) { - ws.close(); - resolve(msg.body.file); - } - }); + const catcher = makeStreamCatcher( + user, + 'main', + (msg) => msg.type === 'urlUploadFinished' && msg.body.marker === marker, + (msg) => msg.body.file as Packed<'DriveFile'>, + 60 * 1000, + ); await api('drive/files/upload-from-url', { url, @@ -402,6 +418,35 @@ export const waitFire = async (user: UserToken, channel: string, trgr: () => any }); }; +/** + * @brief WebSocketストリームから特定条件の通知を拾うプロミスを生成 + * @param user ユーザー認証情報 + * @param channel チャンネル + * @param cond 条件 + * @param extractor 取り出し処理 + * @param timeout ミリ秒タイムアウト + * @returns 時間内に正常に処理できた場合に通知からextractorを通した値を得る + */ +export function makeStreamCatcher( + user: UserToken, + channel: string, + cond: (message: Record) => boolean, + extractor: (message: Record) => T, + timeout = 60 * 1000): Promise { + let ws: WebSocket; + const p = new Promise(async (resolve) => { + ws = await connectStream(user, channel, (msg) => { + if (cond(msg)) { + resolve(extractor(msg)); + } + }); + }).finally(() => { + ws.close(); + }); + + return timeoutPromise(p, timeout); +} + export type SimpleGetResponse = { status: number, body: any | JSDOM | null, @@ -425,6 +470,14 @@ export const simpleGet = async (path: string, accept = '*/*', cookie: any = unde 'text/html; charset=utf-8', ]; + if (res.ok && ( + accept.startsWith('application/activity+json') || + (accept.startsWith('application/ld+json') && accept.includes('https://www.w3.org/ns/activitystreams')) + )) { + // validateContentTypeSetAsActivityPubのテストを兼ねる + validateContentTypeSetAsActivityPub(res); + } + const body = jsonTypes.includes(res.headers.get('content-type') ?? '') ? await res.json() : htmlTypes.includes(res.headers.get('content-type') ?? '') ? new JSDOM(await res.text()) : From fef7a7b99a0126b43e11689b43237c11d7b78ce2 Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 17 Feb 2024 12:38:01 +0000 Subject: [PATCH 16/20] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 3aae34b8b..a7b105b31 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2023.12.0", + "version": "2023.12.1", "codename": "shonk", "repository": { "type": "git", From 6132bc3b3e51ad059d8e005536a990c0e1226bd2 Mon Sep 17 00:00:00 2001 From: syuilo <4439005+syuilo@users.noreply.github.com> Date: Sat, 17 Feb 2024 14:26:48 +0900 Subject: [PATCH 17/20] fix of 9a70ce8f5ea9df00001894809f5ce7bc69b14c8a Co-Authored-By: RyotaK <49341894+Ry0taK@users.noreply.github.com> --- packages/backend/src/core/activitypub/misc/validator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/activitypub/misc/validator.ts b/packages/backend/src/core/activitypub/misc/validator.ts index 6ba14a222..690beeffe 100644 --- a/packages/backend/src/core/activitypub/misc/validator.ts +++ b/packages/backend/src/core/activitypub/misc/validator.ts @@ -20,7 +20,7 @@ export function validateContentTypeSetAsActivityPub(response: Response): void { throw new Error('Validate content type of AP response: Content type is not application/activity+json or application/ld+json'); } -const plusJsonSuffixRegex = /(application|text)\/[a-zA-Z0-9\.\-\+]+\+json/; +const plusJsonSuffixRegex = /^\s*(application|text)\/[a-zA-Z0-9\.\-\+]+\+json\s*(;|$)/; export function validateContentTypeSetAsJsonLD(response: Response): void { const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); From 7f5492a3954c35490a8f54b5d00523fd7d72db08 Mon Sep 17 00:00:00 2001 From: Marie Date: Sat, 24 Feb 2024 18:20:48 +0000 Subject: [PATCH 18/20] Add missing IMPORTANT_NOTES.md from Sharkey/OldJoinSharkey --- IMPORTANT_NOTES.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 IMPORTANT_NOTES.md diff --git a/IMPORTANT_NOTES.md b/IMPORTANT_NOTES.md new file mode 100644 index 000000000..fba40d21c --- /dev/null +++ b/IMPORTANT_NOTES.md @@ -0,0 +1,13 @@ +# Basic Precautions + +When using a service with Sharkey, there are several important points to keep in mind. + +1. Because it is decentralized, there is no guarantee that data you upload will be deleted from all other servers even if you delete it once. (However, this applies to the internet in general.) + +2. Even for posts made in private, there is no guarantee that the recipient's server will treat them as private in the same way. Please exercise caution when posting personal or confidential information. (Again, this applies to the internet in general.) + +3. Account deletion can be a resource-intensive process and may take a long time. In cases with a lot of uploaded data, it may even be impossible to delete an account. + +4. Please disable ad blockers. Some servers may rely on advertising revenue to cover operating costs. Additionally, ad blockers can mistakenly block content and features unrelated to ads, potentially causing issues with the client's functionality and preventing normal use of Sharkey. Therefore, we recommend turning off ad blockers and similar features when using Sharkey. + +Please understand these points and enjoy using the service. \ No newline at end of file From 6ecfe7c7c3c8d11b486c2a78d93f307ba266b5ec Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 2 Mar 2024 17:34:31 +0000 Subject: [PATCH 19/20] remove duplicate method --- .../queue/processors/ImportNotesProcessorService.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 10cd90c4a..7cef858c5 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -130,17 +130,6 @@ export class ImportNotesProcessorService { return typeof obj[Symbol.iterator] === 'function'; } - private parseTwitterFile(str : string) : null | [{ tweet: any }] { - const removed = str.replace(new RegExp('window\\.YTD\\.tweets\\.part0 = ', 'g'), ''); - - try { - return JSON.parse(removed); - } catch (error) { - //The format is not what we expected. Either this file was tampered with or twitters exports changed - return null; - } - } - @bindThis private parseTwitterFile(str : string) : { tweet: object }[] { const jsonStr = str.replace(/^\s*window\.YTD\.tweets\.part0\s*=\s*/, ''); From 30bb0f60a274131267d28df41d50b5523bea607e Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 30 Mar 2024 11:09:00 +0000 Subject: [PATCH 20/20] version bump --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5e89cba21..f3bfd1db1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sharkey", - "version": "2024.3.1", + "version": "2024.3.2", "codename": "shonk", "repository": { "type": "git",