diff --git a/.circleci/misskey/default.yml b/.circleci/misskey/default.yml index c842431d2..5cdb7330c 100644 --- a/.circleci/misskey/default.yml +++ b/.circleci/misskey/default.yml @@ -6,8 +6,6 @@ mongodb: db: misskey user: syuilo pass: '' -drive: - storage: 'db' redis: host: localhost port: 6379 diff --git a/.circleci/misskey/test.yml b/.circleci/misskey/test.yml index 450c5a79d..99ad50876 100644 --- a/.circleci/misskey/test.yml +++ b/.circleci/misskey/test.yml @@ -6,8 +6,6 @@ mongodb: db: test-misskey user: admin pass: '' -drive: - storage: 'db' # __REDIS__ redis: host: localhost diff --git a/.config/example.yml b/.config/example.yml index db278ecc2..0babd037c 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -78,61 +78,6 @@ redis: # port: 9200 # pass: null -# ┌────────────────────────────────────┐ -#───┘ File storage (Drive) configuration └────────────────────── - -drive: - storage: 'fs' - -# OR - -#drive: -# storage: 'minio' -# bucket: -# prefix: -# config: -# endPoint: -# port: -# useSSL: -# accessKey: -# secretKey: - -# S3/GCS example -# -# * Replace to -# S3: see https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -# GCS: use 'storage.googleapis.com' -# -# * Replace to -# S3: see https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region -# GCS: not needed (just delete the region line) -# -#drive: -# storage: 'minio' -# bucket: bucket-name -# prefix: files -# baseUrl: https://bucket-name. -# config: -# endPoint: -# region: -# useSSL: true -# accessKey: XXX -# secretKey: YYY - -# S3/GCS example (with CDN, custom domain) -# -#drive: -# storage: 'minio' -# bucket: drive.example.com -# prefix: files -# baseUrl: https://drive.example.com -# config: -# endPoint: -# region: -# useSSL: true -# accessKey: XXX -# secretKey: YYY - # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/CHANGELOG.md b/CHANGELOG.md index a17528251..389fdf230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,32 +8,13 @@ If you encounter any problems with updating, please try the following: Migration ------------------------------ #### 1 -`ormconfig.json`という名前で、Misskeyのインストール場所(package.jsonとかがあるディレクトリ)に新たなファイルを作る。中身は次のようにします: -``` json -{ - "type": "postgres", - "host": "PostgreSQLのホスト", - "port": 5432, - "username": "PostgreSQLのユーザー名", - "password": "PostgreSQLのパスワード", - "database": "PostgreSQLのデータベース名", - "entities": ["src/models/entities/*.ts"], - "migrations": ["migration/*.ts"], - "cli": { - "migrationsDir": "migration" - } -} -``` -上記の各種PostgreSQLの設定(ポートも)は、設定ファイルに書いてあるものをコピーしてください。 - -#### 2 ``` npm i -g ts-node ``` -#### 3 +#### 2 ``` -ts-node ./node_modules/typeorm/cli.js migration:run +npm run migrate ``` How to migrate to v11 from v10 @@ -73,6 +54,20 @@ mongodb: 8. master ブランチに戻す 9. enjoy +11.14.0 (2019/05/16) +-------------------- +### 注意 +このバージョンからオブジェクトストレージの設定は設定ファイルではなく管理画面から行うようになりました。 +オブジェクトストレージを使用している場合、アップデートした後管理画面にアクセスしオブジェクトストレージの設定を再度行ってください。 + +### ✨Improvements +* 特定のユーザーのファイルをすべて削除できるように +* インスタンスの設定画面を整理 + +### 🐛Fixes +* GIF画像のサムネイルが生成されないのを修正 +* 管理画面の「ログ」で複数の除外条件を設定できない問題を修正 + 11.13.0 (2019/05/14) -------------------- ### 注意 @@ -85,12 +80,13 @@ mongodb: * ユーザーや外部インスタンスが生成するリンクにnofollowを追加 * リモートのユーザーページやノートページにnoindexを追加 * 自分のユーザーメニューにはミュートなどを表示しないように +* デザインの調整 ### 🐛Fixes * インスタンスブロックを設定できない問題を修正 * ピン留め投稿の表示順がおかしい問題を修正 * 設定の「アップデートを確認」でメッセージが正しく表示されない問題を修正 -* FFirefoxで自分のメニューが開けない問題を修正 +* Firefoxで自分のメニューが開けない問題を修正 * Welcomeページのタグクラウドが動かない問題を修正 11.12.0 (2019/05/10) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0c72c470..ace822c63 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -199,7 +199,7 @@ const user = await Users.findOne(userId).then(ensure); ``` ### Migration作成方法 -コードの変更をした後、`ormconfig.json`（書き方はCONTRIBUTING.mdを参照）を用意し、 +コードの変更をした後、`ormconfig.json`（`npm run ormconfig`で生成）を用意し、 ``` npm i -g ts-node diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index ed0da44d6..bb991459c 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1187,7 +1187,6 @@ admin/views/index.vue: users: "ユーザー" federation: "連合" announcements: "お知らせ" - hashtags: "ハッシュタグ" abuse: "スパム報告" queue: "ジョブキュー" logs: "ログ" @@ -1230,7 +1229,22 @@ admin/views/instance.vue: maintainer-config: "管理者情報" maintainer-name: "管理者名" maintainer-email: "管理者の連絡先" + advanced-config: "その他の設定" + note-and-tl: "投稿とタイムライン" drive-config: "ドライブの設定" + use-object-storage: "オブジェクトストレージを使用する" + object-storage-base-url: "URL" + object-storage-bucket: "バケット名" + object-storage-prefix: "プレフィックス" + object-storage-endpoint: "エンドポイント" + object-storage-region: "リージョン" + object-storage-port: "ポート" + object-storage-access-key: "アクセスキー" + object-storage-secret-key: "シークレットキー" + object-storage-use-ssl: "SSLを使用" + object-storage-s3-info: "Amazon S3をオブジェクトストレージとして使用する場合の「エンドポイント」と「リージョン」の設定については{0}をご確認ください。" + object-storage-s3-info-here: "こちら" + object-storage-gcs-info: "Google Cloud Storageをオブジェクトストレージとして使用する場合、「エンドポイント」は storage.googleapis.com に設定し、「リージョン」は空欄にします。" cache-remote-files: "リモートのファイルをキャッシュする" cache-remote-files-desc: "この設定を無効にすると、リモートファイルをキャッシュせず直リンクするようになります。そのためサーバーのストレージを節約できますが、プライバシー設定で直リンクを無効にしているユーザーにはファイルが見えなくなったり、サムネイルが生成されないので通信量が増加します。通常はこの設定をオンにしておくことをおすすめします。" local-drive-capacity-mb: "ローカルユーザーひとりあたりのドライブ容量" @@ -1241,6 +1255,9 @@ admin/views/instance.vue: enable-recaptcha: "reCAPTCHAを有効にする" recaptcha-site-key: "reCAPTCHA site key" recaptcha-secret-key: "reCAPTCHA secret key" + hidden-tags: "非表示ハッシュタグ" + hidden-tags-info: "集計から除外するハッシュタグを改行で区切って記述します。" + external-service-integration-config: "外部サービス連携" twitter-integration-config: "Twitter連携の設定" twitter-integration-info: "コールバックURLは {url} に設定します。" enable-twitter-integration: "Twitter連携を有効にする" @@ -1361,6 +1378,8 @@ admin/views/users.vue: unsilence-confirm: "サイレンスを解除しますか？" update-remote-user: "リモートユーザー情報の更新" remote-user-updated: "リモートユーザー情報を更新しました" + delete-all-files: "すべてのファイルを削除" + delete-all-files-confirm: "すべてのファイルを削除しますか？" users: title: "ユーザー" sort: diff --git a/migration/1557932705754-ObjectStorageSetting.ts b/migration/1557932705754-ObjectStorageSetting.ts new file mode 100644 index 000000000..dde6aa65f --- /dev/null +++ b/migration/1557932705754-ObjectStorageSetting.ts @@ -0,0 +1,31 @@ +import {MigrationInterface, QueryRunner} from "typeorm"; + +export class ObjectStorageSetting1557932705754 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" ADD "useObjectStorage" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageBucket" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStoragePrefix" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageBaseUrl" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageEndpoint" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageRegion" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageAccessKey" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageSecretKey" character varying(512)`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStoragePort" integer`); + await queryRunner.query(`ALTER TABLE "meta" ADD "objectStorageUseSSL" boolean NOT NULL DEFAULT true`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageUseSSL"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStoragePort"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageSecretKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageAccessKey"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageRegion"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageEndpoint"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageBaseUrl"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStoragePrefix"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "objectStorageBucket"`); + await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "useObjectStorage"`); + } + +} diff --git a/package.json b/package.json index 71fa1509e..8759edd4a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "misskey", "author": "syuilo ", - "version": "11.13.0", + "version": "11.14.0", "codename": "daybreak", "repository": { "type": "git", @@ -12,6 +12,8 @@ "scripts": { "start": "node ./index.js", "init": "node ./built/init.js", + "ormconfig": "node ./built/ormconfig.js", + "migrate": "npm run ormconfig && ts-node ./node_modules/typeorm/cli.js migration:run", "build": "webpack && gulp build", "webpack": "webpack", "watch": "webpack --watch", diff --git a/src/client/app/admin/views/hashtags.vue b/src/client/app/admin/views/hashtags.vue deleted file mode 100644 index e1cc4b494..000000000 --- a/src/client/app/admin/views/hashtags.vue +++ /dev/null @@ -1,41 +0,0 @@ - - - diff --git a/src/client/app/admin/views/index.vue b/src/client/app/admin/views/index.vue index 4bce197ed..43e47038f 100644 --- a/src/client/app/admin/views/index.vue +++ b/src/client/app/admin/views/index.vue @@ -28,7 +28,6 @@
  • {{ $t('federation') }}
  • {{ $t('emoji') }}
  • {{ $t('announcements') }}
    • -
  • {{ $t('hashtags') }}
  • {{ $t('abuse') }}
    • @@ -48,7 +47,6 @@
    -
    @@ -68,7 +66,6 @@ import XLogs from "./logs.vue"; import XModerators from "./moderators.vue"; import XEmoji from "./emoji.vue"; import XAnnouncements from "./announcements.vue"; -import XHashtags from "./hashtags.vue"; import XUsers from "./users.vue"; import XDrive from "./drive.vue"; import XAbuse from "./abuse.vue"; @@ -91,7 +88,6 @@ export default Vue.extend({ XModerators, XEmoji, XAnnouncements, - XHashtags, XUsers, XDrive, XAbuse, diff --git a/src/client/app/admin/views/instance.vue b/src/client/app/admin/views/instance.vue index 5cdd22296..3ac4d6d72 100644 --- a/src/client/app/admin/views/instance.vue +++ b/src/client/app/admin/views/instance.vue @@ -2,7 +2,7 @@
    -
    +
    {{ $t('host') }} {{ $t('instance-name') }} {{ $t('instance-description') }} @@ -11,77 +11,83 @@ {{ $t('banner-url') }} {{ $t('error-image-url') }} {{ $t('tos-url') }} - {{ $t('repository-url') }} - {{ $t('feedback-url') }} {{ $t('languages') }} +
    + {{ $t('advanced-config') }} + {{ $t('repository-url') }} + {{ $t('feedback-url') }} +
    {{ $t('maintainer-config') }}
    {{ $t('maintainer-name') }} {{ $t('maintainer-email') }}
    +
    + {{ $t('disable-registration') }} + {{ $t('invite') }} +
    +
    + {{ $t('save') }} +
    + + + +
    {{ $t('max-note-text-length') }}
    - {{ $t('disable-registration') }} {{ $t('disable-local-timeline') }} {{ $t('disable-global-timeline') }} {{ $t('disabling-timelines-info') }} +
    +
    {{ $t('enable-emoji-reaction') }} {{ $t('use-star-for-reaction-fallback') }}
    -
    -
    {{ $t('drive-config') }}
    +
    + {{ $t('save') }} +
    + + + + +
    + {{ $t('use-object-storage') }} + +
    +
    {{ $t('cache-remote-files') }} +
    +
    {{ $t('local-drive-capacity-mb') }} {{ $t('remote-drive-capacity-mb') }}
    -
    -
    {{ $t('recaptcha-config') }}
    - {{ $t('enable-recaptcha') }} - {{ $t('recaptcha-info') }} - - {{ $t('recaptcha-site-key') }} - {{ $t('recaptcha-secret-key') }} - -
    -
    {{ $t('proxy-account-config') }}
    - {{ $t('proxy-account-info') }} - {{ $t('proxy-account-username') }} - {{ $t('proxy-account-warn') }} -
    -
    -
    {{ $t('email-config') }}
    - {{ $t('enable-email') }} - {{ $t('email') }} - - {{ $t('smtp-host') }} - {{ $t('smtp-port') }} - - {{ $t('smtp-auth') }} - - {{ $t('smtp-user') }} - {{ $t('smtp-pass') }} - - {{ $t('smtp-secure') }} -
    -
    -
    {{ $t('serviceworker-config') }}
    - {{ $t('enable-serviceworker') }} - {{ $t('vapid-info') }}
    npm i web-push -g
    web-push generate-vapid-keys     - - {{ $t('vapid-publickey') }} - {{ $t('vapid-privatekey') }} - -
    -
    -
    summaly Proxy
    - URL -
    -
    - {{ $t('save') }} + {{ $t('save') }}
    @@ -91,56 +97,142 @@ - {{ $t('save') }} + {{ $t('save') }}
    - +
    - {{ $t('invite') }} -

    Code: {{ inviteCode }}

    + {{ $t('proxy-account-info') }} + {{ $t('proxy-account-username') }} + {{ $t('proxy-account-warn') }} +
    +
    + {{ $t('save') }}
    - +
    + {{ $t('enable-email') }} + +
    +
    + {{ $t('save') }} +
    +     + + + +
    + {{ $t('enable-serviceworker') }} + +
    +
    + {{ $t('save') }} +
    +     + + + +
    + {{ $t('enable-recaptcha') }} + +
    +
    + {{ $t('save') }} +
    +     + + + +
    +
    {{ $t('twitter-integration-config') }}
    {{ $t('enable-twitter-integration') }} - - {{ $t('twitter-integration-consumer-key') }} - {{ $t('twitter-integration-consumer-secret') }} - - {{ $t('twitter-integration-info', { url: `${url}/api/tw/cb` }) }} - {{ $t('save') }} +
    -     - - -
    +
    {{ $t('github-integration-config') }}
    {{ $t('enable-github-integration') }} - - {{ $t('github-integration-client-id') }} - {{ $t('github-integration-client-secret') }} - - {{ $t('github-integration-info', { url: `${url}/api/gh/cb` }) }} - {{ $t('save') }} + +
    +
    +
    {{ $t('discord-integration-config') }}
    + {{ $t('enable-discord-integration') }} + +
    +
    + {{ $t('save') }}
    - - -
    - {{ $t('enable-discord-integration') }} - - {{ $t('discord-integration-client-id') }} - {{ $t('discord-integration-client-secret') }} - - {{ $t('discord-integration-info', { url: `${url}/api/dc/cb` }) }} - {{ $t('save') }} -
    -     +
    + {{ $t('advanced-config') }} + + + +
    + + + + {{ $t('save') }} +
    +     + + + +
    + URL +
    +
    + {{ $t('save') }} +
    +     +
    @@ -149,8 +241,8 @@ import Vue from 'vue'; import i18n from '../../i18n'; import { url, host } from '../../config'; import { toUnicode } from 'punycode'; -import { faHeadset, faShieldAlt, faGhost, faUserPlus, faBolt, faThumbtack } from '@fortawesome/free-solid-svg-icons'; -import { faEnvelope as farEnvelope } from '@fortawesome/free-regular-svg-icons'; +import { faHeadset, faShieldAlt, faGhost, faUserPlus, faBolt, faThumbtack, faPencilAlt, faHashtag } from '@fortawesome/free-solid-svg-icons'; +import { faEnvelope as farEnvelope, faSave } from '@fortawesome/free-regular-svg-icons'; export default Vue.extend({ i18n: i18n('admin/views/instance.vue'), @@ -193,7 +285,6 @@ export default Vue.extend({ discordClientId: null, discordClientSecret: null, proxyAccount: null, - inviteCode: null, summalyProxy: null, enableEmail: false, email: null, @@ -207,7 +298,18 @@ export default Vue.extend({ swPublicKey: null, swPrivateKey: null, pinnedUsers: '', - faHeadset, faShieldAlt, faGhost, faUserPlus, farEnvelope, faBolt, faThumbtack + hiddenTags: '', + useObjectStorage: false, + objectStorageBaseUrl: null, + objectStorageBucket: null, + objectStoragePrefix: null, + objectStorageEndpoint: null, + objectStorageRegion: null, + objectStoragePort: null, + objectStorageAccessKey: null, + objectStorageSecretKey: null, + objectStorageUseSSL: false, + faHeadset, faShieldAlt, faGhost, faUserPlus, farEnvelope, faBolt, faThumbtack, faPencilAlt, faSave, faHashtag }; }, @@ -260,13 +362,27 @@ export default Vue.extend({ this.swPublicKey = meta.swPublickey; this.swPrivateKey = meta.swPrivateKey; this.pinnedUsers = meta.pinnedUsers.join('\n'); + this.hiddenTags = meta.hiddenTags.join('\n'); + this.useObjectStorage = meta.useObjectStorage; + this.objectStorageBaseUrl = meta.objectStorageBaseUrl; + this.objectStorageBucket = meta.objectStorageBucket; + this.objectStoragePrefix = meta.objectStoragePrefix; + this.objectStorageEndpoint = meta.objectStorageEndpoint; + this.objectStorageRegion = meta.objectStorageRegion; + this.objectStoragePort = meta.objectStoragePort; + this.objectStorageAccessKey = meta.objectStorageAccessKey; + this.objectStorageSecretKey = meta.objectStorageSecretKey; + this.objectStorageUseSSL = meta.objectStorageUseSSL; }); }, methods: { invite() { this.$root.api('admin/invite').then(x => { - this.inviteCode = x.code; + this.$root.dialog({ + type: 'info', + text: x.code + }); }).catch(e => { this.$root.dialog({ type: 'error', @@ -322,7 +438,18 @@ export default Vue.extend({ enableServiceWorker: this.enableServiceWorker, swPublicKey: this.swPublicKey, swPrivateKey: this.swPrivateKey, - pinnedUsers: this.pinnedUsers.split('\n') + pinnedUsers: this.pinnedUsers.split('\n'), + hiddenTags: this.hiddenTags.split('\n'), + useObjectStorage: this.useObjectStorage, + objectStorageBaseUrl: this.objectStorageBaseUrl ? this.objectStorageBaseUrl : null, + objectStorageBucket: this.objectStorageBucket ? this.objectStorageBucket : null, + objectStoragePrefix: this.objectStoragePrefix ? this.objectStoragePrefix : null, + objectStorageEndpoint: this.objectStorageEndpoint ? this.objectStorageEndpoint : null, + objectStorageRegion: this.objectStorageRegion ? this.objectStorageRegion : null, + objectStoragePort: this.objectStoragePort ? this.objectStoragePort : null, + objectStorageAccessKey: this.objectStorageAccessKey ? this.objectStorageAccessKey : null, + objectStorageSecretKey: this.objectStorageSecretKey ? this.objectStorageSecretKey : null, + objectStorageUseSSL: this.objectStorageUseSSL, }).then(() => { this.$root.dialog({ type: 'success', diff --git a/src/client/app/admin/views/users.vue b/src/client/app/admin/views/users.vue index cc3810853..fd9f0dd8b 100644 --- a/src/client/app/admin/views/users.vue +++ b/src/client/app/admin/views/users.vue @@ -9,8 +9,9 @@ {{ $t('lookup') }}
    - +
    + {{ $t('update-remote-user') }} {{ $t('reset-password') }} {{ $t('make-silence') }} @@ -20,7 +21,7 @@ {{ $t('suspend') }} {{ $t('unsuspend') }} - {{ $t('update-remote-user') }} + {{ $t('delete-all-files') }}
    @@ -67,7 +68,7 @@ import Vue from 'vue'; import i18n from '../../i18n'; import parseAcct from "../../../../misc/acct/parse"; import { faUsers, faTerminal, faSearch, faKey, faSync, faMicrophoneSlash } from '@fortawesome/free-solid-svg-icons'; -import { faSnowflake } from '@fortawesome/free-regular-svg-icons'; +import { faSnowflake, faTrashAlt } from '@fortawesome/free-regular-svg-icons'; import XUser from './users.user.vue'; export default Vue.extend({ @@ -88,7 +89,7 @@ export default Vue.extend({ offset: 0, users: [], existMore: false, - faTerminal, faUsers, faSnowflake, faSearch, faKey, faSync, faMicrophoneSlash + faTerminal, faUsers, faSnowflake, faSearch, faKey, faSync, faMicrophoneSlash, faTrashAlt }; }, @@ -277,6 +278,25 @@ export default Vue.extend({ this.refreshUser(); }, + async deleteAllFiles() { + if (!await this.getConfirmed(this.$t('delete-all-files-confirm'))) return; + + const process = async () => { + await this.$root.api('admin/delete-all-files-of-a-user', { userId: this.user.id }); + this.$root.dialog({ + type: 'success', + splash: true + }); + }; + + await process().catch(e => { + this.$root.dialog({ + type: 'error', + text: e.toString() + }); + }); + }, + async getConfirmed(text: string): Promise { const confirm = await this.$root.dialog({ type: 'warning', diff --git a/src/config/types.ts b/src/config/types.ts index d312a5a18..7da9820f2 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -27,13 +27,6 @@ export type Source = { port: number; pass: string; }; - drive?: { - storage: string; - bucket?: string; - prefix?: string; - baseUrl?: string; - config?: any; - }; autoAdmin?: boolean; diff --git a/src/models/entities/meta.ts b/src/models/entities/meta.ts index c3797a9ed..fdd281823 100644 --- a/src/models/entities/meta.ts +++ b/src/models/entities/meta.ts @@ -288,4 +288,61 @@ export class Meta { nullable: true }) public feedbackUrl: string | null; + + @Column('boolean', { + default: false, + }) + public useObjectStorage: boolean; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageBucket: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStoragePrefix: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageBaseUrl: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageEndpoint: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageRegion: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageAccessKey: string | null; + + @Column('varchar', { + length: 512, + nullable: true + }) + public objectStorageSecretKey: string | null; + + @Column('integer', { + nullable: true + }) + public objectStoragePort: number | null; + + @Column('boolean', { + default: true, + }) + public objectStorageUseSSL: boolean; } diff --git a/src/ormconfig.ts b/src/ormconfig.ts new file mode 100644 index 000000000..91f33181f --- /dev/null +++ b/src/ormconfig.ts @@ -0,0 +1,18 @@ +import * as fs from 'fs'; +import config from './config'; + +const json = { + type: 'postgres', + host: config.db.host, + port: config.db.port, + username: config.db.user, + password: config.db.pass, + database: config.db.db, + entities: ['src/models/entities/*.ts'], + migrations: ['migration/*.ts'], + cli: { + migrationsDir: 'migration' + } +}; + +fs.writeFileSync('ormconfig.json', JSON.stringify(json)); diff --git a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts new file mode 100644 index 000000000..84e9c363e --- /dev/null +++ b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts @@ -0,0 +1,32 @@ +import $ from 'cafy'; +import define from '../../define'; +import del from '../../../../services/drive/delete-file'; +import { DriveFiles } from '../../../../models'; +import { ID } from '../../../../misc/cafy-id'; + +export const meta = { + tags: ['admin'], + + requireCredential: true, + requireModerator: true, + + params: { + userId: { + validator: $.type(ID), + desc: { + 'ja-JP': '対象のユーザーID', + 'en-US': 'The user ID which you want to suspend' + } + }, + } +}; + +export default define(meta, async (ps, me) => { + const files = await DriveFiles.find({ + userId: ps.userId + }); + + for (const file of files) { + del(file); + } +}); diff --git a/src/server/api/endpoints/admin/logs.ts b/src/server/api/endpoints/admin/logs.ts index 86e99730c..060df09ad 100644 --- a/src/server/api/endpoints/admin/logs.ts +++ b/src/server/api/endpoints/admin/logs.ts @@ -53,16 +53,18 @@ export default define(meta, async (ps) => { if (blackDomains.length > 0) { query.andWhere(new Brackets(qb => { for (const blackDomain of blackDomains) { - const subDomains = blackDomain.split('.'); - let i = 0; - for (const subDomain of subDomains) { - const p = `blackSubDomain_${subDomain}_${i}`; - // 全体で否定できないのでド・モルガンの法則で - // !(P && Q) を !P || !Q で表す - // SQL is 1 based, so we need '+ 1' - qb.orWhere(`log.domain[${i + 1}] != :${p}`, { [p]: subDomain }); - i++; - } + qb.andWhere(new Brackets(qb => { + const subDomains = blackDomain.split('.'); + let i = 0; + for (const subDomain of subDomains) { + const p = `blackSubDomain_${subDomain}_${i}`; + // 全体で否定できないのでド・モルガンの法則で + // !(P && Q) を !P || !Q で表す + // SQL is 1 based, so we need '+ 1' + qb.orWhere(`log.domain[${i + 1}] != :${p}`, { [p]: subDomain }); + i++; + } + })); } })); } diff --git a/src/server/api/endpoints/admin/update-meta.ts b/src/server/api/endpoints/admin/update-meta.ts index e4f2e86aa..8e98d203f 100644 --- a/src/server/api/endpoints/admin/update-meta.ts +++ b/src/server/api/endpoints/admin/update-meta.ts @@ -357,7 +357,47 @@ export const meta = { desc: { 'ja-JP': 'フィードバックのURL' } - } + }, + + useObjectStorage: { + validator: $.optional.bool + }, + + objectStorageBaseUrl: { + validator: $.optional.nullable.str + }, + + objectStorageBucket: { + validator: $.optional.nullable.str + }, + + objectStoragePrefix: { + validator: $.optional.nullable.str + }, + + objectStorageEndpoint: { + validator: $.optional.nullable.str + }, + + objectStorageRegion: { + validator: $.optional.nullable.str + }, + + objectStoragePort: { + validator: $.optional.nullable.num + }, + + objectStorageAccessKey: { + validator: $.optional.nullable.str + }, + + objectStorageSecretKey: { + validator: $.optional.nullable.str + }, + + objectStorageUseSSL: { + validator: $.optional.bool + }, } }; @@ -560,6 +600,46 @@ export default define(meta, async (ps) => { set.feedbackUrl = ps.feedbackUrl; } + if (ps.useObjectStorage !== undefined) { + set.useObjectStorage = ps.useObjectStorage; + } + + if (ps.objectStorageBaseUrl !== undefined) { + set.objectStorageBaseUrl = ps.objectStorageBaseUrl; + } + + if (ps.objectStorageBucket !== undefined) { + set.objectStorageBucket = ps.objectStorageBucket; + } + + if (ps.objectStoragePrefix !== undefined) { + set.objectStoragePrefix = ps.objectStoragePrefix; + } + + if (ps.objectStorageEndpoint !== undefined) { + set.objectStorageEndpoint = ps.objectStorageEndpoint; + } + + if (ps.objectStorageRegion !== undefined) { + set.objectStorageRegion = ps.objectStorageRegion; + } + + if (ps.objectStoragePort !== undefined) { + set.objectStoragePort = ps.objectStoragePort; + } + + if (ps.objectStorageAccessKey !== undefined) { + set.objectStorageAccessKey = ps.objectStorageAccessKey; + } + + if (ps.objectStorageSecretKey !== undefined) { + set.objectStorageSecretKey = ps.objectStorageSecretKey; + } + + if (ps.objectStorageUseSSL !== undefined) { + set.objectStorageUseSSL = ps.objectStorageUseSSL; + } + await getConnection().transaction(async transactionalEntityManager => { const meta = await transactionalEntityManager.findOne(Meta, { order: { diff --git a/src/server/api/endpoints/meta.ts b/src/server/api/endpoints/meta.ts index 1bd88a1e6..4f418c63c 100644 --- a/src/server/api/endpoints/meta.ts +++ b/src/server/api/endpoints/meta.ts @@ -153,7 +153,7 @@ export default define(meta, async (ps, me) => { globalTimeLine: !instance.disableGlobalTimeline, elasticsearch: config.elasticsearch ? true : false, recaptcha: instance.enableRecaptcha, - objectStorage: config.drive && config.drive.storage === 'minio', + objectStorage: instance.useObjectStorage, twitter: instance.enableTwitterIntegration, github: instance.enableGithubIntegration, discord: instance.enableDiscordIntegration, @@ -182,6 +182,16 @@ export default define(meta, async (ps, me) => { response.smtpUser = instance.smtpUser; response.smtpPass = instance.smtpPass; response.swPrivateKey = instance.swPrivateKey; + response.useObjectStorage = instance.useObjectStorage; + response.objectStorageBaseUrl = instance.objectStorageBaseUrl; + response.objectStorageBucket = instance.objectStorageBucket; + response.objectStoragePrefix = instance.objectStoragePrefix; + response.objectStorageEndpoint = instance.objectStorageEndpoint; + response.objectStorageRegion = instance.objectStorageRegion; + response.objectStoragePort = instance.objectStoragePort; + response.objectStorageAccessKey = instance.objectStorageAccessKey; + response.objectStorageSecretKey = instance.objectStorageSecretKey; + response.objectStorageUseSSL = instance.objectStorageUseSSL; } return response; diff --git a/src/server/proxy/proxy-media.ts b/src/server/proxy/proxy-media.ts index 357715bb9..e16665f6c 100644 --- a/src/server/proxy/proxy-media.ts +++ b/src/server/proxy/proxy-media.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as Koa from 'koa'; import { serverLogger } from '..'; -import { IImage, ConvertToPng, ConvertToJpeg } from '../../services/drive/image-processor'; +import { IImage, convertToPng, convertToJpeg } from '../../services/drive/image-processor'; import { createTemp } from '../../misc/create-temp'; import { downloadUrl } from '../../misc/donwload-url'; import { detectMine } from '../../misc/detect-mine'; @@ -20,9 +20,9 @@ export async function proxyMedia(ctx: Koa.BaseContext) { let image: IImage; if ('static' in ctx.query && ['image/png', 'image/gif'].includes(type)) { - image = await ConvertToPng(path, 498, 280); + image = await convertToPng(path, 498, 280); } else if ('preview' in ctx.query && ['image/jpeg', 'image/png', 'image/gif'].includes(type)) { - image = await ConvertToJpeg(path, 200, 200); + image = await convertToJpeg(path, 200, 200); } else { image = { data: fs.readFileSync(path), diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index c67ee475a..701878b28 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -8,11 +8,10 @@ import * as sharp from 'sharp'; import { publishMainStream, publishDriveStream } from '../stream'; import delFile from './delete-file'; -import config from '../../config'; import { fetchMeta } from '../../misc/fetch-meta'; import { GenerateVideoThumbnail } from './generate-video-thumbnail'; import { driveLogger } from './logger'; -import { IImage, ConvertToJpeg, ConvertToWebp, ConvertToPng } from './image-processor'; +import { IImage, convertToJpeg, convertToWebp, convertToPng, convertToGif, convertToApng } from './image-processor'; import { contentDisposition } from '../../misc/content-disposition'; import { detectMine } from '../../misc/detect-mine'; import { DriveFiles, DriveFolders, Users, Instances, UserProfiles } from '../../models'; @@ -37,7 +36,9 @@ async function save(file: DriveFile, path: string, name: string, type: string, h // thunbnail, webpublic を必要なら生成 const alts = await generateAlts(path, type, !file.uri); - if (config.drive && config.drive.storage == 'minio') { + const meta = await fetchMeta(); + + if (meta.useObjectStorage) { //#region ObjectStorage params let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) || ['']); @@ -47,11 +48,11 @@ async function save(file: DriveFile, path: string, name: string, type: string, h if (type === 'image/webp') ext = '.webp'; } - const baseUrl = config.drive.baseUrl - || `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`; + const baseUrl = meta.objectStorageBaseUrl + || `${ meta.objectStorageUseSSL ? 'https' : 'http' }://${ meta.objectStorageEndpoint }${ meta.objectStoragePort ? `:${meta.objectStoragePort}` : '' }/${ meta.objectStorageBucket }`; // for original - const key = `${config.drive.prefix}/${uuid.v4()}${ext}`; + const key = `${meta.objectStoragePrefix}/${uuid.v4()}${ext}`; const url = `${ baseUrl }/${ key }`; // for alts @@ -68,7 +69,7 @@ async function save(file: DriveFile, path: string, name: string, type: string, h ]; if (alts.webpublic) { - webpublicKey = `${config.drive.prefix}/${uuid.v4()}.${alts.webpublic.ext}`; + webpublicKey = `${meta.objectStoragePrefix}/${uuid.v4()}.${alts.webpublic.ext}`; webpublicUrl = `${ baseUrl }/${ webpublicKey }`; logger.info(`uploading webpublic: ${webpublicKey}`); @@ -76,7 +77,7 @@ async function save(file: DriveFile, path: string, name: string, type: string, h } if (alts.thumbnail) { - thumbnailKey = `${config.drive.prefix}/${uuid.v4()}.${alts.thumbnail.ext}`; + thumbnailKey = `${meta.objectStoragePrefix}/${uuid.v4()}.${alts.thumbnail.ext}`; thumbnailUrl = `${ baseUrl }/${ thumbnailKey }`; logger.info(`uploading thumbnail: ${thumbnailKey}`); @@ -149,11 +150,15 @@ export async function generateAlts(path: string, type: string, generateWeb: bool logger.info(`creating web image`); if (['image/jpeg'].includes(type)) { - webpublic = await ConvertToJpeg(path, 2048, 2048); + webpublic = await convertToJpeg(path, 2048, 2048); } else if (['image/webp'].includes(type)) { - webpublic = await ConvertToWebp(path, 2048, 2048); + webpublic = await convertToWebp(path, 2048, 2048); } else if (['image/png'].includes(type)) { - webpublic = await ConvertToPng(path, 2048, 2048); + webpublic = await convertToPng(path, 2048, 2048); + } else if (['image/apng', 'image/vnd.mozilla.apng'].includes(type)) { + webpublic = await convertToApng(path); + } else if (['image/gif'].includes(type)) { + webpublic = await convertToGif(path); } else { logger.info(`web image not created (not an image)`); } @@ -166,9 +171,11 @@ export async function generateAlts(path: string, type: string, generateWeb: bool let thumbnail: IImage | null = null; if (['image/jpeg', 'image/webp'].includes(type)) { - thumbnail = await ConvertToJpeg(path, 498, 280); + thumbnail = await convertToJpeg(path, 498, 280); } else if (['image/png'].includes(type)) { - thumbnail = await ConvertToPng(path, 498, 280); + thumbnail = await convertToPng(path, 498, 280); + } else if (['image/gif'].includes(type)) { + thumbnail = await convertToGif(path); } else if (type.startsWith('video/')) { try { thumbnail = await GenerateVideoThumbnail(path); @@ -188,7 +195,15 @@ export async function generateAlts(path: string, type: string, generateWeb: bool * Upload to ObjectStorage */ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string, filename?: string) { - const minio = new Minio.Client(config.drive!.config); + const meta = await fetchMeta(); + + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); const metadata = { 'Content-Type': type, @@ -197,7 +212,7 @@ async function upload(key: string, stream: fs.ReadStream | Buffer, type: string, if (filename) metadata['Content-Disposition'] = contentDisposition('inline', filename); - await minio.putObject(config.drive!.bucket!, key, stream, undefined, metadata); + await minio.putObject(meta.objectStorageBucket!, key, stream, undefined, metadata); } async function deleteOldFile(user: IRemoteUser) { diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index f1280822a..ba0482dbe 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,9 +1,9 @@ import * as Minio from 'minio'; -import config from '../../config'; import { DriveFile } from '../../models/entities/drive-file'; import { InternalStorage } from './internal-storage'; import { DriveFiles, Instances, Notes } from '../../models'; import { driveChart, perUserDriveChart, instanceChart } from '../chart'; +import { fetchMeta } from '../../misc/fetch-meta'; export default async function(file: DriveFile, isExpired = false) { if (file.storedInternal) { @@ -17,16 +17,24 @@ export default async function(file: DriveFile, isExpired = false) { InternalStorage.del(file.webpublicAccessKey!); } } else if (!file.isLink) { - const minio = new Minio.Client(config.drive!.config); + const meta = await fetchMeta(); - await minio.removeObject(config.drive!.bucket!, file.accessKey!); + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); + + await minio.removeObject(meta.objectStorageBucket!, file.accessKey!); if (file.thumbnailUrl) { - await minio.removeObject(config.drive!.bucket!, file.thumbnailAccessKey!); + await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!); } if (file.webpublicUrl) { - await minio.removeObject(config.drive!.bucket!, file.webpublicAccessKey!); + await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!); } } diff --git a/src/services/drive/generate-video-thumbnail.ts b/src/services/drive/generate-video-thumbnail.ts index 5d7efff27..c2646182d 100644 --- a/src/services/drive/generate-video-thumbnail.ts +++ b/src/services/drive/generate-video-thumbnail.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; import * as tmp from 'tmp'; -import { IImage, ConvertToJpeg } from './image-processor'; +import { IImage, convertToJpeg } from './image-processor'; const ThumbnailGenerator = require('video-thumbnail-generator').default; export async function GenerateVideoThumbnail(path: string): Promise { @@ -23,7 +23,7 @@ export async function GenerateVideoThumbnail(path: string): Promise { const outPath = `${outDir}/output.png`; - const thumbnail = await ConvertToJpeg(outPath, 498, 280); + const thumbnail = await convertToJpeg(outPath, 498, 280); // cleanup fs.unlinkSync(outPath); diff --git a/src/services/drive/image-processor.ts b/src/services/drive/image-processor.ts index 89ac331ca..4b8db0e0c 100644 --- a/src/services/drive/image-processor.ts +++ b/src/services/drive/image-processor.ts @@ -1,4 +1,5 @@ import * as sharp from 'sharp'; +import * as fs from 'fs'; export type IImage = { data: Buffer; @@ -10,7 +11,7 @@ export type IImage = { * Convert to JPEG * with resize, remove metadata, resolve orientation, stop animation */ -export async function ConvertToJpeg(path: string, width: number, height: number): Promise { +export async function convertToJpeg(path: string, width: number, height: number): Promise { const data = await sharp(path) .resize(width, height, { fit: 'inside', @@ -34,7 +35,7 @@ export async function ConvertToJpeg(path: string, width: number, height: number) * Convert to WebP * with resize, remove metadata, resolve orientation, stop animation */ -export async function ConvertToWebp(path: string, width: number, height: number): Promise { +export async function convertToWebp(path: string, width: number, height: number): Promise { const data = await sharp(path) .resize(width, height, { fit: 'inside', @@ -57,7 +58,7 @@ export async function ConvertToWebp(path: string, width: number, height: number) * Convert to PNG * with resize, remove metadata, resolve orientation, stop animation */ -export async function ConvertToPng(path: string, width: number, height: number): Promise { +export async function convertToPng(path: string, width: number, height: number): Promise { const data = await sharp(path) .resize(width, height, { fit: 'inside', @@ -73,3 +74,29 @@ export async function ConvertToPng(path: string, width: number, height: number): type: 'image/png' }; } + +/** + * Convert to GIF (Actually just NOP) + */ +export async function convertToGif(path: string): Promise { + const data = await fs.promises.readFile(path); + + return { + data, + ext: 'gif', + type: 'image/gif' + }; +} + +/** + * Convert to APNG (Actually just NOP) + */ +export async function convertToApng(path: string): Promise { + const data = await fs.promises.readFile(path); + + return { + data, + ext: 'apng', + type: 'image/apng' + }; +}