Use PostgreSQL instead of MongoDB (#4572)

* wip

* Update note.ts

* Update timeline.ts

* Update core.ts

* wip

* Update generate-visibility-query.ts

* wip

* wip

* wip

* wip

* wip

* Update global-timeline.ts

* wip

* wip

* wip

* Update vote.ts

* wip

* wip

* Update create.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update files.ts

* wip

* wip

* Update CONTRIBUTING.md

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update read-notification.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Update cancel.ts

* wip

* wip

* wip

* Update show.ts

* wip

* wip

* Update gen-id.ts

* Update create.ts

* Update id.ts

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Docker: Update files about Docker (#4599)

* Docker: Use cache if files used by `yarn install` was not updated

This patch reduces the number of times to installing node_modules.
For example, `yarn install` step will be skipped when only ".config/default.yml" is updated.

* Docker: Migrate MongoDB to Postgresql

Misskey uses Postgresql as a database instead of Mongodb since version 11.

* Docker: Uncomment about data persistence

This patch will save a lot of databases.

* wip

* wip

* wip

* Update activitypub.ts

* wip

* wip

* wip

* Update logs.ts

* wip

* Update drive-file.ts

* Update register.ts

* wip

* wip

* Update mentions.ts

* wip

* wip

* wip

* Update recommendation.ts

* wip

* Update index.ts

* wip

* Update recommendation.ts

* Doc: Update docker.ja.md and docker.en.md (#1) (#4608)

Update how to set up misskey.

* wip

* ✌️

* wip

* Update note.ts

* Update postgre.ts

* wip

* wip

* wip

* wip

* Update add-file.ts

* wip

* wip

* wip

* Clean up

* Update logs.ts

* wip

* 🍕

* wip

* Ad notes

* wip

* Update api-visibility.ts

* Update note.ts

* Update add-file.ts

* tests

* tests

* Update postgre.ts

* Update utils.ts

* wip

* wip

* Refactor

* wip

* Refactor

* wip

* wip

* Update show-users.ts

* Update update-instance.ts

* wip

* Update feed.ts

* Update outbox.ts

* Update outbox.ts

* Update user.ts

* wip

* Update list.ts

* Update update-hashtag.ts

* wip

* Update update-hashtag.ts

* Refactor

* Update update.ts

* wip

* wip

* ✌️

* clean up

* docs

* Update push.ts

* wip

* Update api.ts

* wip

* ✌️

* Update make-pagination-query.ts

* ✌️

* Delete hashtags.ts

* Update instances.ts

* Update instances.ts

* Update create.ts

* Update search.ts

* Update reversi-game.ts

* Update signup.ts

* Update user.ts

* id

* Update example.yml

* 🎨

* objectid

* fix

* reversi

* reversi

* Fix bug of chart engine

* Add test of chart engine

* Improve test

* Better testing

* Improve chart engine

* Refactor

* Add test of chart engine

* Refactor

* Add chart test

* Fix bug

* コミットし忘れ

* Refactoring

* ✌️

* Add tests

* Add test

* Extarct note tests

* Refactor

* 存在しないユーザーにメンションできなくなっていた問題を修正

* Fix bug

* Update update-meta.ts

* Fix bug

* Update mention.vue

* Fix bug

* Update meta.ts

* Update CONTRIBUTING.md

* Fix bug

* Fix bug

* Fix bug

* Clean up

* Clean up

* Update notification.ts

* Clean up

* Add mute tests

* Add test

* Refactor

* Add test

* Fix test

* Refactor

* Refactor

* Add tests

* Update utils.ts

* Update utils.ts

* Fix test

* Update package.json

* Update update.ts

* Update manifest.ts

* Fix bug

* Fix bug

* Add test

* 🎨

* Update endpoint permissions

* Updaye permisison

* Update person.ts

#4299

* データベースと同期しないように

* Fix bug

* Fix bug

* Update reversi-game.ts

* Use a feature of Node v11.7.0 to extract a public key (#4644)

* wip

* wip

* ✌️

* Refactoring

#1540

* test

* test

* test

* test

* test

* test

* test

* Fix bug

* Fix test

* 🍣

* wip

* #4471

* Add test for #4335

* Refactor

* Fix test

* Add tests

* 🕓

* Fix bug

* Add test

* Add test

* rename

* Fix bug

src/misc/aid.ts

@@ -0,0 +1,26 @@
+// AID
+// 長さ8の[2000年1月1日からの経過ミリ秒をbase36でエンコードしたもの] + 長さnの[ランダムな文字列]
+
+const CHARS = '0123456789abcdefghijklmnopqrstuvwxyz';
+const TIME2000 = 946684800000;
+
+function getTime(time: number) {
+	time = time - TIME2000;
+	if (time < 0) time = 0;
+
+	return time.toString(36);
+}
+
+function getRandom(length: number) {
+	let str = '';
+
+	for (let i = 0; i < length; i++) {
+		str += CHARS[Math.floor(Math.random() * CHARS.length)];
+	}
+
+	return str;
+}
+
+export function genAid(date: Date, rand: number): string {
+	return getTime(date.getTime()).padStart(8, CHARS[0]) + getRandom(rand);
+}

src/misc/aidc.ts

@@ -0,0 +1,26 @@
+// AID(Cheep)
+// 長さ6の[2000年1月1日からの経過秒をbase36でエンコードしたもの] + 長さ3の[ランダムな文字列]
+
+const CHARS = '0123456789abcdefghijklmnopqrstuvwxyz';
+const TIME2000 = 946684800000;
+
+function getTime(time: number) {
+	time = time - TIME2000;
+	if (time < 0) time = 0;
+	time = Math.floor(time / 1000);
+	return time.toString(36);
+}
+
+function getRandom() {
+	let str = '';
+
+	for (let i = 0; i < 3; i++) {
+		str += CHARS[Math.floor(Math.random() * CHARS.length)];
+	}
+
+	return str;
+}
+
+export function genAidc(date: Date): string {
+	return getTime(date.getTime()).padStart(6, CHARS[0]) + getRandom();
+}

src/misc/cafy-id.ts

@@ -1,38 +1,13 @@
-import * as mongo from 'mongodb';
 import { Context } from 'cafy';
-import isObjectId from './is-objectid';
 
-export const isAnId = (x: any) => mongo.ObjectID.isValid(x);
-export const isNotAnId = (x: any) => !isAnId(x);
-export const transform = (x: string | mongo.ObjectID): mongo.ObjectID => {
-	if (x === undefined) return undefined;
-	if (x === null) return null;
-
-	if (isAnId(x) && !isObjectId(x)) {
-		return new mongo.ObjectID(x);
-	} else {
-		return x as mongo.ObjectID;
-	}
-};
-export const transformMany = (xs: (string | mongo.ObjectID)[]): mongo.ObjectID[] => {
-	if (xs == null) return null;
-
-	return xs.map(x => transform(x));
-};
-
-export type ObjectId = mongo.ObjectID;
-
-/**
- * ID
- */
-export default class ID<Maybe = string> extends Context<string | Maybe> {
+export class ID<Maybe = string> extends Context<string | (Maybe extends {} ? string : Maybe)> {
 	public readonly name = 'ID';
 
 	constructor(optional = false, nullable = false) {
 		super(optional, nullable);
 
 		this.push((v: any) => {
-			if (!isObjectId(v) && isNotAnId(v)) {
+			if (typeof v !== 'string') {
 				return new Error('must-be-an-id');
 			}
 			return true;

src/misc/check-mongodb.ts

@@ -1,37 +0,0 @@
-import { nativeDbConn } from '../db/mongodb';
-import { Config } from '../config/types';
-import Logger from '../services/logger';
-import { lessThan } from '../prelude/array';
-
-const requiredMongoDBVersion = [3, 6];
-
-export function checkMongoDB(config: Config, logger: Logger) {
-	return new Promise((res, rej) => {
-		const mongoDBLogger = logger.createSubLogger('db');
-		const u = config.mongodb.user ? encodeURIComponent(config.mongodb.user) : null;
-		const p = config.mongodb.pass ? encodeURIComponent(config.mongodb.pass) : null;
-		const uri = `mongodb://${u && p ? `${u}:****@` : ''}${config.mongodb.host}:${config.mongodb.port}/${config.mongodb.db}`;
-		mongoDBLogger.info(`Connecting to ${uri} ...`);
-
-		nativeDbConn().then(db => {
-			mongoDBLogger.succ('Connectivity confirmed');
-
-			db.admin().serverInfo().then(x => {
-				const version = x.version as string;
-				mongoDBLogger.info(`Version: ${version}`);
-				if (lessThan(version.split('.').map(x => parseInt(x, 10)), requiredMongoDBVersion)) {
-					mongoDBLogger.error(`MongoDB version is less than ${requiredMongoDBVersion.join('.')}. Please upgrade it.`);
-					rej('outdated version');
-				} else {
-					res();
-				}
-			}).catch(err => {
-				mongoDBLogger.error(`Failed to fetch server info: ${err.message}`);
-				rej(err);
-			});
-		}).catch(err => {
-			mongoDBLogger.error(err.message);
-			rej(err);
-		});
-	});
-}

src/misc/fetch-meta.ts

@@ -1,32 +1,15 @@
-import Meta, { IMeta } from '../models/meta';
+import { Meta } from '../models/entities/meta';
+import { Metas } from '../models';
+import { genId } from './gen-id';
 
-const defaultMeta: any = {
-	name: 'Misskey',
-	maintainer: {},
-	langs: [],
-	cacheRemoteFiles: true,
-	localDriveCapacityMb: 256,
-	remoteDriveCapacityMb: 8,
-	hidedTags: [],
-	stats: {
-		originalNotesCount: 0,
-		originalUsersCount: 0
-	},
-	maxNoteTextLength: 1000,
-	enableEmojiReaction: true,
-	enableTwitterIntegration: false,
-	enableGithubIntegration: false,
-	enableDiscordIntegration: false,
-	enableExternalUserRecommendation: false,
-	externalUserRecommendationEngine: 'https://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-misskey-api.cgi?{{host}}+{{user}}+{{limit}}+{{offset}}',
-	externalUserRecommendationTimeout: 300000,
-	mascotImageUrl: '/assets/ai.png',
-	errorImageUrl: 'https://ai.misskey.xyz/aiart/yubitun.png',
-	enableServiceWorker: false
-};
-
-export default async function(): Promise<IMeta> {
-	const meta = await Meta.findOne({});
-
-	return Object.assign({}, defaultMeta, meta);
+export default async function(): Promise<Meta> {
+	const meta = await Metas.findOne();
+	if (meta) {
+		return meta;
+	} else {
+		return Metas.save({
+			id: genId(),
+			hiddenTags: []
+		} as Meta);
+	}
 }

src/misc/fetch-proxy-account.ts

@@ -0,0 +1,8 @@
+import fetchMeta from './fetch-meta';
+import { ILocalUser } from '../models/entities/user';
+import { Users } from '../models';
+
+export async function fetchProxyAccount(): Promise<ILocalUser> {
+	const meta = await fetchMeta();
+	return await Users.findOne({ username: meta.proxyAccount, host: null }) as ILocalUser;
+}

src/misc/gen-id.ts

@@ -0,0 +1,22 @@
+import { ulid } from 'ulid';
+import { genAid } from './aid';
+import { genAidc } from './aidc';
+import { genObjectId } from './object-id';
+import config from '../config';
+
+const metohd = config.id.toLowerCase();
+
+export function genId(date?: Date): string {
+	if (!date || (date > new Date())) date = new Date();
+
+	switch (metohd) {
+		case 'aidc': return genAidc(date);
+		case 'aid1': return genAid(date, 1);
+		case 'aid2': return genAid(date, 2);
+		case 'aid3': return genAid(date, 3);
+		case 'aid4': return genAid(date, 4);
+		case 'ulid': return ulid(date.getTime());
+		case 'objectid': return genObjectId(date);
+		default: throw 'unknown id generation method';
+	}
+}

src/misc/get-drive-file-url.ts

@@ -1,31 +0,0 @@
-import { IDriveFile } from '../models/drive-file';
-import config from '../config';
-
-export default function(file: IDriveFile, thumbnail = false): string {
-	if (file == null) return null;
-
-	const isImage = file.contentType && file.contentType.startsWith('image/');
-
-	if (file.metadata.withoutChunks) {
-		if (thumbnail) {
-			return file.metadata.thumbnailUrl || file.metadata.webpublicUrl || (isImage ? file.metadata.url : null);
-		} else {
-			return file.metadata.webpublicUrl || file.metadata.url;
-		}
-	} else {
-		if (thumbnail) {
-			return `${config.driveUrl}/${file._id}?thumbnail`;
-		} else {
-			return `${config.driveUrl}/${file._id}?web`;
-		}
-	}
-}
-
-export function getOriginalUrl(file: IDriveFile) {
-	if (file.metadata && file.metadata.url) {
-		return file.metadata.url;
-	}
-
-	const accessKey = file.metadata ? file.metadata.accessKey : null;
-	return `${config.driveUrl}/${file._id}${accessKey ? '?original=' + accessKey : ''}`;
-}

src/misc/get-notification-summary.ts

@@ -20,7 +20,7 @@ export default function(notification: any): string {
 			return `引用されました:\n${getUserName(notification.user)}「${getNoteSummary(notification.note)}」`;
 		case 'reaction':
 			return `リアクションされました:\n${getUserName(notification.user)} <${getReactionEmoji(notification.reaction)}>「${getNoteSummary(notification.note)}」`;
-		case 'poll_vote':
+		case 'pollVote':
 			return `投票されました:\n${getUserName(notification.user)}「${getNoteSummary(notification.note)}」`;
 		default:
 			return `<不明な通知タイプ: ${notification.type}>`;

src/misc/get-user-name.ts

@@ -1,5 +1,5 @@
-import { IUser } from '../models/user';
+import { User } from '../models/entities/user';
 
-export default function(user: IUser): string {
+export default function(user: User): string {
 	return user.name || user.username;
 }

src/misc/get-user-summary.ts

@@ -1,17 +1,18 @@
-import { IUser, isLocalUser } from '../models/user';
 import getAcct from './acct/render';
 import getUserName from './get-user-name';
+import { User } from '../models/entities/user';
+import { Users } from '../models';
 
 /**
  * ユーザーを表す文字列を取得します。
  * @param user ユーザー
  */
-export default function(user: IUser): string {
+export default function(user: User): string {
 	let string = `${getUserName(user)} (@${getAcct(user)})\n` +
 		`${user.notesCount}投稿、${user.followingCount}フォロー、${user.followersCount}フォロワー\n`;
 
-	if (isLocalUser(user)) {
-		string += `場所: ${user.profile.location}、誕生日: ${user.profile.birthday}\n`;
+	if (Users.isLocalUser(user)) {
+		string += `場所: ${user.location}、誕生日: ${user.birthday}\n`;
 	}
 
 	return string + `「${user.description}」`;

src/misc/is-duplicate-key-value-error.ts

@@ -0,0 +1,3 @@
+export function isDuplicateKeyValueError(e: Error): boolean {
+	return e.message.startsWith('duplicate key value');
+}

src/misc/is-objectid.ts

@@ -1,5 +0,0 @@
-import { ObjectID } from 'mongodb';
-
-export default function(x: any): x is ObjectID {
-	return x && typeof x === 'object' && (x.hasOwnProperty('toHexString') || x.hasOwnProperty('_bsontype'));
-}

src/misc/is-quote.ts

@@ -1,5 +1,5 @@
-import { INote } from '../models/note';
+import { Note } from '../models/entities/note';
 
-export default function(note: INote): boolean {
-	return note.renoteId != null && (note.text != null || note.poll != null || (note.fileIds != null && note.fileIds.length > 0));
+export default function(note: Note): boolean {
+	return note.renoteId != null && (note.text != null || note.hasPoll || (note.fileIds != null && note.fileIds.length > 0));
 }

src/misc/nyaize.ts

@@ -0,0 +1,9 @@
+export function nyaize(text: string): string {
+	return text
+		// ja-JP
+		.replace(/な/g, 'にゃ').replace(/ナ/g, 'ニャ').replace(/ﾅ/g, 'ﾆｬ')
+		// ko-KR
+		.replace(/[나-낳]/g, (match: string) => String.fromCharCode(
+			match.codePointAt(0)  + '냐'.charCodeAt(0) - '나'.charCodeAt(0)
+		));
+}

src/misc/object-id.ts

@@ -0,0 +1,26 @@
+const CHARS = '0123456789abcdef';
+
+function getTime(time: number) {
+	if (time < 0) time = 0;
+	if (time === 0) {
+		return CHARS[0];
+	}
+
+	time = Math.floor(time / 1000);
+
+	return time.toString(16);
+}
+
+function getRandom() {
+	let str = '';
+
+	for (let i = 0; i < 16; i++) {
+		str += CHARS[Math.floor(Math.random() * CHARS.length)];
+	}
+
+	return str;
+}
+
+export function genObjectId(date: Date): string {
+	return getTime(date.getTime()) + getRandom();
+}

src/misc/reaction-lib.ts

@@ -1,6 +1,6 @@
-import Emoji from '../models/emoji';
 import { emojiRegex } from './emoji-regex';
 import fetchMeta from './fetch-meta';
+import { Emojis } from '../models';
 
 const basic10: Record<string, string> = {
 	'👍': 'like',
@@ -49,7 +49,7 @@ export async function toDbReaction(reaction: string, enableEmoji = true): Promis
 
 	const custom = reaction.match(/^:([\w+-]+):$/);
 	if (custom) {
-		const emoji = await Emoji.findOne({
+		const emoji = await Emojis.findOne({
 			host: null,
 			name: custom[1],
 		});

src/misc/should-mute-this-note.ts

@@ -1,20 +1,13 @@
-import * as mongo from 'mongodb';
-import isObjectId from './is-objectid';
-
-function toString(id: any) {
-	return isObjectId(id) ? (id as mongo.ObjectID).toHexString() : id;
-}
-
 export default function(note: any, mutedUserIds: string[]): boolean {
-	if (mutedUserIds.includes(toString(note.userId))) {
+	if (mutedUserIds.includes(note.userId)) {
 		return true;
 	}
 
-	if (note.reply != null && mutedUserIds.includes(toString(note.reply.userId))) {
+	if (note.reply != null && mutedUserIds.includes(note.reply.userId)) {
 		return true;
 	}
 
-	if (note.renote != null && mutedUserIds.includes(toString(note.renote.userId))) {
+	if (note.renote != null && mutedUserIds.includes(note.renote.userId)) {
 		return true;
 	}