Merge branch 'develop' into migration

* migration: adds missing packages (ref:#4808)

* migrate add missing: user

* migrate add missing: file

* migrate add missing: notes

* migrate add validation: note

- remove attachments which are already deleted
  - => (v11 can't hold them, causes error)
- sets attachementTypes by already migrated
- skips note migration when referenced note does not exist

* migrate add validation: vote, favorite, reaction

* migrate: typo

binding.gyp

@@ -0,0 +1,9 @@
+{
+	'targets': [
+		{
+			'target_name': 'crypto_key',
+			'sources': ['src/crypto_key.cc'],
+			'include_dirs': ['<!(node -e "require(\'nan\')")']
+		}
+	]
+}

gulpfile.ts

@@ -49,6 +49,7 @@ gulp.task('build:copy:views', () =>
 
 gulp.task('build:copy', gulp.parallel('build:copy:views', () =>
 	gulp.src([
+		'./build/Release/crypto_key.node',
 		'./src/const.json',
 		'./src/server/web/views/**/*',
 		'./src/**/assets/**/*',

package.json

@@ -12,9 +12,7 @@
 	"scripts": {
 		"start": "node ./index.js",
 		"init": "node ./built/init.js",
-		"ormconfig": "node ./built/ormconfig.js",
-		"migrate": "npm run ormconfig && ts-node ./node_modules/typeorm/cli.js migration:run",
-		"migrateandstart": "npm run migrate && npm run start",
+		"migrate": "node ./built/migrate.js",
 		"build": "webpack && gulp build",
 		"webpack": "webpack",
 		"watch": "webpack --watch",
@@ -68,8 +66,9 @@
 		"@types/lolex": "3.1.1",
 		"@types/minio": "7.0.1",
 		"@types/mocha": "5.2.6",
-		"@types/node": "11.13.8",
-		"@types/nodemailer": "4.6.8",
+		"@types/mongodb": "3.1.26",
+		"@types/node": "11.13.4",
+		"@types/nodemailer": "4.6.7",
 		"@types/nprogress": "0.0.29",
 		"@types/oauth": "0.9.1",
 		"@types/parse5": "5.0.0",
@@ -169,7 +168,10 @@
 		"mocha": "6.1.4",
 		"moji": "0.5.1",
 		"moment": "2.24.0",
+		"mongodb": "3.2.3",
+		"monk": "6.0.6",
 		"ms": "2.1.1",
+		"nan": "2.12.1",
 		"nested-property": "0.0.7",
 		"node-fetch": "2.6.0",
 		"nodemailer": "6.2.1",

src/crypto_key.cc

@@ -0,0 +1,111 @@
+#include <nan.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+#include <openssl/crypto.h>
+#include <openssl/pem.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+
+NAN_METHOD(extractPublic)
+{
+	const auto sourceString = info[0]->ToString(Nan::GetCurrentContext()).ToLocalChecked();
+	if (!sourceString->IsOneByte()) {
+		Nan::ThrowError("Malformed character found");
+		return;
+	}
+
+	size_t sourceLength = sourceString->Length();
+	const auto sourceBuf = new char[sourceLength];
+
+	Nan::DecodeWrite(sourceBuf, sourceLength, sourceString);
+
+	const auto source = BIO_new_mem_buf(sourceBuf, sourceLength);
+	if (source == nullptr) {
+		Nan::ThrowError("Memory allocation failed");
+		delete[] sourceBuf;
+		return;
+	}
+
+	const auto rsa = PEM_read_bio_RSAPrivateKey(source, nullptr, nullptr, nullptr);
+
+	BIO_free(source);
+	delete[] sourceBuf;
+
+	if (rsa == nullptr) {
+		Nan::ThrowError("Decode failed");
+		return;
+	}
+
+	const auto destination = BIO_new(BIO_s_mem());
+	if (destination == nullptr) {
+		Nan::ThrowError("Memory allocation failed");
+		return;
+	}
+
+	const auto result = PEM_write_bio_RSAPublicKey(destination, rsa);
+
+	RSA_free(rsa);
+
+	if (result != 1) {
+		Nan::ThrowError("Public key extraction failed");
+		BIO_free(destination);
+		return;
+	}
+
+	char *pem;
+	const auto pemLength = BIO_get_mem_data(destination, &pem);
+
+	info.GetReturnValue().Set(Nan::Encode(pem, pemLength));
+	BIO_free(destination);
+}
+
+NAN_METHOD(generate)
+{
+	const auto exponent = BN_new();
+	const auto mem = BIO_new(BIO_s_mem());
+	const auto rsa = RSA_new();
+	char *data;
+	long result;
+
+	if (exponent == nullptr || mem == nullptr || rsa == nullptr) {
+		Nan::ThrowError("Memory allocation failed");
+		goto done;
+	}
+
+	result = BN_set_word(exponent, 65537);
+	if (result != 1) {
+		Nan::ThrowError("Exponent setting failed");
+		goto done;
+	}
+
+	result = RSA_generate_key_ex(rsa, 2048, exponent, nullptr);
+	if (result != 1) {
+		Nan::ThrowError("Key generation failed");
+		goto done;
+	}
+
+	result = PEM_write_bio_RSAPrivateKey(mem, rsa, NULL, NULL, 0, NULL, NULL);
+	if (result != 1) {
+		Nan::ThrowError("Key export failed");
+		goto done;
+	}
+
+	result = BIO_get_mem_data(mem, &data);
+	info.GetReturnValue().Set(Nan::Encode(data, result));
+
+done:
+	RSA_free(rsa);
+	BIO_free(mem);
+	BN_free(exponent);
+}
+
+NAN_MODULE_INIT(InitAll)
+{
+	Nan::Set(target, Nan::New<v8::String>("extractPublic").ToLocalChecked(),
+		Nan::GetFunction(Nan::New<v8::FunctionTemplate>(extractPublic)).ToLocalChecked());
+
+	Nan::Set(target, Nan::New<v8::String>("generate").ToLocalChecked(),
+		Nan::GetFunction(Nan::New<v8::FunctionTemplate>(generate)).ToLocalChecked());
+}
+
+NODE_MODULE(crypto_key, InitAll);

src/crypto_key.d.ts

@@ -0,0 +1,2 @@
+export function extractPublic(keypair: string): string;
+export function generate(): string;

src/migrate.ts

@@ -0,0 +1,698 @@
+process.env.NODE_ENV = 'production';
+
+import monk from 'monk';
+import * as mongo from 'mongodb';
+import * as fs from 'fs';
+import * as uuid from 'uuid';
+import chalk from 'chalk';
+import config from './config';
+import { initDb } from './db/postgre';
+import { User } from './models/entities/user';
+import { getRepository } from 'typeorm';
+import generateUserToken from './server/api/common/generate-native-user-token';
+import { DriveFile } from './models/entities/drive-file';
+import { DriveFolder } from './models/entities/drive-folder';
+import { InternalStorage } from './services/drive/internal-storage';
+import { createTemp } from './misc/create-temp';
+import { Note } from './models/entities/note';
+import { Following } from './models/entities/following';
+import { Poll } from './models/entities/poll';
+import { PollVote } from './models/entities/poll-vote';
+import { NoteFavorite } from './models/entities/note-favorite';
+import { NoteReaction } from './models/entities/note-reaction';
+import { UserPublickey } from './models/entities/user-publickey';
+import { UserKeypair } from './models/entities/user-keypair';
+import { extractPublic } from './crypto_key';
+import { Emoji } from './models/entities/emoji';
+import { toPuny as _toPuny } from './misc/convert-host';
+import { UserProfile } from './models/entities/user-profile';
+import { MessagingMessage } from './models/entities/messaging-message';
+import { Muting } from './models/entities/muting';
+import { Blocking } from './models/entities/blocking';
+
+function toPuny(x: string | null): string | null {
+	if (x == null) return null;
+	return _toPuny(x);
+}
+
+const u = (config as any).mongodb.user ? encodeURIComponent((config as any).mongodb.user) : null;
+const p = (config as any).mongodb.pass ? encodeURIComponent((config as any).mongodb.pass) : null;
+
+const uri = `mongodb://${u && p ? `${u}:${p}@` : ''}${(config as any).mongodb.host}:${(config as any).mongodb.port}/${(config as any).mongodb.db}`;
+
+const db = monk(uri);
+let mdb: mongo.Db;
+
+const test = false;
+const limit = 500;
+
+const nativeDbConn = async (): Promise<mongo.Db> => {
+	if (mdb) return mdb;
+
+	const db = await ((): Promise<mongo.Db> => new Promise((resolve, reject) => {
+		mongo.MongoClient.connect(uri, { useNewUrlParser: true }, (e: Error, client: any) => {
+			if (e) return reject(e);
+			resolve(client.db((config as any).mongodb.db));
+		});
+	}))();
+
+	mdb = db;
+
+	return db;
+};
+
+const _User = db.get<any>('users');
+const _DriveFile = db.get<any>('driveFiles.files');
+const _DriveFolder = db.get<any>('driveFolders');
+const _Note = db.get<any>('notes');
+const _Following = db.get<any>('following');
+const _Blocking = db.get<any>('blocking');
+const _Muting = db.get<any>('mute');
+const _PollVote = db.get<any>('pollVotes');
+const _Favorite = db.get<any>('favorites');
+const _NoteReaction = db.get<any>('noteReactions');
+const _Emoji = db.get<any>('emoji');
+const _MessagingMessage = db.get<any>('messagingMessages');
+const getDriveFileBucket = async (): Promise<mongo.GridFSBucket> => {
+	const db = await nativeDbConn();
+	const bucket = new mongo.GridFSBucket(db, {
+		bucketName: 'driveFiles'
+	});
+	return bucket;
+};
+
+const isMigrateRemoteNote = false; // making this true will try to migrate remote notes (possibly could cause errors)
+
+async function main() {
+	await initDb();
+	const Users = getRepository(User);
+	const UserProfiles = getRepository(UserProfile);
+	const DriveFiles = getRepository(DriveFile);
+	const DriveFolders = getRepository(DriveFolder);
+	const Notes = getRepository(Note);
+	const Followings = getRepository(Following);
+	const Blockings = getRepository(Blocking);
+	const Mutings = getRepository(Muting);
+	const Polls = getRepository(Poll);
+	const PollVotes = getRepository(PollVote);
+	const NoteFavorites = getRepository(NoteFavorite);
+	const NoteReactions = getRepository(NoteReaction);
+	const UserPublickeys = getRepository(UserPublickey);
+	const UserKeypairs = getRepository(UserKeypair);
+	const Emojis = getRepository(Emoji);
+	const MessagingMessages = getRepository(MessagingMessage);
+
+	async function validateNoteExistOnMigrated(noteId: string) {
+		if (!isMigrateRemoteNote) {
+			const noteMigrated = await Notes.findOne(noteId);
+
+			if (noteMigrated === undefined) {
+				throw `=> ${chalk.yellow('SKIP')}: referenced note does not exist in migrated notes: ${noteId}`;
+			}
+		}
+	}
+
+	async function migrateUser(user: any) {
+		await Users.save({
+			id: user._id.toHexString(),
+			createdAt: typeof user.createdAt === 'number' ? new Date(user.createdAt) : (user.createdAt || new Date()),
+			updatedAt: typeof user.updatedAt === 'number' ? new Date(user.updatedAt) : (user.updatedAt || null),
+			username: user.username,
+			usernameLower: user.username.toLowerCase(),
+			host: toPuny(user.host),
+			token: generateUserToken(),
+			isAdmin: user.isAdmin || false,
+			name: user.name,
+			followersCount: user.followersCount || 0,
+			followingCount: user.followingCount || 0,
+			notesCount: user.notesCount || 0,
+			isBot: user.isBot || false,
+			isCat: user.isCat || false,
+			isVerified: user.isVerified || false,
+			inbox: user.inbox,
+			sharedInbox: user.sharedInbox,
+			uri: user.uri,
+			emojis: user.emojis || [] as string[],
+			tags: user.tags || [] as string[],
+			isSuspended: user.isSuspended,
+			isSilenced: user.isSilenced,
+			isLocked: user.isLocked || false,
+		});
+
+		const userProfileToSave: any = {
+			userId: user._id.toHexString(),
+			description: user.description,
+			userHost: toPuny(user.host),
+			autoAcceptFollowed: true,
+			autoWatch: false,
+			alwaysMarkNsfw: user.settings ? user.settings.alwaysMarkNsfw : false,
+			password: user.password,
+			location: user.profile ? user.profile.location : null,
+			birthday: user.profile ? user.profile.birthday : null,
+			email: user.email,
+			emailVerified: user.emailVerified || false,
+			emailVerifyCode: user.emailVerifyCode,
+			twoFactorSecret: user.twoFactorSecret,
+			twoFactorEnabled: user.twoFactorEnabled,
+			twoFactorTempSecret: user.twoFactorTempSecret,
+			carefulBot: user.carefulBot
+		};
+
+		if (user.twitter) {
+			userProfileToSave.twitter = true;
+			userProfileToSave.twitterAccessToken = user.twitter.accessToken;
+			userProfileToSave.twitterAccessTokenSecret = user.twitter.accessTokenSecret;
+			userProfileToSave.twitterUserId = user.twitter.userId;
+			userProfileToSave.twitterScreenName = user.twitter.screenName;
+		}
+
+		if (user.github) {
+			userProfileToSave.github = true;
+			userProfileToSave.githubAccessToken = user.github.accessToken;
+			userProfileToSave.githubId = Number(user.github.id);
+			userProfileToSave.githubLogin = user.github.login;
+		}
+
+		if (user.discord) {
+			userProfileToSave.discord = true;
+			userProfileToSave.discordAccessToken = user.discord.accessToken;
+			userProfileToSave.discordrefreshToken = user.discord.refreshToken;
+			userProfileToSave.discordExpiresDate = user.discord.expiresDate; // number.
+			userProfileToSave.discordId = user.discord.id;
+			userProfileToSave.discordUsername = user.discord.username;
+			userProfileToSave.discordDiscriminator = user.discord.discriminator;
+		}
+
+		await UserProfiles.save(userProfileToSave);
+
+		if (user.publicKey) {
+			await UserPublickeys.save({
+				userId: user._id.toHexString(),
+				keyId: user.publicKey.id,
+				keyPem: user.publicKey.publicKeyPem
+			});
+		}
+		if (user.keypair) {
+			await UserKeypairs.save({
+				userId: user._id.toHexString(),
+				publicKey: extractPublic(user.keypair),
+				privateKey: user.keypair,
+			});
+		}
+	}
+
+	async function migrateFollowing(following: any) {
+		await Followings.save({
+			id: following._id.toHexString(),
+			createdAt: new Date(),
+			followerId: following.followerId.toHexString(),
+			followeeId: following.followeeId.toHexString(),
+
+			// 非正規化
+			followerHost: following._follower ? toPuny(following._follower.host) : null,
+			followerInbox: following._follower ? following._follower.inbox : null,
+			followerSharedInbox: following._follower ? following._follower.sharedInbox : null,
+			followeeHost: following._followee ? toPuny(following._followee.host) : null,
+			followeeInbox: following._followee ? following._followee.inbox : null,
+			followeeSharedInbox: following._followee ? following._followee.sharedInbo : null
+		});
+	}
+
+	async function migrateBlocking(blocking: any) {
+		await Blockings.save({
+			id: blocking._id.toHexString(),
+			createdAt: new Date(),
+			blockerId: blocking.blockerId.toHexString(),
+			blockeeId: blocking.blockeeId.toHexString(),
+		});
+	}
+
+	async function migrateMuting(muting: any) {
+		await Mutings.save({
+			id: muting._id.toHexString(),
+			createdAt: new Date(),
+			muterId: muting.muterId.toHexString(),
+			muteeId: muting.muteeId.toHexString(),
+		});
+	}
+
+	async function migrateDriveFolder(folder: any) {
+		await DriveFolders.save({
+			id: folder._id.toHexString(),
+			userId: folder.userId.toHexString(),
+			createdAt: folder.createdAt || new Date(),
+			name: folder.name,
+			parentId: folder.parentId ? folder.parentId.toHexString() : null,
+		});
+	}
+
+	async function migrateDriveFile(file: any) {
+		const user = await _User.findOne({
+			_id: file.metadata.userId
+		});
+		if (user == null) return;
+
+		const fileToSave: any = {
+			id: file._id.toHexString(),
+			userId: user._id.toHexString(),
+			userHost: toPuny(user.host),
+			createdAt: file.uploadDate || new Date(),
+			md5: file.md5,
+			name: file.filename,
+			type: file.contentType,
+			properties: file.metadata.properties || {},
+			size: file.length,
+			// url: [different],
+			uri: file.metadata.uri,
+			// accessKey: [different],
+			folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null,
+			// storedInternal: [different],
+			// isLink: [different],
+			isSensitive: file.metadata.isSensitive === true,
+			comment: file.metadata.comment && (file.metadata.comment.length > 0) && file.metadata.comment || null,
+			thumbnailUrl: file.metadata.thumbnailUrl,
+			thumbnailAccessKey: file.metadata.storageProps && file.metadata.storageProps.thumbnailAccessKey || null,
+			webpublicUrl: file.metadata.webpublicUrl,
+			webpublicAccessKey: file.metadata.storageProps && file.metadata.storageProps.webpublicAccessKey || null,
+		};
+
+		if (file.metadata.storageProps && file.metadata.storageProps.key) { // when object storage
+			fileToSave.url = file.metadata.url;
+			fileToSave.accessKey = file.metadata.storageProps.key;
+			fileToSave.storedInternal = false;
+			fileToSave.isLink = false;
+
+			await DriveFiles.save(fileToSave);
+		} else if (!file.metadata.isLink) {
+			const [temp, clean] = await createTemp();
+			await new Promise(async (res, rej) => {
+				const bucket = await getDriveFileBucket();
+				const readable = bucket.openDownloadStream(file._id);
+				const dest = fs.createWriteStream(temp);
+				readable.pipe(dest);
+				readable.on('end', () => {
+					dest.end();
+					res();
+				});
+			});
+
+			const key = uuid.v4();
+			const url = InternalStorage.saveFromPath(key, temp);
+
+			fileToSave.url = url;
+			fileToSave.accessKey = key;
+			fileToSave.storedInternal = true;
+			fileToSave.isLink = false;
+
+			await DriveFiles.save(fileToSave);
+			clean();
+		} else {
+			fileToSave.url = file.metadata.url;
+			fileToSave.accessKey = null;
+			fileToSave.storedInternal = false;
+			fileToSave.isLink = true;
+
+			await DriveFiles.save(fileToSave);
+		}
+	}
+
+	async function migrateNote(note: any) {
+		const noteToSave = {
+			id: note._id.toHexString(),
+			createdAt: note.createdAt || new Date(),
+			text: note.text,
+			cw: note.cw || null,
+			tags: note.tags || [],
+			userId: note.userId.toHexString(),
+			viaMobile: note.viaMobile || false,
+			geo: note.geo,
+			appId: null,
+			visibility: note.visibility && (note.visibility === 'private' ? 'specified' : note.visibility) || 'public', // there is no 'private' visibility more.
+			visibleUserIds: note.visibleUserIds ? note.visibleUserIds.map((id: any) => id.toHexString()) : [],
+			replyId: note.replyId ? note.replyId.toHexString() : null,
+			renoteId: note.renoteId ? note.renoteId.toHexString() : null,
+			userHost: null,
+			fileIds: note.fileIds ? note.fileIds.map((id: any) => id.toHexString()) : [],
+			attachedFileTypes: ([] as string[]), // see below
+			localOnly: note.localOnly || false,
+			hasPoll: note.poll != null,
+			name: note.name && (note.name.length > 0) && note.name || null,
+			emojis: note.emojis || ([] as string[]),
+			renoteCount: note.renoteCount || 0,
+			repliesCount: note.repliesCount || 0,
+			mentions: note.mentions && note.mentions.map((id: any) => id.toHexString()) || [],
+			mentionedRemoteUsers: note.mentionedRemoteUsers && JSON.stringify(note.mentionedRemoteUsers) || '[]',
+			score: note.score || 0,
+			uri: note.uri || null
+		};
+
+		// validate existance of referenced notes (on migrated)
+		if ((!isMigrateRemoteNote) && (noteToSave.replyId !== null || noteToSave.renoteId !== null)) {
+			// skip when reply does not exist on local
+			if (noteToSave.replyId !== null) {
+				const mongoReplyNoteLocal = await _Note.findOne({
+					'_user.host': null,
+					'_id': note.replyId
+				});
+
+				if (mongoReplyNoteLocal === null) {
+					throw `=> ${chalk.yellow('SKIP')}: referenced "local" reply note does not exist: ${note.replyId}`;
+				}
+			}
+
+			// skip when reply does not exist on local
+			if (noteToSave.renoteId !== null) {
+				const mongoRenoteNoteLocal = await _Note.findOne({
+					'_user.host': null,
+					'_id': note.renoteId
+				});
+
+				if (mongoRenoteNoteLocal === null) {
+					throw `=> ${chalk.yellow('SKIP')}: referenced "local" renote note does not exist: ${note.renoteId}`;
+				}
+			}
+		}
+
+		if (noteToSave.fileIds.length !== 0) {
+			const filesMigrated = await DriveFiles.findByIds(noteToSave.fileIds);
+
+			// remove attachments which user removed after creating note
+			if (noteToSave.fileIds.length !== filesMigrated.length) {
+				console.warn(`NOTE ${noteToSave.id} ${chalk.yellow('MODIFIED')}: file count is different: before: ${noteToSave.fileIds.length} => after: ${filesMigrated.length}`);
+				noteToSave.fileIds = filesMigrated.map(file => file.id) || [];
+			}
+
+			noteToSave.attachedFileTypes = filesMigrated.map(file => file.type);
+		}
+
+		await Notes.save(noteToSave);
+
+		if (note.poll) {
+			await Polls.save({
+				noteId: note._id.toHexString(),
+				choices: note.poll.choices.map((x: any) => x.text),
+				expiresAt: note.poll.expiresAt,
+				multiple: note.poll.multiple || false,
+				votes: note.poll.choices.map((x: any) => x.votes),
+				noteVisibility: note.visibility && (note.visibility === 'private' ? 'specified' : note.visibility) || 'public', // there is no 'private' visibility more.
+				userId: note.userId.toHexString(),
+				userHost: null
+			});
+		}
+	}
+
+	async function migratePollVote(vote: any) {
+		const voteToSave = {
+			id: vote._id.toHexString(),
+			createdAt: vote.createdAt,
+			noteId: vote.noteId.toHexString(),
+			userId: vote.userId.toHexString(),
+			choice: vote.choice
+		};
+
+		await validateNoteExistOnMigrated(voteToSave.noteId);
+
+		await PollVotes.save(voteToSave);
+	}
+
+	async function migrateNoteFavorite(favorite: any) {
+		const favoriteToSave = {
+			id: favorite._id.toHexString(),
+			createdAt: favorite.createdAt,
+			noteId: favorite.noteId.toHexString(),
+			userId: favorite.userId.toHexString(),
+		};
+
+		await validateNoteExistOnMigrated(favoriteToSave.noteId);
+
+		await NoteFavorites.save(favoriteToSave);
+	}
+
+	async function migrateNoteReaction(reaction: any) {
+		const reactionToSave = {
+			id: reaction._id.toHexString(),
+			createdAt: reaction.createdAt,
+			noteId: reaction.noteId.toHexString(),
+			userId: reaction.userId.toHexString(),
+			reaction: reaction.reaction
+		};
+
+		await validateNoteExistOnMigrated(reactionToSave.noteId);
+
+		await NoteReactions.save(reactionToSave);
+	}
+
+	async function reMigrateUser(user: any) {
+		const u = await _User.findOne({
+			_id: new mongo.ObjectId(user.id)
+		});
+		const avatar = u.avatarId ? await DriveFiles.findOne(u.avatarId.toHexString()) : null;
+		const banner = u.bannerId ? await DriveFiles.findOne(u.bannerId.toHexString()) : null;
+		await Users.update(user.id, {
+			avatarId: avatar ? avatar.id : null,
+			bannerId: banner ? banner.id : null,
+			avatarUrl: avatar ? avatar.url : null,
+			bannerUrl: banner ? banner.url : null
+		});
+	}
+
+	async function migrateEmoji(emoji: any) {
+		await Emojis.save({
+			id: emoji._id.toHexString(),
+			updatedAt: emoji.createdAt,
+			aliases: emoji.aliases,
+			url: emoji.url,
+			uri: emoji.uri,
+			host: toPuny(emoji.host),
+			name: emoji.name
+		});
+	}
+
+	async function migrateMessagingMessage(message: any) {
+		await MessagingMessages.save({
+			id: message._id.toHexString(),
+			createdAt: message.createdAt,
+			text: message.text,
+			userId: message.userId.toHexString(),
+			recipientId: message.recipientId.toHexString(),
+			fileId: message.fileId ? message.fileId.toHexString() : null,
+			isRead: message.isRead || false,
+		});
+	}
+
+	let allUsersCount = await _User.count({
+		deletedAt: { $exists: false }
+	});
+	if (test && allUsersCount > limit) allUsersCount = limit;
+	for (let i = 0; i < allUsersCount; i++) {
+		const user = await _User.findOne({
+			deletedAt: { $exists: false }
+		}, {
+			skip: i
+		});
+		try {
+			await migrateUser(user);
+			console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allFollowingsCount = await _Following.count();
+	if (test && allFollowingsCount > limit) allFollowingsCount = limit;
+	for (let i = 0; i < allFollowingsCount; i++) {
+		const following = await _Following.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateFollowing(following);
+			console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allBlockingsCount = await _Blocking.count();
+	if (test && allBlockingsCount > limit) allBlockingsCount = limit;
+	for (let i = 0; i < allBlockingsCount; i++) {
+		const blocking = await _Blocking.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateBlocking(blocking);
+			console.log(`BLOCKING (${i + 1}/${allBlockingsCount}) ${blocking._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`BLOCKING (${i + 1}/${allBlockingsCount}) ${blocking._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allMutingsCount = await _Muting.count();
+	if (test && allMutingsCount > limit) allMutingsCount = limit;
+	for (let i = 0; i < allMutingsCount; i++) {
+		const muting = await _Muting.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateMuting(muting);
+			console.log(`MUTING (${i + 1}/${allMutingsCount}) ${muting._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`MUTING (${i + 1}/${allMutingsCount}) ${muting._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allDriveFoldersCount = await _DriveFolder.count();
+	if (test && allDriveFoldersCount > limit) allDriveFoldersCount = limit;
+	for (let i = 0; i < allDriveFoldersCount; i++) {
+		const folder = await _DriveFolder.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateDriveFolder(folder);
+			console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allDriveFilesCount = await _DriveFile.count({
+		'metadata._user.host': null,
+		'metadata.deletedAt': { $exists: false }
+	});
+	if (test && allDriveFilesCount > limit) allDriveFilesCount = limit;
+	for (let i = 0; i < allDriveFilesCount; i++) {
+		const file = await _DriveFile.findOne({
+			'metadata._user.host': null,
+			'metadata.deletedAt': { $exists: false }
+		}, {
+			skip: i
+		});
+		try {
+			await migrateDriveFile(file);
+			console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	const noteCondition = {
+		'_user.host': null,
+		'metadata.deletedAt': { $exists: false }
+	};
+	if (isMigrateRemoteNote) {
+		delete noteCondition['_user.host'];
+	}
+
+	let allNotesCount = await _Note.count(noteCondition);
+	if (test && allNotesCount > limit) allNotesCount = limit;
+	for (let i = 0; i < allNotesCount; i++) {
+		const note = await _Note.findOne(noteCondition, {
+			skip: i
+		});
+		try {
+			await migrateNote(note);
+			console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allPollVotesCount = await _PollVote.count();
+	if (test && allPollVotesCount > limit) allPollVotesCount = limit;
+	for (let i = 0; i < allPollVotesCount; i++) {
+		const vote = await _PollVote.findOne({}, {
+			skip: i
+		});
+		try {
+			await migratePollVote(vote);
+			console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allNoteFavoritesCount = await _Favorite.count();
+	if (test && allNoteFavoritesCount > limit) allNoteFavoritesCount = limit;
+	for (let i = 0; i < allNoteFavoritesCount; i++) {
+		const favorite = await _Favorite.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateNoteFavorite(favorite);
+			console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allNoteReactionsCount = await _NoteReaction.count();
+	if (test && allNoteReactionsCount > limit) allNoteReactionsCount = limit;
+	for (let i = 0; i < allNoteReactionsCount; i++) {
+		const reaction = await _NoteReaction.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateNoteReaction(reaction);
+			console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	let allActualUsersCount = await Users.count();
+	if (test && allActualUsersCount > limit) allActualUsersCount = limit;
+	for (let i = 0; i < allActualUsersCount; i++) {
+		const [user] = await Users.find({
+			take: 1,
+			skip: i
+		});
+		try {
+			await reMigrateUser(user);
+			console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	const allEmojisCount = await _Emoji.count();
+	for (let i = 0; i < allEmojisCount; i++) {
+		const emoji = await _Emoji.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateEmoji(emoji);
+			console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	const allMessagingMessagesCount = await _MessagingMessage.count();
+	for (let i = 0; i < allMessagingMessagesCount; i++) {
+		const message = await _MessagingMessage.findOne({}, {
+			skip: i
+		});
+		try {
+			await migrateMessagingMessage(message);
+			console.log(`MESSAGE (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.green('DONE')}`);
+		} catch (e) {
+			console.log(`MESSAGE (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.red('ERR')}`);
+			console.error(e);
+		}
+	}
+
+	console.log('DONE :)');
+}
+
+main();