From 9c454e59450fe7850175a5c117d2f337aaa23141 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 20 Apr 2019 01:23:48 +0900 Subject: [PATCH 1/8] Revert "Clean up" This reverts commit a1f7b009810cd94c3d43752ae035d4ff123dd4de. --- binding.gyp | 9 + gulpfile.ts | 1 + package.json | 2 + src/crypto_key.cc | 111 ++++++++++ src/crypto_key.d.ts | 2 + src/migrate.ts | 502 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 627 insertions(+) create mode 100644 binding.gyp create mode 100644 src/crypto_key.cc create mode 100644 src/crypto_key.d.ts create mode 100644 src/migrate.ts diff --git a/binding.gyp b/binding.gyp new file mode 100644 index 000000000..0349526d5 --- /dev/null +++ b/binding.gyp @@ -0,0 +1,9 @@ +{ + 'targets': [ + { + 'target_name': 'crypto_key', + 'sources': ['src/crypto_key.cc'], + 'include_dirs': [' gulp.task('build:copy', gulp.parallel('build:copy:views', () => gulp.src([ + './build/Release/crypto_key.node', './src/const.json', './src/server/web/views/**/*', './src/**/assets/**/*', diff --git a/package.json b/package.json index 409bdda15..1584899c2 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "scripts": { "start": "node ./index.js", "init": "node ./built/init.js", + "migrate": "node ./built/migrate.js", "build": "webpack && gulp build", "webpack": "webpack", "watch": "webpack --watch", @@ -166,6 +167,7 @@ "moji": "0.5.1", "moment": "2.24.0", "ms": "2.1.1", + "nan": "2.12.1", "nested-property": "0.0.7", "node-fetch": "2.3.0", "nodemailer": "6.1.0", diff --git a/src/crypto_key.cc b/src/crypto_key.cc new file mode 100644 index 000000000..658586bae --- /dev/null +++ b/src/crypto_key.cc @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include +#include +#include + +NAN_METHOD(extractPublic) +{ + const auto sourceString = info[0]->ToString(Nan::GetCurrentContext()).ToLocalChecked(); + if (!sourceString->IsOneByte()) { + Nan::ThrowError("Malformed character found"); + return; + } + + size_t sourceLength = sourceString->Length(); + const auto sourceBuf = new char[sourceLength]; + + Nan::DecodeWrite(sourceBuf, sourceLength, sourceString); + + const auto source = BIO_new_mem_buf(sourceBuf, sourceLength); + if (source == nullptr) { + Nan::ThrowError("Memory allocation failed"); + delete[] sourceBuf; + return; + } + + const auto rsa = PEM_read_bio_RSAPrivateKey(source, nullptr, nullptr, nullptr); + + BIO_free(source); + delete[] sourceBuf; + + if (rsa == nullptr) { + Nan::ThrowError("Decode failed"); + return; + } + + const auto destination = BIO_new(BIO_s_mem()); + if (destination == nullptr) { + Nan::ThrowError("Memory allocation failed"); + return; + } + + const auto result = PEM_write_bio_RSAPublicKey(destination, rsa); + + RSA_free(rsa); + + if (result != 1) { + Nan::ThrowError("Public key extraction failed"); + BIO_free(destination); + return; + } + + char *pem; + const auto pemLength = BIO_get_mem_data(destination, &pem); + + info.GetReturnValue().Set(Nan::Encode(pem, pemLength)); + BIO_free(destination); +} + +NAN_METHOD(generate) +{ + const auto exponent = BN_new(); + const auto mem = BIO_new(BIO_s_mem()); + const auto rsa = RSA_new(); + char *data; + long result; + + if (exponent == nullptr || mem == nullptr || rsa == nullptr) { + Nan::ThrowError("Memory allocation failed"); + goto done; + } + + result = BN_set_word(exponent, 65537); + if (result != 1) { + Nan::ThrowError("Exponent setting failed"); + goto done; + } + + result = RSA_generate_key_ex(rsa, 2048, exponent, nullptr); + if (result != 1) { + Nan::ThrowError("Key generation failed"); + goto done; + } + + result = PEM_write_bio_RSAPrivateKey(mem, rsa, NULL, NULL, 0, NULL, NULL); + if (result != 1) { + Nan::ThrowError("Key export failed"); + goto done; + } + + result = BIO_get_mem_data(mem, &data); + info.GetReturnValue().Set(Nan::Encode(data, result)); + +done: + RSA_free(rsa); + BIO_free(mem); + BN_free(exponent); +} + +NAN_MODULE_INIT(InitAll) +{ + Nan::Set(target, Nan::New("extractPublic").ToLocalChecked(), + Nan::GetFunction(Nan::New(extractPublic)).ToLocalChecked()); + + Nan::Set(target, Nan::New("generate").ToLocalChecked(), + Nan::GetFunction(Nan::New(generate)).ToLocalChecked()); +} + +NODE_MODULE(crypto_key, InitAll); diff --git a/src/crypto_key.d.ts b/src/crypto_key.d.ts new file mode 100644 index 000000000..9aa81a687 --- /dev/null +++ b/src/crypto_key.d.ts @@ -0,0 +1,2 @@ +export function extractPublic(keypair: string): string; +export function generate(): string; diff --git a/src/migrate.ts b/src/migrate.ts new file mode 100644 index 000000000..18217f408 --- /dev/null +++ b/src/migrate.ts @@ -0,0 +1,502 @@ +process.env.NODE_ENV = 'production'; + +import monk from 'monk'; +import * as mongo from 'mongodb'; +import * as fs from 'fs'; +import * as uuid from 'uuid'; +import chalk from 'chalk'; +import config from './config'; +import { initDb } from './db/postgre'; +import { User } from './models/entities/user'; +import { getRepository } from 'typeorm'; +import generateUserToken from './server/api/common/generate-native-user-token'; +import { DriveFile } from './models/entities/drive-file'; +import { DriveFolder } from './models/entities/drive-folder'; +import { InternalStorage } from './services/drive/internal-storage'; +import { createTemp } from './misc/create-temp'; +import { Note } from './models/entities/note'; +import { Following } from './models/entities/following'; +import { Poll } from './models/entities/poll'; +import { PollVote } from './models/entities/poll-vote'; +import { NoteFavorite } from './models/entities/note-favorite'; +import { NoteReaction } from './models/entities/note-reaction'; +import { UserPublickey } from './models/entities/user-publickey'; +import { UserKeypair } from './models/entities/user-keypair'; +import { extractPublic } from './crypto_key'; +import { Emoji } from './models/entities/emoji'; +import { toPuny as _toPuny } from './misc/convert-host'; +import { UserProfile } from './models/entities/user-profile'; + +function toPuny(x: string | null): string | null { + if (x == null) return null; + return _toPuny(x); +} + +const u = (config as any).mongodb.user ? encodeURIComponent((config as any).mongodb.user) : null; +const p = (config as any).mongodb.pass ? encodeURIComponent((config as any).mongodb.pass) : null; + +const uri = `mongodb://${u && p ? `${u}:${p}@` : ''}${(config as any).mongodb.host}:${(config as any).mongodb.port}/${(config as any).mongodb.db}`; + +const db = monk(uri); +let mdb: mongo.Db; + +const test = false; +const limit = 500; + +const nativeDbConn = async (): Promise => { + if (mdb) return mdb; + + const db = await ((): Promise => new Promise((resolve, reject) => { + mongo.MongoClient.connect(uri, { useNewUrlParser: true }, (e: Error, client: any) => { + if (e) return reject(e); + resolve(client.db((config as any).mongodb.db)); + }); + }))(); + + mdb = db; + + return db; +}; + +const _User = db.get('users'); +const _DriveFile = db.get('driveFiles.files'); +const _DriveFolder = db.get('driveFolders'); +const _Note = db.get('notes'); +const _Following = db.get('following'); +const _PollVote = db.get('pollVotes'); +const _Favorite = db.get('favorites'); +const _NoteReaction = db.get('noteReactions'); +const _Emoji = db.get('emoji'); +const getDriveFileBucket = async (): Promise => { + const db = await nativeDbConn(); + const bucket = new mongo.GridFSBucket(db, { + bucketName: 'driveFiles' + }); + return bucket; +}; + +async function main() { + await initDb(); + const Users = getRepository(User); + const UserProfiles = getRepository(UserProfile); + const DriveFiles = getRepository(DriveFile); + const DriveFolders = getRepository(DriveFolder); + const Notes = getRepository(Note); + const Followings = getRepository(Following); + const Polls = getRepository(Poll); + const PollVotes = getRepository(PollVote); + const NoteFavorites = getRepository(NoteFavorite); + const NoteReactions = getRepository(NoteReaction); + const UserPublickeys = getRepository(UserPublickey); + const UserKeypairs = getRepository(UserKeypair); + const Emojis = getRepository(Emoji); + + async function migrateUser(user: any) { + await Users.save({ + id: user._id.toHexString(), + createdAt: user.createdAt || new Date(), + username: user.username, + usernameLower: user.username.toLowerCase(), + host: toPuny(user.host), + token: generateUserToken(), + isAdmin: user.isAdmin || false, + name: user.name, + followersCount: user.followersCount || 0, + followingCount: user.followingCount || 0, + notesCount: user.notesCount || 0, + isBot: user.isBot || false, + isCat: user.isCat || false, + isVerified: user.isVerified || false, + inbox: user.inbox, + sharedInbox: user.sharedInbox, + uri: user.uri, + }); + await UserProfiles.save({ + userId: user._id.toHexString(), + description: user.description, + userHost: toPuny(user.host), + autoAcceptFollowed: true, + autoWatch: false, + password: user.password, + location: user.profile ? user.profile.location : null, + birthday: user.profile ? user.profile.birthday : null, + }); + if (user.publicKey) { + await UserPublickeys.save({ + userId: user._id.toHexString(), + keyId: user.publicKey.id, + keyPem: user.publicKey.publicKeyPem + }); + } + if (user.keypair) { + await UserKeypairs.save({ + userId: user._id.toHexString(), + publicKey: extractPublic(user.keypair), + privateKey: user.keypair, + }); + } + } + + async function migrateFollowing(following: any) { + await Followings.save({ + id: following._id.toHexString(), + createdAt: new Date(), + followerId: following.followerId.toHexString(), + followeeId: following.followeeId.toHexString(), + + // 非正規化 + followerHost: following._follower ? toPuny(following._follower.host) : null, + followerInbox: following._follower ? following._follower.inbox : null, + followerSharedInbox: following._follower ? following._follower.sharedInbox : null, + followeeHost: following._followee ? toPuny(following._followee.host) : null, + followeeInbox: following._followee ? following._followee.inbox : null, + followeeSharedInbox: following._followee ? following._followee.sharedInbo : null + }); + } + + async function migrateDriveFolder(folder: any) { + await DriveFolders.save({ + id: folder._id.toHexString(), + createdAt: folder.createdAt || new Date(), + name: folder.name, + parentId: folder.parentId ? folder.parentId.toHexString() : null, + }); + } + + async function migrateDriveFile(file: any) { + const user = await _User.findOne({ + _id: file.metadata.userId + }); + if (user == null) return; + if (file.metadata.storage && file.metadata.storage.key) { // when object storage + await DriveFiles.save({ + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties || {}, + size: file.length, + url: file.metadata.url, + uri: file.metadata.uri, + accessKey: file.metadata.storage.key, + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + storedInternal: false, + isLink: false + }); + } else if (!file.metadata.isLink) { + const [temp, clean] = await createTemp(); + await new Promise(async (res, rej) => { + const bucket = await getDriveFileBucket(); + const readable = bucket.openDownloadStream(file._id); + const dest = fs.createWriteStream(temp); + readable.pipe(dest); + readable.on('end', () => { + dest.end(); + res(); + }); + }); + + const key = uuid.v4(); + const url = InternalStorage.saveFromPath(key, temp); + await DriveFiles.save({ + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties, + size: file.length, + url: url, + uri: file.metadata.uri, + accessKey: key, + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + storedInternal: true, + isLink: false + }); + clean(); + } else { + await DriveFiles.save({ + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties, + size: file.length, + url: file.metadata.url, + uri: file.metadata.uri, + accessKey: null, + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + storedInternal: false, + isLink: true + }); + } + } + + async function migrateNote(note: any) { + await Notes.save({ + id: note._id.toHexString(), + createdAt: note.createdAt || new Date(), + text: note.text, + cw: note.cw || null, + tags: note.tags || [], + userId: note.userId.toHexString(), + viaMobile: note.viaMobile || false, + geo: note.geo, + appId: null, + visibility: note.visibility || 'public', + visibleUserIds: note.visibleUserIds ? note.visibleUserIds.map((id: any) => id.toHexString()) : [], + replyId: note.replyId ? note.replyId.toHexString() : null, + renoteId: note.renoteId ? note.renoteId.toHexString() : null, + userHost: null, + fileIds: note.fileIds ? note.fileIds.map((id: any) => id.toHexString()) : [], + localOnly: note.localOnly || false, + hasPoll: note.poll != null + }); + + if (note.poll) { + await Polls.save({ + noteId: note._id.toHexString(), + choices: note.poll.choices.map((x: any) => x.text), + expiresAt: note.poll.expiresAt, + multiple: note.poll.multiple, + votes: note.poll.choices.map((x: any) => x.votes), + noteVisibility: note.visibility, + userId: note.userId.toHexString(), + userHost: null + }); + } + } + + async function migratePollVote(vote: any) { + await PollVotes.save({ + id: vote._id.toHexString(), + createdAt: vote.createdAt, + noteId: vote.noteId.toHexString(), + userId: vote.userId.toHexString(), + choice: vote.choice + }); + } + + async function migrateNoteFavorite(favorite: any) { + await NoteFavorites.save({ + id: favorite._id.toHexString(), + createdAt: favorite.createdAt, + noteId: favorite.noteId.toHexString(), + userId: favorite.userId.toHexString(), + }); + } + + async function migrateNoteReaction(reaction: any) { + await NoteReactions.save({ + id: reaction._id.toHexString(), + createdAt: reaction.createdAt, + noteId: reaction.noteId.toHexString(), + userId: reaction.userId.toHexString(), + reaction: reaction.reaction + }); + } + + async function reMigrateUser(user: any) { + const u = await _User.findOne({ + _id: new mongo.ObjectId(user.id) + }); + const avatar = u.avatarId ? await DriveFiles.findOne(u.avatarId.toHexString()) : null; + const banner = u.bannerId ? await DriveFiles.findOne(u.bannerId.toHexString()) : null; + await Users.update(user.id, { + avatarId: avatar ? avatar.id : null, + bannerId: banner ? banner.id : null, + avatarUrl: avatar ? avatar.url : null, + bannerUrl: banner ? banner.url : null + }); + } + + async function migrateEmoji(emoji: any) { + await Emojis.save({ + id: emoji._id.toHexString(), + updatedAt: emoji.createdAt, + aliases: emoji.aliases, + url: emoji.url, + uri: emoji.uri, + host: toPuny(emoji.host), + name: emoji.name + }); + } + + let allUsersCount = await _User.count({ + deletedAt: { $exists: false } + }); + if (test && allUsersCount > limit) allUsersCount = limit; + for (let i = 0; i < allUsersCount; i++) { + const user = await _User.findOne({ + deletedAt: { $exists: false } + }, { + skip: i + }); + try { + await migrateUser(user); + console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`USER (${i + 1}/${allUsersCount}) ${user._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allFollowingsCount = await _Following.count(); + if (test && allFollowingsCount > limit) allFollowingsCount = limit; + for (let i = 0; i < allFollowingsCount; i++) { + const following = await _Following.findOne({}, { + skip: i + }); + try { + await migrateFollowing(following); + console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FOLLOWING (${i + 1}/${allFollowingsCount}) ${following._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allDriveFoldersCount = await _DriveFolder.count(); + if (test && allDriveFoldersCount > limit) allDriveFoldersCount = limit; + for (let i = 0; i < allDriveFoldersCount; i++) { + const folder = await _DriveFolder.findOne({}, { + skip: i + }); + try { + await migrateDriveFolder(folder); + console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FOLDER (${i + 1}/${allDriveFoldersCount}) ${folder._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allDriveFilesCount = await _DriveFile.count({ + 'metadata._user.host': null, + 'metadata.deletedAt': { $exists: false } + }); + if (test && allDriveFilesCount > limit) allDriveFilesCount = limit; + for (let i = 0; i < allDriveFilesCount; i++) { + const file = await _DriveFile.findOne({ + 'metadata._user.host': null, + 'metadata.deletedAt': { $exists: false } + }, { + skip: i + }); + try { + await migrateDriveFile(file); + console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FILE (${i + 1}/${allDriveFilesCount}) ${file._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allNotesCount = await _Note.count({ + '_user.host': null, + 'metadata.deletedAt': { $exists: false } + }); + if (test && allNotesCount > limit) allNotesCount = limit; + for (let i = 0; i < allNotesCount; i++) { + const note = await _Note.findOne({ + '_user.host': null, + 'metadata.deletedAt': { $exists: false } + }, { + skip: i + }); + try { + await migrateNote(note); + console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`NOTE (${i + 1}/${allNotesCount}) ${note._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allPollVotesCount = await _PollVote.count(); + if (test && allPollVotesCount > limit) allPollVotesCount = limit; + for (let i = 0; i < allPollVotesCount; i++) { + const vote = await _PollVote.findOne({}, { + skip: i + }); + try { + await migratePollVote(vote); + console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`VOTE (${i + 1}/${allPollVotesCount}) ${vote._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allNoteFavoritesCount = await _Favorite.count(); + if (test && allNoteFavoritesCount > limit) allNoteFavoritesCount = limit; + for (let i = 0; i < allNoteFavoritesCount; i++) { + const favorite = await _Favorite.findOne({}, { + skip: i + }); + try { + await migrateNoteFavorite(favorite); + console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`FAVORITE (${i + 1}/${allNoteFavoritesCount}) ${favorite._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allNoteReactionsCount = await _NoteReaction.count(); + if (test && allNoteReactionsCount > limit) allNoteReactionsCount = limit; + for (let i = 0; i < allNoteReactionsCount; i++) { + const reaction = await _NoteReaction.findOne({}, { + skip: i + }); + try { + await migrateNoteReaction(reaction); + console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`REACTION (${i + 1}/${allNoteReactionsCount}) ${reaction._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allActualUsersCount = await Users.count(); + if (test && allActualUsersCount > limit) allActualUsersCount = limit; + for (let i = 0; i < allActualUsersCount; i++) { + const [user] = await Users.find({ + take: 1, + skip: i + }); + try { + await reMigrateUser(user); + console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`RE:USER (${i + 1}/${allActualUsersCount}) ${user.id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + const allEmojisCount = await _Emoji.count(); + for (let i = 0; i < allEmojisCount; i++) { + const emoji = await _Emoji.findOne({}, { + skip: i + }); + try { + await migrateEmoji(emoji); + console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`EMOJI (${i + 1}/${allEmojisCount}) ${emoji._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + console.log('DONE :)'); +} + +main(); From bbfde2e461faf7653cc59baa8556124dfcfc68e3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 20 Apr 2019 01:32:43 +0900 Subject: [PATCH 2/8] Update migrate.ts --- src/migrate.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/migrate.ts b/src/migrate.ts index 18217f408..53b681fbe 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -26,6 +26,7 @@ import { extractPublic } from './crypto_key'; import { Emoji } from './models/entities/emoji'; import { toPuny as _toPuny } from './misc/convert-host'; import { UserProfile } from './models/entities/user-profile'; +import { MessagingMessage } from './models/entities/messaging-message'; function toPuny(x: string | null): string | null { if (x == null) return null; @@ -67,6 +68,7 @@ const _PollVote = db.get('pollVotes'); const _Favorite = db.get('favorites'); const _NoteReaction = db.get('noteReactions'); const _Emoji = db.get('emoji'); +const _MessagingMessage = db.get('messagingMessages'); const getDriveFileBucket = async (): Promise => { const db = await nativeDbConn(); const bucket = new mongo.GridFSBucket(db, { @@ -90,6 +92,7 @@ async function main() { const UserPublickeys = getRepository(UserPublickey); const UserKeypairs = getRepository(UserKeypair); const Emojis = getRepository(Emoji); + const MessagingMessages = getRepository(MessagingMessage); async function migrateUser(user: any) { await Users.save({ @@ -330,6 +333,18 @@ async function main() { }); } + async function migrateMessagingMessage(message: any) { + await MessagingMessages.save({ + id: message._id.toHexString(), + createdAt: message.createdAt, + text: message.text, + userId: message.userId.toHexString(), + recipientId: message.recipientId.toHexString(), + fileId: message.fileId ? message.fileId.toHexString() : null, + isRead: message.isRead || false, + }); + } + let allUsersCount = await _User.count({ deletedAt: { $exists: false } }); @@ -496,6 +511,20 @@ async function main() { } } + const allMessagingMessagesCount = await _MessagingMessage.count(); + for (let i = 0; i < allMessagingMessagesCount; i++) { + const message = await _MessagingMessage.findOne({}, { + skip: i + }); + try { + await migrateMessagingMessage(message); + console.log(`EMOJI (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`EMOJI (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + console.log('DONE :)'); } From 369dce4701c4c9ec4701f6c0e6d3b766acea270f Mon Sep 17 00:00:00 2001 From: syuilo Date: Sat, 20 Apr 2019 01:43:01 +0900 Subject: [PATCH 3/8] Update migrate.ts --- src/migrate.ts | 58 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/src/migrate.ts b/src/migrate.ts index 53b681fbe..c7eded26b 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -27,6 +27,8 @@ import { Emoji } from './models/entities/emoji'; import { toPuny as _toPuny } from './misc/convert-host'; import { UserProfile } from './models/entities/user-profile'; import { MessagingMessage } from './models/entities/messaging-message'; +import { Muting } from './models/entities/muting'; +import { Blocking } from './models/entities/blocking'; function toPuny(x: string | null): string | null { if (x == null) return null; @@ -64,6 +66,8 @@ const _DriveFile = db.get('driveFiles.files'); const _DriveFolder = db.get('driveFolders'); const _Note = db.get('notes'); const _Following = db.get('following'); +const _Blocking = db.get('blocking'); +const _Muting = db.get('mute'); const _PollVote = db.get('pollVotes'); const _Favorite = db.get('favorites'); const _NoteReaction = db.get('noteReactions'); @@ -85,6 +89,8 @@ async function main() { const DriveFolders = getRepository(DriveFolder); const Notes = getRepository(Note); const Followings = getRepository(Following); + const Blockings = getRepository(Blocking); + const Mutings = getRepository(Muting); const Polls = getRepository(Poll); const PollVotes = getRepository(PollVote); const NoteFavorites = getRepository(NoteFavorite); @@ -157,6 +163,24 @@ async function main() { }); } + async function migrateBlocking(blocking: any) { + await Blockings.save({ + id: blocking._id.toHexString(), + createdAt: new Date(), + blockerId: blocking.blockerId.toHexString(), + blockeeId: blocking.blockeeId.toHexString(), + }); + } + + async function migrateMuting(muting: any) { + await Blockings.save({ + id: muting._id.toHexString(), + createdAt: new Date(), + muterId: muting.muterId.toHexString(), + muteeId: muting.muteeId.toHexString(), + }); + } + async function migrateDriveFolder(folder: any) { await DriveFolders.save({ id: folder._id.toHexString(), @@ -379,6 +403,36 @@ async function main() { } } + let allBlockingsCount = await _Blocking.count(); + if (test && allBlockingsCount > limit) allBlockingsCount = limit; + for (let i = 0; i < allBlockingsCount; i++) { + const blocking = await _Blocking.findOne({}, { + skip: i + }); + try { + await migrateBlocking(blocking); + console.log(`BLOCKING (${i + 1}/${allBlockingsCount}) ${blocking._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`BLOCKING (${i + 1}/${allBlockingsCount}) ${blocking._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + + let allMutingsCount = await _Muting.count(); + if (test && allMutingsCount > limit) allMutingsCount = limit; + for (let i = 0; i < allMutingsCount; i++) { + const muting = await _Muting.findOne({}, { + skip: i + }); + try { + await migrateMuting(muting); + console.log(`MUTING (${i + 1}/${allMutingsCount}) ${muting._id} ${chalk.green('DONE')}`); + } catch (e) { + console.log(`MUTING (${i + 1}/${allMutingsCount}) ${muting._id} ${chalk.red('ERR')}`); + console.error(e); + } + } + let allDriveFoldersCount = await _DriveFolder.count(); if (test && allDriveFoldersCount > limit) allDriveFoldersCount = limit; for (let i = 0; i < allDriveFoldersCount; i++) { @@ -518,9 +572,9 @@ async function main() { }); try { await migrateMessagingMessage(message); - console.log(`EMOJI (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.green('DONE')}`); + console.log(`MESSAGE (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.green('DONE')}`); } catch (e) { - console.log(`EMOJI (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.red('ERR')}`); + console.log(`MESSAGE (${i + 1}/${allMessagingMessagesCount}) ${message._id} ${chalk.red('ERR')}`); console.error(e); } } From 747bf3bb8534bfae09e70fdf1a453e6a7a582b23 Mon Sep 17 00:00:00 2001 From: syuilo Date: Wed, 24 Apr 2019 13:22:36 +0900 Subject: [PATCH 4/8] Update migrate.ts --- src/migrate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrate.ts b/src/migrate.ts index c7eded26b..31603a554 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -195,7 +195,7 @@ async function main() { _id: file.metadata.userId }); if (user == null) return; - if (file.metadata.storage && file.metadata.storage.key) { // when object storage + if (file.metadata.storageProps && file.metadata.storageProps.key) { // when object storage await DriveFiles.save({ id: file._id.toHexString(), userId: user._id.toHexString(), @@ -208,7 +208,7 @@ async function main() { size: file.length, url: file.metadata.url, uri: file.metadata.uri, - accessKey: file.metadata.storage.key, + accessKey: file.metadata.storageProps.key, folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, storedInternal: false, isLink: false From 6ce986fcbf50dfa414608c2f37f7b930f2891d8b Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Fri, 26 Apr 2019 07:45:29 +0900 Subject: [PATCH 5/8] for some situations, user.createdAt is UNIX timestamp number (... really??) this accuracy is seems to milliseconds. related: #4808 --- src/migrate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrate.ts b/src/migrate.ts index 31603a554..de9fbcc39 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -103,7 +103,7 @@ async function main() { async function migrateUser(user: any) { await Users.save({ id: user._id.toHexString(), - createdAt: user.createdAt || new Date(), + createdAt: typeof user.createdAt === 'number' ? new Date(user.createdAt) : (user.createdAt || new Date()), username: user.username, usernameLower: user.username.toLowerCase(), host: toPuny(user.host), From 318ae0035fb3baf2968414c96eadde40611e1961 Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Fri, 26 Apr 2019 07:56:47 +0900 Subject: [PATCH 6/8] =?UTF-8?q?=E3=83=9F=E3=83=A5=E3=83=BC=E3=83=88?= =?UTF-8?q?=E3=81=AE=E3=82=A4=E3=83=B3=E3=83=9D=E3=83=BC=E3=83=88=E3=81=A7?= =?UTF-8?q?=E3=82=82=E3=83=96=E3=83=AD=E3=83=83=E3=82=AF=E3=81=A8=E3=81=97?= =?UTF-8?q?=E3=81=A6=E5=85=A5=E3=82=8C=E3=82=88=E3=81=86=E3=81=A8=E3=81=97?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit related #4808 --- src/migrate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrate.ts b/src/migrate.ts index de9fbcc39..90960174d 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -173,7 +173,7 @@ async function main() { } async function migrateMuting(muting: any) { - await Blockings.save({ + await Mutings.save({ id: muting._id.toHexString(), createdAt: new Date(), muterId: muting.muterId.toHexString(), From fe1e7770f9b113e4ae5a0fd4f1dbbc13df07849b Mon Sep 17 00:00:00 2001 From: "Ch. (Chanhwi Choi)" Date: Thu, 2 May 2019 06:24:54 +0900 Subject: [PATCH 7/8] add userId to migrateFolder (#4829) after migration folders are blown up because folder has no userId. --- src/migrate.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/migrate.ts b/src/migrate.ts index 90960174d..4a636d0d8 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -184,6 +184,7 @@ async function main() { async function migrateDriveFolder(folder: any) { await DriveFolders.save({ id: folder._id.toHexString(), + userId: folder.userId.toHexString(), createdAt: folder.createdAt || new Date(), name: folder.name, parentId: folder.parentId ? folder.parentId.toHexString() : null, From ebcf62e13900b500af6e54d99fbf68f00960d5c7 Mon Sep 17 00:00:00 2001 From: "Ch. (Chanhwi Choi)" Date: Sun, 12 May 2019 09:41:32 +0900 Subject: [PATCH 8/8] fix some of migration problems (#4903) * migration: adds missing packages (ref:#4808) * migrate add missing: user * migrate add missing: file * migrate add missing: notes * migrate add validation: note - remove attachments which are already deleted - => (v11 can't hold them, causes error) - sets attachementTypes by already migrated - skips note migration when referenced note does not exist * migrate add validation: vote, favorite, reaction * migrate: typo --- package.json | 3 + src/migrate.ts | 254 +++++++++++++++++++++++++++++++++++-------------- 2 files changed, 186 insertions(+), 71 deletions(-) diff --git a/package.json b/package.json index acf9fac69..606d57ee1 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "@types/lolex": "3.1.1", "@types/minio": "7.0.1", "@types/mocha": "5.2.6", + "@types/mongodb": "3.1.26", "@types/node": "11.13.4", "@types/nodemailer": "4.6.7", "@types/nprogress": "0.0.29", @@ -166,6 +167,8 @@ "mocha": "6.1.3", "moji": "0.5.1", "moment": "2.24.0", + "mongodb": "3.2.3", + "monk": "6.0.6", "ms": "2.1.1", "nan": "2.12.1", "nested-property": "0.0.7", diff --git a/src/migrate.ts b/src/migrate.ts index 4a636d0d8..a4e22f46b 100644 --- a/src/migrate.ts +++ b/src/migrate.ts @@ -81,6 +81,8 @@ const getDriveFileBucket = async (): Promise => { return bucket; }; +const isMigrateRemoteNote = false; // making this true will try to migrate remote notes (possibly could cause errors) + async function main() { await initDb(); const Users = getRepository(User); @@ -100,10 +102,21 @@ async function main() { const Emojis = getRepository(Emoji); const MessagingMessages = getRepository(MessagingMessage); + async function validateNoteExistOnMigrated(noteId: string) { + if (!isMigrateRemoteNote) { + const noteMigrated = await Notes.findOne(noteId); + + if (noteMigrated === undefined) { + throw `=> ${chalk.yellow('SKIP')}: referenced note does not exist in migrated notes: ${noteId}`; + } + } + } + async function migrateUser(user: any) { await Users.save({ id: user._id.toHexString(), createdAt: typeof user.createdAt === 'number' ? new Date(user.createdAt) : (user.createdAt || new Date()), + updatedAt: typeof user.updatedAt === 'number' ? new Date(user.updatedAt) : (user.updatedAt || null), username: user.username, usernameLower: user.username.toLowerCase(), host: toPuny(user.host), @@ -119,17 +132,59 @@ async function main() { inbox: user.inbox, sharedInbox: user.sharedInbox, uri: user.uri, + emojis: user.emojis || [] as string[], + tags: user.tags || [] as string[], + isSuspended: user.isSuspended, + isSilenced: user.isSilenced, + isLocked: user.isLocked || false, }); - await UserProfiles.save({ + + const userProfileToSave: any = { userId: user._id.toHexString(), description: user.description, userHost: toPuny(user.host), autoAcceptFollowed: true, autoWatch: false, + alwaysMarkNsfw: user.settings ? user.settings.alwaysMarkNsfw : false, password: user.password, location: user.profile ? user.profile.location : null, birthday: user.profile ? user.profile.birthday : null, - }); + email: user.email, + emailVerified: user.emailVerified || false, + emailVerifyCode: user.emailVerifyCode, + twoFactorSecret: user.twoFactorSecret, + twoFactorEnabled: user.twoFactorEnabled, + twoFactorTempSecret: user.twoFactorTempSecret, + carefulBot: user.carefulBot + }; + + if (user.twitter) { + userProfileToSave.twitter = true; + userProfileToSave.twitterAccessToken = user.twitter.accessToken; + userProfileToSave.twitterAccessTokenSecret = user.twitter.accessTokenSecret; + userProfileToSave.twitterUserId = user.twitter.userId; + userProfileToSave.twitterScreenName = user.twitter.screenName; + } + + if (user.github) { + userProfileToSave.github = true; + userProfileToSave.githubAccessToken = user.github.accessToken; + userProfileToSave.githubId = Number(user.github.id); + userProfileToSave.githubLogin = user.github.login; + } + + if (user.discord) { + userProfileToSave.discord = true; + userProfileToSave.discordAccessToken = user.discord.accessToken; + userProfileToSave.discordrefreshToken = user.discord.refreshToken; + userProfileToSave.discordExpiresDate = user.discord.expiresDate; // number. + userProfileToSave.discordId = user.discord.id; + userProfileToSave.discordUsername = user.discord.username; + userProfileToSave.discordDiscriminator = user.discord.discriminator; + } + + await UserProfiles.save(userProfileToSave); + if (user.publicKey) { await UserPublickeys.save({ userId: user._id.toHexString(), @@ -196,24 +251,38 @@ async function main() { _id: file.metadata.userId }); if (user == null) return; + + const fileToSave: any = { + id: file._id.toHexString(), + userId: user._id.toHexString(), + userHost: toPuny(user.host), + createdAt: file.uploadDate || new Date(), + md5: file.md5, + name: file.filename, + type: file.contentType, + properties: file.metadata.properties || {}, + size: file.length, + // url: [different], + uri: file.metadata.uri, + // accessKey: [different], + folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, + // storedInternal: [different], + // isLink: [different], + isSensitive: file.metadata.isSensitive === true, + comment: file.metadata.comment && (file.metadata.comment.length > 0) && file.metadata.comment || null, + thumbnailUrl: file.metadata.thumbnailUrl, + thumbnailAccessKey: file.metadata.storageProps && file.metadata.storageProps.thumbnailAccessKey || null, + webpublicUrl: file.metadata.webpublicUrl, + webpublicAccessKey: file.metadata.storageProps && file.metadata.storageProps.webpublicAccessKey || null, + }; + if (file.metadata.storageProps && file.metadata.storageProps.key) { // when object storage - await DriveFiles.save({ - id: file._id.toHexString(), - userId: user._id.toHexString(), - userHost: toPuny(user.host), - createdAt: file.uploadDate || new Date(), - md5: file.md5, - name: file.filename, - type: file.contentType, - properties: file.metadata.properties || {}, - size: file.length, - url: file.metadata.url, - uri: file.metadata.uri, - accessKey: file.metadata.storageProps.key, - folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, - storedInternal: false, - isLink: false - }); + fileToSave.url = file.metadata.url; + fileToSave.accessKey = file.metadata.storageProps.key; + fileToSave.storedInternal = false; + fileToSave.isLink = false; + + await DriveFiles.save(fileToSave); } else if (!file.metadata.isLink) { const [temp, clean] = await createTemp(); await new Promise(async (res, rej) => { @@ -229,47 +298,26 @@ async function main() { const key = uuid.v4(); const url = InternalStorage.saveFromPath(key, temp); - await DriveFiles.save({ - id: file._id.toHexString(), - userId: user._id.toHexString(), - userHost: toPuny(user.host), - createdAt: file.uploadDate || new Date(), - md5: file.md5, - name: file.filename, - type: file.contentType, - properties: file.metadata.properties, - size: file.length, - url: url, - uri: file.metadata.uri, - accessKey: key, - folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, - storedInternal: true, - isLink: false - }); + + fileToSave.url = url; + fileToSave.accessKey = key; + fileToSave.storedInternal = true; + fileToSave.isLink = false; + + await DriveFiles.save(fileToSave); clean(); } else { - await DriveFiles.save({ - id: file._id.toHexString(), - userId: user._id.toHexString(), - userHost: toPuny(user.host), - createdAt: file.uploadDate || new Date(), - md5: file.md5, - name: file.filename, - type: file.contentType, - properties: file.metadata.properties, - size: file.length, - url: file.metadata.url, - uri: file.metadata.uri, - accessKey: null, - folderId: file.metadata.folderId ? file.metadata.folderId.toHexString() : null, - storedInternal: false, - isLink: true - }); + fileToSave.url = file.metadata.url; + fileToSave.accessKey = null; + fileToSave.storedInternal = false; + fileToSave.isLink = true; + + await DriveFiles.save(fileToSave); } } async function migrateNote(note: any) { - await Notes.save({ + const noteToSave = { id: note._id.toHexString(), createdAt: note.createdAt || new Date(), text: note.text, @@ -279,24 +327,74 @@ async function main() { viaMobile: note.viaMobile || false, geo: note.geo, appId: null, - visibility: note.visibility || 'public', + visibility: note.visibility && (note.visibility === 'private' ? 'specified' : note.visibility) || 'public', // there is no 'private' visibility more. visibleUserIds: note.visibleUserIds ? note.visibleUserIds.map((id: any) => id.toHexString()) : [], replyId: note.replyId ? note.replyId.toHexString() : null, renoteId: note.renoteId ? note.renoteId.toHexString() : null, userHost: null, fileIds: note.fileIds ? note.fileIds.map((id: any) => id.toHexString()) : [], + attachedFileTypes: ([] as string[]), // see below localOnly: note.localOnly || false, - hasPoll: note.poll != null - }); + hasPoll: note.poll != null, + name: note.name && (note.name.length > 0) && note.name || null, + emojis: note.emojis || ([] as string[]), + renoteCount: note.renoteCount || 0, + repliesCount: note.repliesCount || 0, + mentions: note.mentions && note.mentions.map((id: any) => id.toHexString()) || [], + mentionedRemoteUsers: note.mentionedRemoteUsers && JSON.stringify(note.mentionedRemoteUsers) || '[]', + score: note.score || 0, + uri: note.uri || null + }; + + // validate existance of referenced notes (on migrated) + if ((!isMigrateRemoteNote) && (noteToSave.replyId !== null || noteToSave.renoteId !== null)) { + // skip when reply does not exist on local + if (noteToSave.replyId !== null) { + const mongoReplyNoteLocal = await _Note.findOne({ + '_user.host': null, + '_id': note.replyId + }); + + if (mongoReplyNoteLocal === null) { + throw `=> ${chalk.yellow('SKIP')}: referenced "local" reply note does not exist: ${note.replyId}`; + } + } + + // skip when reply does not exist on local + if (noteToSave.renoteId !== null) { + const mongoRenoteNoteLocal = await _Note.findOne({ + '_user.host': null, + '_id': note.renoteId + }); + + if (mongoRenoteNoteLocal === null) { + throw `=> ${chalk.yellow('SKIP')}: referenced "local" renote note does not exist: ${note.renoteId}`; + } + } + } + + if (noteToSave.fileIds.length !== 0) { + const filesMigrated = await DriveFiles.findByIds(noteToSave.fileIds); + + // remove attachments which user removed after creating note + if (noteToSave.fileIds.length !== filesMigrated.length) { + console.warn(`NOTE ${noteToSave.id} ${chalk.yellow('MODIFIED')}: file count is different: before: ${noteToSave.fileIds.length} => after: ${filesMigrated.length}`); + noteToSave.fileIds = filesMigrated.map(file => file.id) || []; + } + + noteToSave.attachedFileTypes = filesMigrated.map(file => file.type); + } + + await Notes.save(noteToSave); if (note.poll) { await Polls.save({ noteId: note._id.toHexString(), choices: note.poll.choices.map((x: any) => x.text), expiresAt: note.poll.expiresAt, - multiple: note.poll.multiple, + multiple: note.poll.multiple || false, votes: note.poll.choices.map((x: any) => x.votes), - noteVisibility: note.visibility, + noteVisibility: note.visibility && (note.visibility === 'private' ? 'specified' : note.visibility) || 'public', // there is no 'private' visibility more. userId: note.userId.toHexString(), userHost: null }); @@ -304,32 +402,44 @@ async function main() { } async function migratePollVote(vote: any) { - await PollVotes.save({ + const voteToSave = { id: vote._id.toHexString(), createdAt: vote.createdAt, noteId: vote.noteId.toHexString(), userId: vote.userId.toHexString(), choice: vote.choice - }); + }; + + await validateNoteExistOnMigrated(voteToSave.noteId); + + await PollVotes.save(voteToSave); } async function migrateNoteFavorite(favorite: any) { - await NoteFavorites.save({ + const favoriteToSave = { id: favorite._id.toHexString(), createdAt: favorite.createdAt, noteId: favorite.noteId.toHexString(), userId: favorite.userId.toHexString(), - }); + }; + + await validateNoteExistOnMigrated(favoriteToSave.noteId); + + await NoteFavorites.save(favoriteToSave); } async function migrateNoteReaction(reaction: any) { - await NoteReactions.save({ + const reactionToSave = { id: reaction._id.toHexString(), createdAt: reaction.createdAt, noteId: reaction.noteId.toHexString(), userId: reaction.userId.toHexString(), reaction: reaction.reaction - }); + }; + + await validateNoteExistOnMigrated(reactionToSave.noteId); + + await NoteReactions.save(reactionToSave); } async function reMigrateUser(user: any) { @@ -470,16 +580,18 @@ async function main() { } } - let allNotesCount = await _Note.count({ + const noteCondition = { '_user.host': null, 'metadata.deletedAt': { $exists: false } - }); + }; + if (isMigrateRemoteNote) { + delete noteCondition['_user.host']; + } + + let allNotesCount = await _Note.count(noteCondition); if (test && allNotesCount > limit) allNotesCount = limit; for (let i = 0; i < allNotesCount; i++) { - const note = await _Note.findOne({ - '_user.host': null, - 'metadata.deletedAt': { $exists: false } - }, { + const note = await _Note.findOne(noteCondition, { skip: i }); try {