From 5ce6fa67d212823be43c796e8632b721ede065bb Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 2 Apr 2018 23:19:07 +0900 Subject: [PATCH] Make migration scripts and use createdAt instead of _id --- src/models/followed-log.ts | 1 + src/models/following-log.ts | 1 + src/processor/http/follow.ts | 2 + .../endpoints/aggregation/users/followers.ts | 4 +- .../endpoints/aggregation/users/following.ts | 4 +- tools/migration/nighthike/10.js | 3 + tools/migration/nighthike/9.js | 79 +++++++++++++++++++ 7 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tools/migration/nighthike/10.js create mode 100644 tools/migration/nighthike/9.js diff --git a/src/models/followed-log.ts b/src/models/followed-log.ts index 4d8ecf684e..9e3ca17822 100644 --- a/src/models/followed-log.ts +++ b/src/models/followed-log.ts @@ -6,6 +6,7 @@ export default FollowedLog; export type IFollowedLog = { _id: ObjectID; + createdAt: Date; userId: ObjectID; count: number; }; diff --git a/src/models/following-log.ts b/src/models/following-log.ts index f18707db80..045ff7bf02 100644 --- a/src/models/following-log.ts +++ b/src/models/following-log.ts @@ -6,6 +6,7 @@ export default FollowingLog; export type IFollowingLog = { _id: ObjectID; + createdAt: Date; userId: ObjectID; count: number; }; diff --git a/src/processor/http/follow.ts b/src/processor/http/follow.ts index 29ac9fa552..cbb7838c68 100644 --- a/src/processor/http/follow.ts +++ b/src/processor/http/follow.ts @@ -24,6 +24,7 @@ export default ({ data }, done) => Following.findOne({ _id: data.following }).th }), promisedFollower.then(({ followingCount }) => FollowingLog.insert({ + createdAt: data.following.createdAt, userId: followerId, count: followingCount + 1 })), @@ -36,6 +37,7 @@ export default ({ data }, done) => Following.findOne({ _id: data.following }).th }), promisedFollowee.then(({ followersCount }) => FollowedLog.insert({ + createdAt: data.following.createdAt, userId: followerId, count: followersCount + 1 })), diff --git a/src/server/api/endpoints/aggregation/users/followers.ts b/src/server/api/endpoints/aggregation/users/followers.ts index 580d31a3f2..f9a5e8dca8 100644 --- a/src/server/api/endpoints/aggregation/users/followers.ts +++ b/src/server/api/endpoints/aggregation/users/followers.ts @@ -42,10 +42,10 @@ module.exports = (params) => new Promise(async (res, rej) => { for (let i = 0; i < 30; i++) { graph.push(FollowedLog.findOne({ - _id: { $lt: ObjectID.createFromTime(cursorTime / 1000) }, + createdAt: { $lt: new Date(cursorTime / 1000) }, userId: user._id }, { - sort: { _id: -1 }, + sort: { createdAt: -1 }, }).then(log => { cursorDate = new Date(today.getTime()); cursorTime = cursorDate.setDate(today.getDate() - i); diff --git a/src/server/api/endpoints/aggregation/users/following.ts b/src/server/api/endpoints/aggregation/users/following.ts index 3ac0e3a539..b30b1282ba 100644 --- a/src/server/api/endpoints/aggregation/users/following.ts +++ b/src/server/api/endpoints/aggregation/users/following.ts @@ -42,10 +42,10 @@ module.exports = (params) => new Promise(async (res, rej) => { for (let i = 0; i < 30; i++) { graph.push(FollowingLog.findOne({ - _id: { $lt: ObjectID.createFromTime(cursorTime / 1000) }, + createdAt: { $lt: new Date(cursorTime / 1000) }, userId: user._id }, { - sort: { _id: -1 }, + sort: { createdAt: -1 }, }).then(log => { cursorDate = new Date(today.getTime()); cursorTime = cursorDate.setDate(today.getDate() - i); diff --git a/tools/migration/nighthike/10.js b/tools/migration/nighthike/10.js new file mode 100644 index 0000000000..3c57b8d484 --- /dev/null +++ b/tools/migration/nighthike/10.js @@ -0,0 +1,3 @@ +db.following.remove({ + deletedAt: { $exists: true } +}); diff --git a/tools/migration/nighthike/9.js b/tools/migration/nighthike/9.js new file mode 100644 index 0000000000..a904feb06f --- /dev/null +++ b/tools/migration/nighthike/9.js @@ -0,0 +1,79 @@ +// for Node.js interpret + +const { default: Following } = require('../../../built/models/following'); +const { default: FollowingLog } = require('../../../built/models/following-log'); +const { default: FollowedLog } = require('../../../built/models/followed-log'); +const { default: zip } = require('@prezzemolo/zip') +const html = require('../../../built/text/html').default; +const parse = require('../../../built/text/parse').default; + +const migrate = async (following) => { + const followingCount = await Following.count({ + followerId: following.followerId, + _id: { $lt: following._id }, + $or: [ + { deletedAt: { $exists: false } }, + { deletedAt: { $gt: following.createdAt } } + ] + }); + await FollowingLog.insert({ + createdAt: following.createdAt, + userId: following.followerId, + count: followingCount + 1 + }); + + const followersCount = await Following.count({ + followeeId: following.followeeId, + _id: { $lt: following._id }, + $or: [ + { deletedAt: { $exists: false } }, + { deletedAt: { $gt: following.createdAt } } + ] + }); + await FollowedLog.insert({ + createdAt: following.createdAt, + userId: following.followeeId, + count: followersCount + 1 + }); + + if (following.deletedAt) { + await FollowingLog.insert({ + createdAt: following.deletedAt, + userId: following.followerId, + count: followingCount - 1 + }); + + await FollowedLog.insert({ + createdAt: following.deletedAt, + userId: following.followeeId, + count: followersCount - 1 + }); + } + + return true; +} + +async function main() { + const count = await Following.count({}); + + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 + + return zip( + 1, + async (time) => { + console.log(`${time} / ${idop}`) + const doc = await Following.find({}, { + limit: dop, skip: time * dop, sort: { _id: 1 } + }) + return Promise.all(doc.map(migrate)) + }, + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv + }) +} + +main().then(console.dir).catch(console.error)