From bf7601fa8d2728969823fb5a6ce24ae47cf4a992 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 7 Nov 2017 22:08:21 +0900 Subject: [PATCH 1/4] run parallely --- package.json | 1 + tools/migration/use-gridfs.js | 33 +++++++++++---------------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/package.json b/package.json index cf291fe2b..96831a3c4 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "format": "gulp format" }, "devDependencies": { + "@prezzemolo/zip": "0.0.3", "@types/bcryptjs": "2.4.1", "@types/body-parser": "1.16.7", "@types/chai": "4.0.4", diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js index 106cbd388..228943f49 100644 --- a/tools/migration/use-gridfs.js +++ b/tools/migration/use-gridfs.js @@ -3,6 +3,7 @@ const { default: db } = require('../../built/db/mongodb') const { default: DriveFile, getGridFSBucket } = require('../../built/api/models/drive-file') const { Duplex } = require('stream') +const { default: zip } = require('@prezzemolo/zip') const writeToGridFS = (bucket, buffer, ...rest) => new Promise((resolve, reject) => { const writeStream = bucket.openUploadStreamWithId(...rest) @@ -45,30 +46,18 @@ const migrateToGridFS = async (doc) => { } async function main() { - let i = 0; + const count = await DriveFile.count({}); - const count = await db.get('drive_files').count({}); + const dop = Number.parseInt(process.argv[2]) || 5 - const iterate = async () => { - if (i == count) return true; - console.log(`${i} / ${count}`); - const doc = (await db.get('drive_files').find({}, { limit: 1, skip: i }))[0] - const res = await migrateToGridFS(doc); - if (!res) { - return false; - } else { - i++ - return await iterate(); - } - } - - const res = await iterate(); - - if (res) { - return 'ok'; - } else { - throw 'something happened'; - } + return zip( + 1, + async (time) => { + const doc = await DriveFile.find({}, { limit: dop, skip: time * dop }) + return Promise.all(doc.map(migrateToGridFS)) + }, + ((count - (count % dop)) / dop) + 1 + ) } main().then(console.dir).catch(console.error) From 36a95c003aa9d66ea29befbcd9ac5c10e6d76938 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 7 Nov 2017 22:39:17 +0900 Subject: [PATCH 2/4] fix --- tools/migration/use-gridfs.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js index 228943f49..bb7070f55 100644 --- a/tools/migration/use-gridfs.js +++ b/tools/migration/use-gridfs.js @@ -48,16 +48,24 @@ const migrateToGridFS = async (doc) => { async function main() { const count = await DriveFile.count({}); + console.log(`there are ${count} files.`) + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 return zip( 1, async (time) => { - const doc = await DriveFile.find({}, { limit: dop, skip: time * dop }) + console.log(`${time} / ${idop}`) + const doc = await db.get('drive_files').find({}, { limit: dop, skip: time * dop }) return Promise.all(doc.map(migrateToGridFS)) }, - ((count - (count % dop)) / dop) + 1 - ) + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv + }) } main().then(console.dir).catch(console.error) From 9f2dc28088ef546f2b552aebb54346e0bcebcb37 Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 7 Nov 2017 22:57:48 +0900 Subject: [PATCH 3/4] fix --- tools/migration/use-gridfs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/migration/use-gridfs.js b/tools/migration/use-gridfs.js index bb7070f55..a9d2b12e9 100644 --- a/tools/migration/use-gridfs.js +++ b/tools/migration/use-gridfs.js @@ -46,7 +46,7 @@ const migrateToGridFS = async (doc) => { } async function main() { - const count = await DriveFile.count({}); + const count = await db.get('drive_files').count({}); console.log(`there are ${count} files.`) From 6875b68f3e80513274dce925b9bd3e2f012f324e Mon Sep 17 00:00:00 2001 From: otofune Date: Tue, 7 Nov 2017 23:10:38 +0900 Subject: [PATCH 4/4] change all migrations to parallely --- ...change-gridfs-metadata-name-to-filename.js | 24 ++++++++++- tools/migration/issue_882.js | 43 ++++++++++--------- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/tools/migration/change-gridfs-metadata-name-to-filename.js b/tools/migration/change-gridfs-metadata-name-to-filename.js index 0d9e977c6..9128d852c 100644 --- a/tools/migration/change-gridfs-metadata-name-to-filename.js +++ b/tools/migration/change-gridfs-metadata-name-to-filename.js @@ -19,12 +19,32 @@ async function applyNewChange (doc) { } async function main () { - const oldTypeDocs = await DriveFile.find({ + const query = { 'metadata.name': { $exists: true } + } + + const count = await DriveFile.count(query) + + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 + + return zip( + 1, + async (time) => { + console.log(`${time} / ${idop}`) + const doc = await db.get('drive_files').find(query, { + limit: dop, skip: time * dop + }) + return Promise.all(doc.map(applyNewChange)) + }, + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv }) - return await Promise.all(oldTypeDocs.map(applyNewChange)) } main().then(console.dir).catch(console.error) diff --git a/tools/migration/issue_882.js b/tools/migration/issue_882.js index 8dab9bb43..aa1141325 100644 --- a/tools/migration/issue_882.js +++ b/tools/migration/issue_882.js @@ -1,6 +1,7 @@ // for Node.js interpret const { default: DriveFile } = require('../../built/api/models/drive-file') +const { default: zip } = require('@prezzemolo/zip') const migrate = async (doc) => { const result = await DriveFile.update(doc._id, { @@ -15,30 +16,32 @@ const migrate = async (doc) => { } async function main() { - let i = 0; - - const count = await DriveFile.count({}); - - const iterate = async () => { - if (i == count) return true; - console.log(`${i} / ${count}`); - const doc = (await DriveFile.find({}, { limit: 1, skip: i }))[0] - const res = await migrate(doc); - if (!res) { - return false; - } else { - i++ - return await iterate(); + const query = { + 'metadata.type': { + $exists: true } } - const res = await iterate(); + const count = await DriveFile.count(query); - if (res) { - return 'ok'; - } else { - throw 'something happened'; - } + const dop = Number.parseInt(process.argv[2]) || 5 + const idop = ((count - (count % dop)) / dop) + 1 + + return zip( + 1, + async (time) => { + console.log(`${time} / ${idop}`) + const doc = await db.get('drive_files').find(query, { + limit: dop, skip: time * dop + }) + return Promise.all(doc.map(migrate)) + }, + idop + ).then(a => { + const rv = [] + a.forEach(e => rv.push(...e)) + return rv + }) } main().then(console.dir).catch(console.error)