From d078a72f38d9722bcb6b6b9c657011c54d34b37b Mon Sep 17 00:00:00 2001 From: Mar0xy Date: Wed, 22 Nov 2023 16:00:46 +0100 Subject: [PATCH] upd: add `Facebook` to note importing --- packages/backend/src/core/QueueService.ts | 8 +- .../src/queue/QueueProcessorService.ts | 1 + .../processors/ImportNotesProcessorService.ts | 74 +++++++++++++++++-- packages/backend/src/queue/types.ts | 1 + .../src/pages/settings/import-export.vue | 1 + 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 7135f1b1b4..0230c9a7b8 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -301,6 +301,12 @@ export class QueueService { return this.dbQueue.addBulk(jobs); } + @bindThis + public createImportFBToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importFBToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + @bindThis public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) { const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies })); @@ -336,7 +342,7 @@ export class QueueService { } @bindThis - private generateToDbJobData>(name: T, data: D): { + private generateToDbJobData>(name: T, data: D): { name: string, data: D, opts: Bull.JobsOptions, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index d0e1a46a10..31814bf9cd 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -179,6 +179,7 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'importNotes': return this.importNotesProcessorService.process(job); case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job); case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job); + case 'importFBToDb': return this.importNotesProcessorService.processFBDb(job); case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job); case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job); case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job); diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 5acf9c9167..a3d7915b96 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -1,13 +1,11 @@ import * as fs from 'node:fs'; import * as vm from 'node:vm'; import { Inject, Injectable } from '@nestjs/common'; -import { IsNull } from 'typeorm'; import { ZipReader } from 'slacc'; import { DI } from '@/di-symbols.js'; -import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository } from '@/models/_.js'; +import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository, MiUser } from '@/models/_.js'; import type Logger from '@/logger.js'; import { DownloadService } from '@/core/DownloadService.js'; -import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; import { QueueService } from '@/core/QueueService.js'; import { createTemp, createTempDir } from '@/misc/create-temp.js'; @@ -35,7 +33,6 @@ export class ImportNotesProcessorService { private notesRepository: NotesRepository, private queueService: QueueService, - private utilityService: UtilityService, private noteCreateService: NoteCreateService, private mfmService: MfmService, private apNoteService: ApNoteService, @@ -47,9 +44,9 @@ export class ImportNotesProcessorService { } @bindThis - private async uploadFiles(dir: any, user: any) { + private async uploadFiles(dir: string, user: MiUser) { const fileList = fs.readdirSync(dir); - for (const file of fileList) { + for await (const file of fileList) { const name = `${dir}/${file}`; if (fs.statSync(name).isDirectory()) { await this.uploadFiles(name, user); @@ -172,6 +169,34 @@ export class ImportNotesProcessorService { } finally { cleanup(); } + } else if (type === 'Facebook' || file.name.startsWith('facebook-') && file.name.endsWith('.zip')) { + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + const destPath = path + '/facebook.zip'; + + try { + fs.writeFileSync(destPath, '', 'binary'); + await this.downloadService.downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const outputPath = path + '/facebook'; + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath)); + const postsJson = fs.readFileSync(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8'); + const posts = JSON.parse(postsJson); + await this.uploadFiles(outputPath + '/your_activity_across_facebook/posts/media', user); + this.queueService.createImportFBToDbJob(job.data.user, posts); + } finally { + cleanup(); + } } else if (file.name.endsWith('.zip')) { const [path, cleanup] = await createTempDir(); @@ -535,4 +560,41 @@ export class ImportNotesProcessorService { this.logger.warn(`Error: ${e}`); } } + + @bindThis + public async processFBDb(job: Bull.Job): Promise { + const post = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + if (!this.isIterable(post.data) || !post.data[0].post) return; + + const date = new Date(post.timestamp * 1000); + const title = decodeFBString(post.data[0].post); + const files: MiDriveFile[] = []; + + function decodeFBString(str: string) { + const arr = []; + for (let i = 0; i < str.length; i++) { + arr.push(str.charCodeAt(i)); + } + return Buffer.from(arr).toString('utf8'); + } + + if (post.attachments && this.isIterable(post.attachments) && this.isIterable(post.attachments.data)) { + for await (const file of post.attachments.data) { + if (!file.media) return; + const slashdex = file.media.uri.lastIndexOf('/'); + const name = file.media.uri.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id }); + if (exists) { + files.push(exists); + } + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: title, files: files }); + } } diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index 4df9e00ccb..8d09e4e197 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -52,6 +52,7 @@ export type DbJobMap = { importNotes: DbNoteImportJobData; importTweetsToDb: DbKeyNoteImportToDbJobData; importIGToDb: DbNoteImportToDbJobData; + importFBToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteImportToDbJobData; importPleroToDb: DbNoteImportToDbJobData; importKeyNotesToDb: DbKeyNoteImportToDbJobData; diff --git a/packages/frontend/src/pages/settings/import-export.vue b/packages/frontend/src/pages/settings/import-export.vue index b129254ce3..31bfd7e732 100644 --- a/packages/frontend/src/pages/settings/import-export.vue +++ b/packages/frontend/src/pages/settings/import-export.vue @@ -22,6 +22,7 @@ SPDX-License-Identifier: AGPL-3.0-only + {{ i18n.ts.import }}