upd: add Facebook to note importing

This commit is contained in:
Mar0xy 2023-11-22 16:00:46 +01:00
parent cb9cbc8f15
commit d078a72f38
No known key found for this signature in database
GPG key ID: 56569BBE47D2C828
5 changed files with 78 additions and 7 deletions

View file

@ -301,6 +301,12 @@ export class QueueService {
return this.dbQueue.addBulk(jobs); return this.dbQueue.addBulk(jobs);
} }
@bindThis
public createImportFBToDbJob(user: ThinUser, targets: string[]) {
const jobs = targets.map(rel => this.generateToDbJobData('importFBToDb', { user, target: rel }));
return this.dbQueue.addBulk(jobs);
}
@bindThis @bindThis
public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) { public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) {
const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies })); const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies }));
@ -336,7 +342,7 @@ export class QueueService {
} }
@bindThis @bindThis
private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): { private generateToDbJobData<T extends 'importFollowingToDb' | 'importBlockingToDb' | 'importTweetsToDb' | 'importIGToDb' | 'importFBToDb' | 'importMastoToDb' | 'importPleroToDb' | 'importKeyNotesToDb', D extends DbJobData<T>>(name: T, data: D): {
name: string, name: string,
data: D, data: D,
opts: Bull.JobsOptions, opts: Bull.JobsOptions,

View file

@ -179,6 +179,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
case 'importNotes': return this.importNotesProcessorService.process(job); case 'importNotes': return this.importNotesProcessorService.process(job);
case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job); case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job);
case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job); case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job);
case 'importFBToDb': return this.importNotesProcessorService.processFBDb(job);
case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job); case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job);
case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job); case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job);
case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job); case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job);

View file

@ -1,13 +1,11 @@
import * as fs from 'node:fs'; import * as fs from 'node:fs';
import * as vm from 'node:vm'; import * as vm from 'node:vm';
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { ZipReader } from 'slacc'; import { ZipReader } from 'slacc';
import { DI } from '@/di-symbols.js'; import { DI } from '@/di-symbols.js';
import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository } from '@/models/_.js'; import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository, MiUser } from '@/models/_.js';
import type Logger from '@/logger.js'; import type Logger from '@/logger.js';
import { DownloadService } from '@/core/DownloadService.js'; import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js'; import { bindThis } from '@/decorators.js';
import { QueueService } from '@/core/QueueService.js'; import { QueueService } from '@/core/QueueService.js';
import { createTemp, createTempDir } from '@/misc/create-temp.js'; import { createTemp, createTempDir } from '@/misc/create-temp.js';
@ -35,7 +33,6 @@ export class ImportNotesProcessorService {
private notesRepository: NotesRepository, private notesRepository: NotesRepository,
private queueService: QueueService, private queueService: QueueService,
private utilityService: UtilityService,
private noteCreateService: NoteCreateService, private noteCreateService: NoteCreateService,
private mfmService: MfmService, private mfmService: MfmService,
private apNoteService: ApNoteService, private apNoteService: ApNoteService,
@ -47,9 +44,9 @@ export class ImportNotesProcessorService {
} }
@bindThis @bindThis
private async uploadFiles(dir: any, user: any) { private async uploadFiles(dir: string, user: MiUser) {
const fileList = fs.readdirSync(dir); const fileList = fs.readdirSync(dir);
for (const file of fileList) { for await (const file of fileList) {
const name = `${dir}/${file}`; const name = `${dir}/${file}`;
if (fs.statSync(name).isDirectory()) { if (fs.statSync(name).isDirectory()) {
await this.uploadFiles(name, user); await this.uploadFiles(name, user);
@ -172,6 +169,34 @@ export class ImportNotesProcessorService {
} finally { } finally {
cleanup(); cleanup();
} }
} else if (type === 'Facebook' || file.name.startsWith('facebook-') && file.name.endsWith('.zip')) {
const [path, cleanup] = await createTempDir();
this.logger.info(`Temp dir is ${path}`);
const destPath = path + '/facebook.zip';
try {
fs.writeFileSync(destPath, '', 'binary');
await this.downloadService.downloadUrl(file.url, destPath);
} catch (e) { // TODO: 何度か再試行
if (e instanceof Error || typeof e === 'string') {
this.logger.error(e);
}
throw e;
}
const outputPath = path + '/facebook';
try {
this.logger.succ(`Unzipping to ${outputPath}`);
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
const postsJson = fs.readFileSync(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8');
const posts = JSON.parse(postsJson);
await this.uploadFiles(outputPath + '/your_activity_across_facebook/posts/media', user);
this.queueService.createImportFBToDbJob(job.data.user, posts);
} finally {
cleanup();
}
} else if (file.name.endsWith('.zip')) { } else if (file.name.endsWith('.zip')) {
const [path, cleanup] = await createTempDir(); const [path, cleanup] = await createTempDir();
@ -535,4 +560,41 @@ export class ImportNotesProcessorService {
this.logger.warn(`Error: ${e}`); this.logger.warn(`Error: ${e}`);
} }
} }
@bindThis
public async processFBDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
const post = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
return;
}
if (!this.isIterable(post.data) || !post.data[0].post) return;
const date = new Date(post.timestamp * 1000);
const title = decodeFBString(post.data[0].post);
const files: MiDriveFile[] = [];
function decodeFBString(str: string) {
const arr = [];
for (let i = 0; i < str.length; i++) {
arr.push(str.charCodeAt(i));
}
return Buffer.from(arr).toString('utf8');
}
if (post.attachments && this.isIterable(post.attachments) && this.isIterable(post.attachments.data)) {
for await (const file of post.attachments.data) {
if (!file.media) return;
const slashdex = file.media.uri.lastIndexOf('/');
const name = file.media.uri.substring(slashdex + 1);
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id });
if (exists) {
files.push(exists);
}
}
}
await this.noteCreateService.import(user, { createdAt: date, text: title, files: files });
}
} }

View file

@ -52,6 +52,7 @@ export type DbJobMap = {
importNotes: DbNoteImportJobData; importNotes: DbNoteImportJobData;
importTweetsToDb: DbKeyNoteImportToDbJobData; importTweetsToDb: DbKeyNoteImportToDbJobData;
importIGToDb: DbNoteImportToDbJobData; importIGToDb: DbNoteImportToDbJobData;
importFBToDb: DbNoteImportToDbJobData;
importMastoToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteImportToDbJobData;
importPleroToDb: DbNoteImportToDbJobData; importPleroToDb: DbNoteImportToDbJobData;
importKeyNotesToDb: DbKeyNoteImportToDbJobData; importKeyNotesToDb: DbKeyNoteImportToDbJobData;

View file

@ -22,6 +22,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<option value="Mastodon">Mastodon/Pleroma/Akkoma</option> <option value="Mastodon">Mastodon/Pleroma/Akkoma</option>
<option value="Twitter">Twitter</option> <option value="Twitter">Twitter</option>
<option value="Instagram">Instagram</option> <option value="Instagram">Instagram</option>
<option value="Facebook">Facebook</option>
</MkRadios> </MkRadios>
<MkButton primary :class="$style.button" inline @click="importNotes($event)"><i class="ph-upload ph-bold ph-lg"></i> {{ i18n.ts.import }}</MkButton> <MkButton primary :class="$style.button" inline @click="importNotes($event)"><i class="ph-upload ph-bold ph-lg"></i> {{ i18n.ts.import }}</MkButton>
</MkFolder> </MkFolder>