heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

properly thread Mastodon imports

Browse source
This commit is contained in:
dakkar 2023-11-30 12:23:09 +00:00
parent e392d523a7
commit 15503b96a0
3 changed files with 25 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/backend/src/core/QueueService.ts
View file

@ -278,8 +278,8 @@ export class QueueService {
} }
@bindThis @bindThis
public createImportMastoToDbJob(user: ThinUser, targets: string[]) { public createImportMastoToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) {
const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel })); const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel, note }));
return this.dbQueue.addBulk(jobs); return this.dbQueue.addBulk(jobs);
} }

31
packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View file

@ -17,7 +17,7 @@ import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js';
import { IdService } from '@/core/IdService.js'; import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js'; import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq'; import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbKeyNoteImportToDbJobData } from '../types.js'; import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js';
@Injectable() @Injectable()
export class ImportNotesProcessorService { export class ImportNotesProcessorService {
@ -274,7 +274,8 @@ export class ImportNotesProcessorService {
if (fs.existsSync(outputPath + '/media_attachments/files') && mastoFolder) { if (fs.existsSync(outputPath + '/media_attachments/files') && mastoFolder) {
await this.uploadFiles(outputPath + '/media_attachments/files', user, mastoFolder.id); await this.uploadFiles(outputPath + '/media_attachments/files', user, mastoFolder.id);
} }
this.queueService.createImportMastoToDbJob(job.data.user, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note')); const processedToots = await this.recreateChain('id', 'inReplyTo', outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'), true);
this.queueService.createImportMastoToDbJob(job.data.user, processedToots, null);
} }
} }
} finally { } finally {
@ -306,7 +307,7 @@ export class ImportNotesProcessorService {
} }
@bindThis @bindThis
public async processKeyNotesToDb(job: Bull.Job<DbKeyNoteImportToDbJobData>): Promise<void> { public async processKeyNotesToDb(job: Bull.Job<DbNoteWithParentImportToDbJobData>): Promise<void> {
const note = job.data.target; const note = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
@ -363,28 +364,33 @@ export class ImportNotesProcessorService {
} }
@bindThis @bindThis
public async processMastoToDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> { public async processMastoToDb(job: Bull.Job<DbNoteWithParentImportToDbJobData>): Promise<void> {
const toot = job.data.target; const toot = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
return; return;
} }
if (toot.directMessage) return;
const date = new Date(toot.object.published); const date = new Date(toot.object.published);
let text = undefined; let text = undefined;
const files: MiDriveFile[] = []; const files: MiDriveFile[] = [];
let reply: MiNote | null = null; let reply: MiNote | null = null;
if (toot.object.inReplyTo != null) { if (toot.object.inReplyTo != null) {
try { const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null;
reply = await this.apNoteService.resolveNote(toot.object.inReplyTo); if (parentNote) {
} catch (error) { reply = parentNote;
reply = null; } else {
try {
reply = await this.apNoteService.resolveNote(toot.object.inReplyTo);
} catch (error) {
reply = null;
}
} }
} }
if (toot.directMessage) return;
const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null); const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null);
try { try {
@ -404,7 +410,8 @@ export class ImportNotesProcessorService {
} }
} }
await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: toot.object.sensitive ? toot.object.summary : null, reply: reply }); const createdNote = await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: toot.object.sensitive ? toot.object.summary : null, reply: reply });
if (toot.childNotes) this.queueService.createImportMastoToDbJob(user, toot.childNotes, createdNote.id);
} }
@bindThis @bindThis
@ -525,7 +532,7 @@ export class ImportNotesProcessorService {
} }
@bindThis @bindThis
public async processTwitterDb(job: Bull.Job<DbKeyNoteImportToDbJobData>): Promise<void> { public async processTwitterDb(job: Bull.Job<DbNoteWithParentImportToDbJobData>): Promise<void> {
const tweet = job.data.target; const tweet = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {

8
packages/backend/src/queue/types.ts
View file

@ -50,12 +50,12 @@ export type DbJobMap = {
exportUserLists: DbJobDataWithUser; exportUserLists: DbJobDataWithUser;
importAntennas: DBAntennaImportJobData; importAntennas: DBAntennaImportJobData;
importNotes: DbNoteImportJobData; importNotes: DbNoteImportJobData;
importTweetsToDb: DbKeyNoteImportToDbJobData; importTweetsToDb: DbNoteWithParentImportToDbJobData;
importIGToDb: DbNoteImportToDbJobData; importIGToDb: DbNoteImportToDbJobData;
importFBToDb: DbNoteImportToDbJobData; importFBToDb: DbNoteImportToDbJobData;
importMastoToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteWithParentImportToDbJobData;
importPleroToDb: DbNoteImportToDbJobData; importPleroToDb: DbNoteImportToDbJobData;
importKeyNotesToDb: DbKeyNoteImportToDbJobData; importKeyNotesToDb: DbNoteWithParentImportToDbJobData;
importFollowing: DbUserImportJobData; importFollowing: DbUserImportJobData;
importFollowingToDb: DbUserImportToDbJobData; importFollowingToDb: DbUserImportToDbJobData;
importMuting: DbUserImportJobData; importMuting: DbUserImportJobData;
@ -113,7 +113,7 @@ export type DbNoteImportToDbJobData = {
target: any; target: any;
}; };
export type DbKeyNoteImportToDbJobData = { export type DbNoteWithParentImportToDbJobData = {
user: ThinUser; user: ThinUser;
target: any; target: any;
note: MiNote['id'] | null; note: MiNote['id'] | null;