diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts index 892d97ad5..5acf9c916 100644 --- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -69,53 +69,42 @@ export class ImportNotesProcessorService { } } - // Function was taken from Firefish and edited to remove renoteId and make it run in only one for loop instead of two + // Function was taken from Firefish and modified for our needs @bindThis - private async recreateChain(arr: any[]) { + private async recreateChain(idField: string, replyField: string, arr: any[]): Promise { type NotesMap = { [id: string]: any; }; const notesTree: any[] = []; - const lookup: NotesMap = {}; + const noteById: NotesMap = {}; + const notesWaitingForParent: NotesMap = {}; + for await (const note of arr) { - lookup[`${note.id}`] = note; + noteById[note[idField]] = note; note.childNotes = []; - let parent = null; - - if (note.replyId == null) { + + const children = notesWaitingForParent[note[idField]]; + if (children) { + note.childNotes.push(...children); + } + + if (note[replyField] == null) { notesTree.push(note); - } else { - parent = lookup[`${note.replyId}`]; + continue; } - if (parent) parent.childNotes.push(note); + const parent = noteById[note[replyField]]; + if (parent) { + parent.childNotes.push(note); + } else { + notesWaitingForParent[note[replyField]] ||= []; + notesWaitingForParent[note[replyField]].push(note); + } } + return notesTree; } - @bindThis - private async recreateTwitChain(arr: any[]) { - type TweetsMap = { - [id: string]: any; - }; - const tweetsTree: any[] = []; - const lookup: TweetsMap = {}; - for await (const tweet of arr) { - lookup[`${tweet.id_str}`] = tweet; - tweet.replies = []; - let parent = null; - - if (!tweet.in_reply_to_status_id_str) { - tweetsTree.push(tweet); - } else { - parent = lookup[`${tweet.in_reply_to_status_id_str}`]; - } - - if (parent) parent.replies.push(tweet); - } - return tweetsTree; - } - @bindThis private isIterable(obj: any) { if (obj == null) { @@ -178,9 +167,7 @@ export class ImportNotesProcessorService { const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => { return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet); }, []); - // Due to the way twitter outputs the tweets the entire array needs to be reversed for the recreate function. - const reversedTweets = tweets.reverse(); - const processedTweets = await this.recreateTwitChain(reversedTweets); + const processedTweets = await this.recreateChain("id_str", "in_reply_to_status_id_str", tweets); this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null); } finally { cleanup(); @@ -248,7 +235,7 @@ export class ImportNotesProcessorService { const notesJson = fs.readFileSync(path, 'utf-8'); const notes = JSON.parse(notesJson); - const processedNotes = await this.recreateChain(notes); + const processedNotes = await this.recreateChain("id", "replyId", notes); this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null); cleanup(); } @@ -484,9 +471,9 @@ export class ImportNotesProcessorService { try { const date = new Date(tweet.created_at); const textReplaceURLs = tweet.entities.urls && tweet.entities.urls.length > 0 ? await replaceTwitterUrls(tweet.full_text, tweet.entities.urls) : tweet.full_text; - const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs; + const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs; const files: MiDriveFile[] = []; - + if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) { for await (const file of tweet.extended_entities.media) { if (file.video_info) { @@ -543,7 +530,7 @@ export class ImportNotesProcessorService { } } const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: text, files: files }); - if (tweet.replies) this.queueService.createImportTweetsToDbJob(user, tweet.replies, createdNote.id); + if (tweet.childNotes) this.queueService.createImportTweetsToDbJob(user, tweet.childNotes, createdNote.id); } catch (e) { this.logger.warn(`Error: ${e}`); }