upd: import own replies to own tweets

This commit is contained in:
Mar0xy 2023-11-17 20:10:16 +01:00
parent 13600e0abc
commit 4929467016
No known key found for this signature in database
GPG key ID: 56569BBE47D2C828
3 changed files with 34 additions and 7 deletions

View file

@ -272,8 +272,8 @@ export class QueueService {
} }
@bindThis @bindThis
public createImportTweetsToDbJob(user: ThinUser, targets: string[]) { public createImportTweetsToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) {
const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel })); const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel, note }));
return this.dbQueue.addBulk(jobs); return this.dbQueue.addBulk(jobs);
} }

View file

@ -93,6 +93,29 @@ export class ImportNotesProcessorService {
return notesTree; return notesTree;
} }
@bindThis
private async recreateTwitChain(arr: any[]) {
type TweetsMap = {
[id: string]: any;
};
const tweetsTree: any[] = [];
const lookup: TweetsMap = {};
for await (const tweet of arr) {
lookup[`${tweet.id_str}`] = tweet;
tweet.replies = [];
let parent = null;
if (!tweet.in_reply_to_status_id_str) {
tweetsTree.push(tweet);
} else {
parent = lookup[`${tweet.in_reply_to_status_id_str}`];
}
if (parent) parent.replies.push(tweet);
}
return tweetsTree;
}
@bindThis @bindThis
private isIterable(obj: any) { private isIterable(obj: any) {
if (obj == null) { if (obj == null) {
@ -155,7 +178,10 @@ export class ImportNotesProcessorService {
const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => { const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => {
return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet); return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet);
}, []); }, []);
this.queueService.createImportTweetsToDbJob(job.data.user, tweets); // Due to the way twitter outputs the tweets the entire array needs to be reversed for the recreate function.
const reversedTweets = tweets.reverse();
const processedTweets = await this.recreateTwitChain(reversedTweets);
this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null);
} finally { } finally {
cleanup(); cleanup();
} }
@ -430,14 +456,14 @@ export class ImportNotesProcessorService {
} }
@bindThis @bindThis
public async processTwitterDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> { public async processTwitterDb(job: Bull.Job<DbKeyNoteImportToDbJobData>): Promise<void> {
const tweet = job.data.target; const tweet = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) { if (user == null) {
return; return;
} }
if (tweet.in_reply_to_status_id_str) return; const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null;
async function replaceTwitterUrls(full_text: string, urls: any) { async function replaceTwitterUrls(full_text: string, urls: any) {
let full_textedit = full_text; let full_textedit = full_text;
@ -516,7 +542,8 @@ export class ImportNotesProcessorService {
} }
} }
} }
await this.noteCreateService.import(user, { createdAt: date, text: text, files: files }); const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: text, files: files });
if (tweet.replies) this.queueService.createImportTweetsToDbJob(user, tweet.replies, createdNote.id);
} catch (e) { } catch (e) {
this.logger.warn(`Error: ${e}`); this.logger.warn(`Error: ${e}`);
} }

View file

@ -50,7 +50,7 @@ export type DbJobMap = {
exportUserLists: DbJobDataWithUser; exportUserLists: DbJobDataWithUser;
importAntennas: DBAntennaImportJobData; importAntennas: DBAntennaImportJobData;
importNotes: DbNoteImportJobData; importNotes: DbNoteImportJobData;
importTweetsToDb: DbNoteImportToDbJobData; importTweetsToDb: DbKeyNoteImportToDbJobData;
importIGToDb: DbNoteImportToDbJobData; importIGToDb: DbNoteImportToDbJobData;
importMastoToDb: DbNoteImportToDbJobData; importMastoToDb: DbNoteImportToDbJobData;
importPleroToDb: DbNoteImportToDbJobData; importPleroToDb: DbNoteImportToDbJobData;