heartles/egirlskey
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity

merge: Rework cache clearing to be fault tolerant (!497)

Browse source 
View MR for information: https://activitypub.software/TransFem-org/Sharkey/-/merge_requests/497

Approved-by: dakkar <dakkar@thenautilus.net>
Approved-by: Marie <marie@kaifa.ch>
This commit is contained in:
dakkar 2024-05-09 08:01:18 +00:00
commit 01256af028
1 changed files with 19 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts
View file

@ -33,6 +33,12 @@ export class CleanRemoteFilesProcessorService {
let deletedCount = 0; let deletedCount = 0;
let cursor: MiDriveFile['id'] | null = null; let cursor: MiDriveFile['id'] | null = null;
let errorCount = 0;
const total = await this.driveFilesRepository.countBy({
userHost: Not(IsNull()),
isLink: false,
});
while (true) { while (true) {
const files = await this.driveFilesRepository.find({ const files = await this.driveFilesRepository.find({
@ -41,7 +47,7 @@ export class CleanRemoteFilesProcessorService {
isLink: false, isLink: false,
...(cursor ? { id: MoreThan(cursor) } : {}), ...(cursor ? { id: MoreThan(cursor) } : {}),
}, },
take: 8, take: 256,
order: { order: {
id: 1, id: 1,
}, },
@ -54,18 +60,22 @@ export class CleanRemoteFilesProcessorService {
cursor = files.at(-1)?.id ?? null; cursor = files.at(-1)?.id ?? null;
await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true))); // Handle deletion in a batch
const results = await Promise.allSettled(files.map(file => this.driveService.deleteFileSync(file, true)));
deletedCount += 8; results.forEach((result, index) => {
if (result.status === 'fulfilled') {
const total = await this.driveFilesRepository.countBy({ deletedCount++;
userHost: Not(IsNull()), } else {
isLink: false, this.logger.error(`Failed to delete file ID ${files[index].id}: ${result.reason}`);
errorCount++;
}
}); });
job.updateProgress(deletedCount / total); await job.updateProgress(100 / total * deletedCount);
} }
this.logger.succ('All cached remote files has been deleted.'); this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`);
} }
} }