From dd3d562a1e4ae547ddb0b54d77678abf801f2e26 Mon Sep 17 00:00:00 2001 From: Latte macchiato Date: Fri, 19 Apr 2024 21:58:37 +0000 Subject: [PATCH 1/2] Rework cache clearing to be fault tolerant --- .../CleanRemoteFilesProcessorService.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index 917de8b72c..ec75f3ba01 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -33,6 +33,12 @@ export class CleanRemoteFilesProcessorService { let deletedCount = 0; let cursor: MiDriveFile['id'] | null = null; + let errorCount = 0; + + const total = await this.driveFilesRepository.countBy({ + userHost: Not(IsNull()), + isLink: false, + }); while (true) { const files = await this.driveFilesRepository.find({ @@ -41,7 +47,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, ...(cursor ? { id: MoreThan(cursor) } : {}), }, - take: 8, + take: 256, // Adjust the batch size as needed order: { id: 1, }, @@ -54,18 +60,21 @@ export class CleanRemoteFilesProcessorService { cursor = files.at(-1)?.id ?? null; - await Promise.all(files.map(file => this.driveService.deleteFileSync(file, true))); + // Handle deletion in a batch + const results = await Promise.allSettled(files.map(file => this.driveService.deleteFileSync(file, true))); - deletedCount += 8; - - const total = await this.driveFilesRepository.countBy({ - userHost: Not(IsNull()), - isLink: false, + results.forEach((result, index) => { + if (result.status === 'fulfilled') { + deletedCount++; + } else { + this.logger.error(`Failed to delete file ID ${files[index].id}: ${result.reason}`); + errorCount++; + } }); - job.updateProgress(deletedCount / total); + await job.updateProgress((deletedCount / total) * 100); } - this.logger.succ('All cached remote files has been deleted.'); + this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`); } } From 493775ad7b1ae20d40a3a6b6dd7eb505ced6648a Mon Sep 17 00:00:00 2001 From: PrivateGER Date: Wed, 24 Apr 2024 16:05:30 +0200 Subject: [PATCH 2/2] reformat expression --- .../src/queue/processors/CleanRemoteFilesProcessorService.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts index ec75f3ba01..4fa414b0b5 100644 --- a/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts +++ b/packages/backend/src/queue/processors/CleanRemoteFilesProcessorService.ts @@ -47,7 +47,7 @@ export class CleanRemoteFilesProcessorService { isLink: false, ...(cursor ? { id: MoreThan(cursor) } : {}), }, - take: 256, // Adjust the batch size as needed + take: 256, order: { id: 1, }, @@ -72,7 +72,8 @@ export class CleanRemoteFilesProcessorService { } }); - await job.updateProgress((deletedCount / total) * 100); + await job.updateProgress(100 / total * deletedCount); + } this.logger.succ(`All cached remote files processed. Total deleted: ${deletedCount}, Failed: ${errorCount}.`);