Merge pull request #24069 from overleaf/bg-backup-errors

more tweaks for backup errors

GitOrigin-RevId: 0f7c7bb5004923c3c22c6e3471bb7152cc3e05e2

services/history-v1/storage/lib/backup_store/index.js

@@ -1,5 +1,6 @@
 const { Binary, ObjectId } = require('mongodb')
 const { projects, backedUpBlobs } = require('../mongodb')
+const OError = require('@overleaf/o-error')
 
 // List projects with pending backups older than the specified interval
 function listPendingBackups(timeIntervalMs = 0) {
@@ -79,7 +80,12 @@ async function setBackupVersion(
     }
   )
   if (result.matchedCount === 0 || result.modifiedCount === 0) {
-    throw new Error('Failed to update backup version')
+    throw new OError('Failed to update backup version', {
+      previousBackedUpVersion,
+      currentBackedUpVersion,
+      currentBackedUpAt,
+      result,
+    })
   }
 }
 

services/history-v1/storage/scripts/backup.mjs

@@ -593,6 +593,7 @@ export async function backupProject(projectId, options) {
   )
 
   let previousBackedUpVersion = lastBackedUpVersion
+  const backupVersions = [previousBackedUpVersion]
 
   for await (const {
     blobsToBackup,
@@ -624,14 +625,23 @@ export async function backupProject(projectId, options) {
     )
 
     // persist the backup status in mongo for the current chunk
-    await updateBackupStatus(
-      projectId,
-      previousBackedUpVersion,
-      chunkRecord,
-      backupStartTime
-    )
+    try {
+      await updateBackupStatus(
+        projectId,
+        previousBackedUpVersion,
+        chunkRecord,
+        backupStartTime
+      )
+    } catch (err) {
+      logger.error(
+        { projectId, chunkRecord, err, backupVersions },
+        'error updating backup status'
+      )
+      throw err
+    }
 
     previousBackedUpVersion = chunkRecord.endVersion
+    backupVersions.push(previousBackedUpVersion)
 
     await cleanBackedUpBlobs(projectId, blobsToBackup)
   }

services/history-v1/storage/scripts/backup_worker.mjs

@@ -13,7 +13,7 @@ const redisOptions = config.get('redis.queue')
 const TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000]
 
 // Configure backup settings to match worker concurrency
-configureBackup({ concurrency: 50, batchConcurrency: 3, useSecondary: true })
+configureBackup({ concurrency: 50, batchConcurrency: 3 })
 
 // Create a Bull queue named 'backup'
 const backupQueue = new Queue('backup', {