Merge pull request #24037 from overleaf/bg-fix-backup-worker-errors

backup worker tweaks

GitOrigin-RevId: 7901bb0257c7c7320afd08ef965f124347a1de30

services/history-v1/storage/scripts/backup.mjs

@@ -63,6 +63,7 @@ const RETRY_DELAY = 1000
 let CONCURRENCY = 4
 let BATCH_CONCURRENCY = 1
 let BLOB_LIMITER = pLimit(CONCURRENCY)
+let USE_SECONDARY = false
 
 /**
  * Configure backup settings
@@ -74,6 +75,7 @@ export function configureBackup(options = {}) {
   CONCURRENCY = options.concurrency || 1
   BATCH_CONCURRENCY = options.batchConcurrency || 1
   BLOB_LIMITER = pLimit(CONCURRENCY)
+  USE_SECONDARY = options.useSecondary || false
 }
 
 let gracefulShutdownInitiated = false
@@ -330,6 +332,11 @@ const optionDefinitions = [
     type: String,
     description: 'End date for initialization (ISO format)',
   },
+  {
+    name: 'use-secondary',
+    type: Boolean,
+    description: 'Use secondary read preference for backup status',
+  },
   {
     name: 'compare',
     alias: 'C',
@@ -387,6 +394,10 @@ function handleOptions() {
     process.exit(1)
   }
 
+  if (options['use-secondary']) {
+    USE_SECONDARY = true
+  }
+
   if (
     options.compare &&
     !options.projectId &&
@@ -416,7 +427,9 @@ async function analyseBackupStatus(projectId) {
     await getBackupStatus(projectId)
   // TODO: when we have confidence that the latestChunkMetadata always matches
   // the values from the backupStatus we can skip loading it here
-  const latestChunkMetadata = await loadLatestRaw(historyId)
+  const latestChunkMetadata = await loadLatestRaw(historyId, {
+    readOnly: Boolean(USE_SECONDARY),
+  })
   if (
     currentEndVersion &&
     currentEndVersion !== latestChunkMetadata.endVersion
@@ -506,6 +519,9 @@ function makeChunkKey(projectId, startVersion) {
 }
 
 export async function backupProject(projectId, options) {
+  if (gracefulShutdownInitiated) {
+    return
+  }
   await ensureGlobalBlobsLoaded()
   // FIXME: flush the project first!
   // Let's assume the the flush happens externally and triggers this backup

services/history-v1/storage/scripts/backup_worker.mjs

@@ -13,11 +13,14 @@ const redisOptions = config.get('redis.queue')
 const TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000]
 
 // Configure backup settings to match worker concurrency
-configureBackup({ concurrency: 5, batchConcurrency: 5 })
+configureBackup({ concurrency: 50, batchConcurrency: 3, useSecondary: true })
 
 // Create a Bull queue named 'backup'
 const backupQueue = new Queue('backup', {
   redis: redisOptions,
+  settings: {
+    stalledInterval: 0, // don't check for stalled jobs
+  },
 })
 
 // Log queue events