Merge pull request #24237 from overleaf/bg-fix-backup-worker

fixes for backup worker

GitOrigin-RevId: 79c99e5b2d99d1b0298f69d968e0e2cb89281dd6

services/history-v1/backup-worker-app.mjs

@@ -38,10 +38,10 @@ app.use((err, req, res, next) => {
 })
 
 async function triggerGracefulShutdown(server, signal) {
-  logger.warn({ signal }, 'graceful shutdown: started shutdown sequence')
+  logger.info({ signal }, 'graceful shutdown: started shutdown sequence')
   await drainQueue()
   server.close(function () {
-    logger.warn({ signal }, 'graceful shutdown: closed server')
+    logger.info({ signal }, 'graceful shutdown: closed server')
     setTimeout(() => {
       process.exit(0)
     }, 1000)

services/history-v1/storage/scripts/backup.mjs

@@ -89,7 +89,7 @@ process.on('SIGTERM', handleSignal)
 
 function handleSignal() {
   gracefulShutdownInitiated = true
-  console.warn('graceful shutdown initiated, draining queue')
+  logger.info({}, 'graceful shutdown initiated, draining queue')
 }
 
 async function retry(fn, times, delayMs) {

services/history-v1/storage/scripts/backup_scheduler.mjs

@@ -33,13 +33,13 @@ const optionDefinitions = [
     name: 'queue-pending',
     type: Number,
     description:
-      'Find projects with pending changes older than N minutes and add them to the queue',
+      'Find projects with pending changes older than N seconds and add them to the queue',
   },
   {
     name: 'show-pending',
     type: Number,
     description:
-      'Show count of pending projects older than N minutes without adding to queue',
+      'Show count of pending projects older than N seconds without adding to queue',
   },
   {
     name: 'limit',
@@ -79,25 +79,20 @@ function isValidDateFormat(dateStr) {
 function validatePendingTime(option, value) {
   if (typeof value !== 'number' || value <= 0) {
     console.error(
-      `Error: --${option} requires a positive numeric TIME argument in minutes`
+      `Error: --${option} requires a positive numeric TIME argument in seconds`
     )
-    console.error(`Example: --${option} 60`)
+    console.error(`Example: --${option} 3600`)
     process.exit(1)
   }
   return value
 }
 
-// Helper to calculate minutes since a given date
-function minutesSince(date) {
-  const now = new Date()
-  const diffMs = now - date
-  return Math.floor(diffMs / (1000 * 60))
-}
-
 // Helper to format the pending time display
 function formatPendingTime(timestamp) {
-  const minutes = minutesSince(timestamp)
-  return `${timestamp.toISOString()} (${minutes} minutes ago)`
+  const now = new Date()
+  const diffMs = now - timestamp
+  const seconds = Math.floor(diffMs / 1000)
+  return `${timestamp.toISOString()} (${seconds} seconds ago)`
 }
 
 // Helper to add a job to the queue, checking for duplicates
@@ -189,17 +184,17 @@ async function addDateRangeJob(input) {
   )
 }
 
-// Process pending projects with changes older than the specified minutes
+// Process pending projects with changes older than the specified seconds
 async function processPendingProjects(
-  minutes,
+  age,
   showOnly,
   limit,
   verbose,
   jobOpts = {}
 ) {
-  const timeIntervalMs = minutes * 60 * 1000
+  const timeIntervalMs = age * 1000
   console.log(
-    `Finding projects with pending changes older than ${minutes} minutes${showOnly ? ' (count only)' : ''}`
+    `Finding projects with pending changes older than ${age} seconds${showOnly ? ' (count only)' : ''}`
   )
 
   let count = 0
@@ -308,26 +303,17 @@ async function run() {
   } else if (options.monitor) {
     setupMonitoring()
   } else if (options['queue-pending'] !== undefined) {
-    const minutes = validatePendingTime(
-      'queue-pending',
-      options['queue-pending']
-    )
-    await processPendingProjects(
-      minutes,
-      false,
-      options.limit,
-      options.verbose,
-      {
-        attempts: options.attempts,
-        backoff: {
-          type: 'exponential',
-          delay: options['backoff-delay'],
-        },
-      }
-    )
+    const age = validatePendingTime('queue-pending', options['queue-pending'])
+    await processPendingProjects(age, false, options.limit, options.verbose, {
+      attempts: options.attempts,
+      backoff: {
+        type: 'exponential',
+        delay: options['backoff-delay'],
+      },
+    })
   } else if (options['show-pending'] !== undefined) {
-    const minutes = validatePendingTime('show-pending', options['show-pending'])
-    await processPendingProjects(minutes, true, options.limit, options.verbose)
+    const age = validatePendingTime('show-pending', options['show-pending'])
+    await processPendingProjects(age, true, options.limit, options.verbose)
   } else {
     console.log('Usage:')
     console.log('  --clean               Clean up completed and failed jobs')
@@ -338,10 +324,10 @@ async function run() {
     )
     console.log('  --monitor             Monitor queue events')
     console.log(
-      '  --queue-pending TIME  Find projects with changes older than TIME minutes and add them to the queue'
+      '  --queue-pending TIME  Find projects with changes older than TIME seconds and add them to the queue'
     )
     console.log(
-      '  --show-pending TIME   Show count of pending projects older than TIME minutes'
+      '  --show-pending TIME   Show count of pending projects older than TIME seconds'
     )
     console.log('  --limit N             Limit the number of jobs to be added')
     console.log(

services/history-v1/storage/scripts/backup_worker.mjs

@@ -27,12 +27,12 @@ const backupQueue = new Queue('backup', {
 
 // Log queue events
 backupQueue.on('active', job => {
-  logger.info({ job }, 'job  is now active')
+  logger.debug({ job }, 'job is now active')
 })
 
 backupQueue.on('completed', (job, result) => {
   metrics.inc('backup_worker_job', 1, { status: 'completed' })
-  logger.info({ job, result }, 'job completed')
+  logger.debug({ job, result }, 'job completed')
 })
 
 backupQueue.on('failed', (job, err) => {
@@ -41,7 +41,7 @@ backupQueue.on('failed', (job, err) => {
 })
 
 backupQueue.on('waiting', jobId => {
-  logger.info({ jobId }, 'job is waiting')
+  logger.debug({ jobId }, 'job is waiting')
 })
 
 backupQueue.on('error', error => {
@@ -85,7 +85,7 @@ async function runBackup(projectId) {
     TIME_BUCKETS
   )
   try {
-    logger.info({ projectId }, 'processing backup for project')
+    logger.debug({ projectId }, 'processing backup for project')
     await backupProject(projectId, {})
     metrics.inc('backup_worker_project', 1, {
       status: 'success',