Merge pull request #29032 from overleaf/bg-retry-failed-backups

Add script and cronjobs to retry failed history-v1 backups

GitOrigin-RevId: c32102086235d4e22b947f4a80089b186c093cea

services/history-v1/storage/scripts/retry_failed_backups.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+# Extract project IDs with pending backups older than the configured timeout (default: 10000 seconds)
+RETRY_TIMEOUT="${RETRY_TIMEOUT:-10000}"
+ids=$(node storage/scripts/backup_scheduler.mjs --show-pending="$RETRY_TIMEOUT" | grep '^{' | jq -r '.projectId' | sort -u)
+# Retry backups for each project ID with a concurrency of 5 blob uploads at a time
+for project_id in $ids ; do
+  LOG_LEVEL=debug node storage/scripts/backup.mjs --projectId="$project_id" -c 5 ;
+done