[web+project-history] Add lastResyncedAt field to projects and set after resync (#33240)

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: f679a190b88694a46f7816d51eff96446f338dec

services/project-history/app/js/SyncManager.js

@@ -222,6 +222,17 @@ async function setResyncState(projectId, syncState) {
     update,
     { upsert: true }
   )
+
+  if (!syncState.isSyncOngoing()) {
+    await db.projects.updateOne(
+      { _id: new ObjectId(projectId) },
+      {
+        $max: {
+          'overleaf.history.lastResyncedAt': new Date(),
+        },
+      }
+    )
+  }
 }
 
 async function clearResyncState(projectId) {

services/project-history/test/acceptance/js/SyncTests.js

@@ -1,7 +1,6 @@
 import nock from 'nock'
 import { expect } from 'chai'
 import assert from 'node:assert'
-import mongodb from 'mongodb-legacy'
 import logger from '@overleaf/logger'
 import Settings from '@overleaf/settings'
 import {
@@ -17,7 +16,7 @@ import sinon from 'sinon'
 import { getFailure } from './helpers/ProjectHistoryClient.js'
 import { fetchNothing, RequestFailedError } from '@overleaf/fetch-utils'
 import { _getBlobHashFromString } from '../../../app/js/HashManager.js'
-const { ObjectId } = mongodb
+import { db, ObjectId } from '../../../app/js/mongodb.js'
 
 const EMPTY_FILE_HASH = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
 
@@ -45,6 +44,8 @@ describe('Syncing with web and doc-updater', function () {
     this.doc_id = new ObjectId().toString()
     this.file_id = new ObjectId().toString()
 
+    await db.projects.insertOne({ _id: new ObjectId(this.project_id) })
+
     MockHistoryStore().post('/api/projects').reply(200, {
       projectId: historyId,
     })
@@ -954,6 +955,7 @@ describe('Syncing with web and doc-updater', function () {
         })
 
         it('should send test updates to the history store', async function () {
+          const beforeResync = new Date()
           const addFile = MockHistoryStore()
             .post(`/api/projects/${historyId}/legacy_changes`, body => {
               expect(body).to.deep.equal([
@@ -1008,6 +1010,14 @@ describe('Syncing with web and doc-updater', function () {
             addFile.isDone(),
             `/api/projects/${historyId}/changes should have been called`
           )
+
+          const project = await db.projects.findOne({
+            _id: new ObjectId(this.project_id),
+          })
+          assert(
+            project.overleaf.history.lastResyncedAt > beforeResync,
+            'lastResyncedAt should have been updated when resync finished'
+          )
         })
 
         it('should strip non-BMP characters in updates before sending to the history store', async function () {

services/project-history/test/unit/js/SyncManager/SyncManagerTests.js

@@ -73,6 +73,9 @@ describe('SyncManager', function () {
         findOne: sinon.stub().resolves(this.syncState),
         updateOne: sinon.stub().resolves(),
       },
+      projects: {
+        updateOne: sinon.stub().resolves(),
+      },
     }
     this.extendLock = sinon.stub().resolves()
 

services/web/app/src/models/Project.mjs

@@ -88,6 +88,7 @@ export const ProjectSchema = new Schema(
         zipFileArchivedInProject: { type: Boolean },
         rangesSupportEnabled: { type: Boolean },
         otMigrationStage: { type: Number },
+        lastResyncedAt: { type: Date },
       },
       isDebugCopyOf: { type: ObjectId },
     },

services/web/scripts/history/resync_projects.mjs

@@ -24,6 +24,7 @@ Options:
     --max-id               Migrate projects to this id
     --last-updated-after   Migrate projects last updated after this date
     --last-updated-before  Migrate projects last updated before this date
+    --skip-resynced-after  Skip projects that have already been resynced after this date
     --commit               Actually perform the resync, instead of just checking which projects would be resynced
     --skip-metadata-checks Skip doing Mongo/Redis-level only checks to determine if the projects needs resyncing (has ranges or linked file data)
     --concurrency          How many jobs to run in parallel
@@ -32,7 +33,7 @@ Options:
 
 /**
  *
- * @returns {{ projectIds?: string[]; minId?: string; maxId?: string; concurrency: number; commit: boolean; skipMetadataChecks: boolean; lastUpdatedAfter?: string; lastUpdatedBefore?: string; }}
+ * @returns {{ projectIds?: string[]; minId?: string; maxId?: string; concurrency: number; commit: boolean; skipMetadataChecks: boolean; lastUpdatedAfter?: string; lastUpdatedBefore?: string; skipResyncedAfter?: string; }}
  */
 function parseArgs() {
   const args = minimist(process.argv.slice(2), {
@@ -43,6 +44,7 @@ function parseArgs() {
       'max-id',
       'last-updated-after',
       'last-updated-before',
+      'skip-resynced-after',
     ],
   })
 
@@ -59,13 +61,15 @@ function parseArgs() {
   const concurrency = parseInt(args.concurrency, 10) || 1
   const commit = args.commit
   const skipMetadataChecks = args['skip-metadata-checks']
+  const skipResyncedAfter = args['skip-resynced-after']
 
   if (
     projectIds == null &&
     minId == null &&
     maxId == null &&
     lastUpdatedAfter == null &&
-    lastUpdatedBefore == null
+    lastUpdatedBefore == null &&
+    skipResyncedAfter == null
   ) {
     console.error('Please specify at least one filter\n')
     usage()
@@ -81,6 +85,7 @@ function parseArgs() {
     skipMetadataChecks,
     lastUpdatedAfter,
     lastUpdatedBefore,
+    skipResyncedAfter,
   }
 }
 
@@ -94,6 +99,7 @@ async function main() {
     skipMetadataChecks,
     lastUpdatedAfter,
     lastUpdatedBefore,
+    skipResyncedAfter,
   } = parseArgs()
 
   // skip projects that don't have full project history
@@ -115,6 +121,13 @@ async function main() {
   if (lastUpdatedBefore) {
     clauses.push({ lastUpdated: { $lt: new Date(lastUpdatedBefore) } })
   }
+  if (skipResyncedAfter) {
+    clauses.push({
+      'overleaf.history.lastResyncedAt': {
+        $not: { $gt: new Date(skipResyncedAfter) },
+      },
+    })
+  }
   const filter = { $and: clauses }
 
   const projects = db.projects