[web] extend project admin page for history debugging (#32437)

* [web] extend project admin page for history debugging

* [web] address review feedback

Co-authored-by: Malik <malik.glossop@overleaf.com>

---------

Co-authored-by: Malik <malik.glossop@overleaf.com>
GitOrigin-RevId: 01866e8c8529bc8332c49baf4ad281e300f8cdd4

services/docstore/app.js

@@ -51,6 +51,7 @@ app.param('doc_id', function (req, res, next, docId) {
 
 app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs)
 app.get('/project/:project_id/doc', HttpController.getAllDocs)
+app.get('/project/:project_id/doc-versions', HttpController.getAllDocVersions)
 app.get('/project/:project_id/ranges', HttpController.getAllRanges)
 app.get(
   '/project/:project_id/comment-thread-ids',

services/docstore/app/js/DocManager.js

@@ -142,6 +142,15 @@ const DocManager = {
     return docs
   },
 
+  async getAllDocVersions(projectId) {
+    // Do not unarchive all the docs: The version of archived docs is retained in mongo.
+    return await MongoManager.getProjectsDocs(
+      projectId,
+      { include_deleted: false },
+      { _id: true, version: true }
+    )
+  },
+
   async getCommentThreadIds(projectId) {
     const docs = await DocManager.getAllNonDeletedDocs(projectId, {
       _id: true,

services/docstore/app/js/HttpController.js

@@ -58,6 +58,12 @@ async function getAllDocs(req, res) {
   res.json(docViews)
 }
 
+async function getAllDocVersions(req, res) {
+  const { project_id: projectId } = req.params
+  const docs = await DocManager.getAllDocVersions(projectId)
+  res.json(docs)
+}
+
 async function getAllDeletedDocs(req, res) {
   const { project_id: projectId } = req.params
   logger.debug({ projectId }, 'getting all deleted docs')
@@ -244,6 +250,7 @@ export default {
   getAllDocs: expressify(getAllDocs),
   getAllDeletedDocs: expressify(getAllDeletedDocs),
   getAllRanges: expressify(getAllRanges),
+  getAllDocVersions: expressify(getAllDocVersions),
   getTrackedChangesUserIds: expressify(getTrackedChangesUserIds),
   getCommentThreadIds: expressify(getCommentThreadIds),
   projectHasRanges: expressify(projectHasRanges),

services/project-history/app/js/HttpController.js

@@ -262,6 +262,35 @@ export function getResyncPending(req, res, next) {
   })
 }
 
+const getDebugInfoSchema = z.object({
+  params: z.object({
+    project_id: zz.objectId(),
+  }),
+})
+
+export function getDebugInfo(req, res, next) {
+  const {
+    params: { project_id: projectId },
+  } = parseReq(req, getDebugInfoSchema)
+  SyncManager.getResyncState(projectId, (err, state) => {
+    if (err) return next(err)
+    ErrorRecorder.getFailureRecord(projectId, (err, failureRecord) => {
+      if (err) return next(err)
+      res.json({
+        failureRecord,
+        syncState: {
+          resyncPending: state.isSyncOngoing(),
+          resyncCount: state.resyncCount,
+          resyncPendingSince: state.resyncPendingSince,
+          lastUpdated: state.lastUpdated,
+          history: state.history,
+          ...state.toRaw(),
+        },
+      })
+    })
+  })
+}
+
 const latestVersionSchema = z.object({
   params: z.object({
     project_id: zz.objectId().or(z.coerce.number()),

services/project-history/app/js/Router.js

@@ -29,6 +29,7 @@ export function initialize(app) {
     '/project/:project_id/resync-pending',
     HttpController.getResyncPending
   )
+  app.get('/project/:project_id/debug-info', HttpController.getDebugInfo)
 
   app.post('/project/:project_id/resync', HttpController.resyncProject)
 

services/project-history/app/js/SyncManager.js

@@ -143,12 +143,14 @@ async function setResyncState(projectId, syncState) {
     // starting a new sync; prevent the entry expiring while sync is in ongoing
     update.$inc = { resyncCount: 1 }
     update.$unset = { expiresAt: true }
+    update.$min = { resyncPendingSince: new Date() }
   } else {
     // successful completion of existing sync; set the entry to expire in the
     // future
     update.$set.expiresAt = new Date(
       Date.now() + EXPIRE_RESYNC_HISTORY_INTERVAL_MS
     )
+    update.$unset = { resyncPendingSince: 1 }
   }
 
   // apply the update
@@ -270,11 +272,24 @@ async function expandSyncUpdates(
 }
 
 class SyncState {
-  constructor(projectId, resyncProjectStructure, resyncDocContents, origin) {
+  constructor(
+    projectId,
+    resyncProjectStructure,
+    resyncDocContents,
+    origin,
+    resyncCount,
+    resyncPendingSince,
+    lastUpdated,
+    history
+  ) {
     this.projectId = projectId
     this.resyncProjectStructure = resyncProjectStructure
     this.resyncDocContents = resyncDocContents
     this.origin = origin
+    this.resyncCount = resyncCount
+    this.resyncPendingSince = resyncPendingSince
+    this.lastUpdated = lastUpdated
+    this.history = history
   }
 
   static fromRaw(projectId, rawSyncState) {
@@ -282,11 +297,37 @@ class SyncState {
     const resyncProjectStructure = rawSyncState.resyncProjectStructure || false
     const resyncDocContents = new Set(rawSyncState.resyncDocContents || [])
     const origin = rawSyncState.origin
+    const resyncCount = rawSyncState.resyncCount || 0
+    let resyncPendingSince = rawSyncState.resyncPendingSince
+    const history = rawSyncState.history || []
+    if (
+      (resyncProjectStructure || resyncDocContents.size > 0) &&
+      !resyncPendingSince &&
+      history.length > 0
+    ) {
+      // The resyncPendingSince field was added later.
+      // Back-fill it as the next ts after a successful sync. History is DESC.
+      for (const other of history.slice().reverse()) {
+        const isSyncOngoing =
+          other.syncState.resyncProjectStructure ||
+          other.syncState.resyncDocContents.length > 0
+        if (isSyncOngoing) {
+          resyncPendingSince = resyncPendingSince || other.timestamp
+        } else {
+          resyncPendingSince = undefined
+        }
+      }
+    }
+    const lastUpdated = rawSyncState.lastUpdated
     return new SyncState(
       projectId,
       resyncProjectStructure,
       resyncDocContents,
-      origin
+      origin,
+      resyncCount,
+      resyncPendingSince,
+      lastUpdated,
+      history
     )
   }
 

services/web/app/src/Features/Docstore/DocstoreManager.mjs

@@ -103,6 +103,31 @@ async function getAllDeletedDocs(projectId) {
   }
 }
 
+/**
+ *
+ * @param {string|ObjectId} projectId
+ * @return {Promise<{_id: string, version: number}[]>}
+ */
+async function getAllDocVersions(projectId) {
+  const url = new URL(settings.apis.docstore.url)
+  url.pathname = path.posix.join(
+    'project',
+    projectId.toString(),
+    'doc-versions'
+  )
+  try {
+    return await fetchJson(url, { signal: AbortSignal.timeout(TIMEOUT) })
+  } catch (error) {
+    if (error instanceof RequestFailedError) {
+      throw new OError('docstore api responded with non-success code', {
+        projectId,
+        status: error.response.status,
+      })
+    }
+    throw OError.tag(error, 'could not get doc versions from docstore')
+  }
+}
+
 /**
  * @param {string} projectId
  */
@@ -368,6 +393,7 @@ export default {
   destroyProject: callbackify(destroyProject),
   promises: {
     deleteDoc,
+    getAllDocVersions,
     getAllDocs,
     getAllDeletedDocs,
     getAllRanges,

services/web/app/src/Features/History/HistoryManager.mjs

@@ -297,6 +297,12 @@ async function ensureNoResyncPending(projectId) {
   if (resyncPending) throw new OError('broken history with pending resync')
 }
 
+async function getDebugInfo(projectId) {
+  return await fetchJson(
+    `${settings.apis.project_history.url}/project/${projectId}/debug-info`
+  )
+}
+
 /**
  * Get history changes since a given version
  *
@@ -471,5 +477,6 @@ export default {
     getBlobStats,
     getLatestHistoryWithHistoryId,
     ensureNoResyncPending,
+    getDebugInfo,
   },
 }