[web] extend scripts for finding and fixing broken file-trees (#22984)

- Validate _id fields to be ObjectIds
- Include more debug output, notably include the folder/doc/file id
- Emit and consume JSON output
- Read broken file-tree details from (ad-hoc) file
- Use generator instead of accumulating all results per project first
- Use batchedUpdate to be able to pause and resume processing

GitOrigin-RevId: 606ac431fff65891e09479c0ba9bcb7e7dc5cbe4

services/web/scripts/find_malformed_filetrees.mjs

@@ -1,107 +1,163 @@
-import {
-  db,
-  READ_PREFERENCE_SECONDARY,
-} from '../app/src/infrastructure/mongodb.js'
+// @ts-check
+import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
+import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
+
+/**
+ * @typedef {Object} Doc
+ * @property {ObjectId} _id
+ * @property {string} name
+ */
+
+/**
+ * @typedef {Object} FileRef
+ * @property {ObjectId} _id
+ * @property {string} name
+ * @property {string} hash
+ */
+
+/**
+ * @typedef {Object} Folder
+ * @property {ObjectId} _id
+ * @property {string} name
+ * @property {Array<Doc>} docs
+ * @property {Array<Folder>} folders
+ * @property {Array<FileRef>} fileRefs
+ */
+
+/**
+ * @typedef {Object} Project
+ * @property {ObjectId} _id
+ * @property {Array<Folder>} rootFolder
+ */
 
 async function main() {
-  const projects = db.projects.find(
+  let projectsProcessed = 0
+  await batchedUpdate(
+    db.projects,
     {},
-    {
-      projection: { rootFolder: 1 },
-      readPreference: READ_PREFERENCE_SECONDARY,
-    }
+    /**
+     * @param {Array<Project>} projects
+     * @return {Promise<void>}
+     */
+    async function projects(projects) {
+      for (const project of projects) {
+        projectsProcessed += 1
+        if (projectsProcessed % 100000 === 0) {
+          console.log(projectsProcessed, 'projects processed')
+        }
+        const projectId = project._id.toString()
+        for (const { reason, path, _id } of processProject(project)) {
+          console.log(
+            JSON.stringify({
+              msg: 'bad file-tree path',
+              projectId,
+              reason,
+              path,
+              _id,
+            })
+          )
+        }
+      }
+    },
+    { _id: 1, rootFolder: 1 }
   )
-  let projectsProcessed = 0
-  for await (const project of projects) {
-    projectsProcessed += 1
-    if (projectsProcessed % 100000 === 0) {
-      console.log(projectsProcessed, 'projects processed')
-    }
-    processProject(project)
-  }
 }
 
-function processProject(project) {
+/**
+ * @param {Project} project
+ * @return {Generator<{path: string, reason: string, _id: any}, void, *>}
+ */
+function* processProject(project) {
   if (!project.rootFolder || !Array.isArray(project.rootFolder)) {
-    console.log('BAD PATH:', project._id, 'rootFolder')
-    return
-  }
-  if (!project.rootFolder[0]) {
-    console.log('BAD PATH:', project._id, 'rootFolder.0')
-    return
-  }
-  const badPaths = findBadPaths(project.rootFolder[0])
-  for (const path of badPaths) {
-    console.log('BAD PATH:', project._id, `rootFolder.0.${path}`)
+    yield { reason: 'bad rootFolder', path: 'rootFolder', _id: null }
+  } else if (!project.rootFolder[0]) {
+    yield { reason: 'missing rootFolder', path: 'rootFolder.0', _id: null }
+  } else {
+    for (const { path, reason, _id } of findBadPaths(project.rootFolder[0])) {
+      yield { reason, path: `rootFolder.0${path}`, _id }
+    }
   }
 }
 
-function findBadPaths(folder) {
-  const result = []
+/**
+ * @param {Folder} folder
+ * @return {Generator<{path: string, reason: string, _id: any}, void, *>}
+ */
+function* findBadPaths(folder) {
+  const folderId = folder._id
 
-  if (!folder._id) {
-    result.push('_id')
+  if (!(folderId instanceof ObjectId)) {
+    yield { path: '._id', reason: 'bad folder id', _id: folderId }
   }
 
   if (typeof folder.name !== 'string' || !folder.name) {
-    result.push('name')
+    yield { path: '.name', reason: 'bad folder name', _id: folderId }
   }
 
   if (folder.folders && Array.isArray(folder.folders)) {
     for (const [i, subfolder] of folder.folders.entries()) {
       if (!subfolder || typeof subfolder !== 'object') {
-        result.push(`folders.${i}`)
+        yield { path: `.folders.${i}`, reason: 'bad folder', _id: folderId }
         continue
       }
-      for (const badPath of findBadPaths(subfolder)) {
-        result.push(`folders.${i}.${badPath}`)
+      for (const { path, reason, _id } of findBadPaths(subfolder)) {
+        yield { path: `.folders.${i}${path}`, reason, _id }
       }
     }
   } else {
-    result.push('folders')
+    yield { path: '.folders', reason: 'missing .folders', _id: folderId }
   }
 
   if (folder.docs && Array.isArray(folder.docs)) {
     for (const [i, doc] of folder.docs.entries()) {
       if (!doc || typeof doc !== 'object') {
-        result.push(`docs.${i}`)
+        yield { path: `.docs.${i}`, reason: 'bad doc', _id: folderId }
         continue
       }
-      if (!doc._id) {
-        result.push(`docs.${i}._id`)
+      const docId = doc._id
+      if (!(docId instanceof ObjectId)) {
+        yield { path: `.docs.${i}._id`, reason: 'bad doc id', _id: docId }
         // no need to check further: this doc can be deleted
         continue
       }
       if (typeof doc.name !== 'string' || !doc.name) {
-        result.push(`docs.${i}.name`)
+        yield { path: `.docs.${i}.name`, reason: 'bad doc name', _id: docId }
       }
     }
   } else {
-    result.push('docs')
+    yield { path: '.docs', reason: 'missing .docs', _id: folderId }
   }
 
   if (folder.fileRefs && Array.isArray(folder.fileRefs)) {
     for (const [i, file] of folder.fileRefs.entries()) {
       if (!file || typeof file !== 'object') {
-        result.push(`fileRefs.${i}`)
+        yield { path: `.fileRefs.${i}`, reason: 'bad file', _id: folderId }
         continue
       }
-      if (!file._id) {
-        result.push(`fileRefs.${i}._id`)
+      const fileId = file._id
+      if (!(fileId instanceof ObjectId)) {
+        yield { path: `.fileRefs.${i}._id`, reason: 'bad file id', _id: fileId }
         // no need to check further: this file can be deleted
         continue
       }
       if (typeof file.name !== 'string' || !file.name) {
-        result.push(`fileRefs.${i}.name`)
+        yield {
+          path: `.fileRefs.${i}.name`,
+          reason: 'bad file name',
+          _id: fileId,
+        }
       }
       if (typeof file.hash !== 'string' || !file.hash) {
-        result.push(`fileRefs.${i}.hash`)
+        yield {
+          path: `.fileRefs.${i}.hash`,
+          reason: 'bad file hash',
+          _id: fileId,
+        }
       }
     }
   } else {
-    result.push('fileRefs')
+    yield { path: '.fileRefs', reason: 'missing .fileRefs', _id: folderId }
   }
-  return result
 }
 
 try {

services/web/scripts/fix_malformed_filetree.mjs

@@ -1,18 +1,73 @@
 /**
  * This script fixes problems found by the find_malformed_filetrees.js script.
  *
- * The script takes two arguments: the project id and the problemtatic path.
- * This is the output format of each line in the find_malformed_filetrees.js
- * script.
+ * The script takes a single argument --logs pointing at the output of a
+ * previous run of the find_malformed_filetrees.js script.
+ *
+ * Alternatively, use an adhoc file: --logs=<(echo '{"projectId":"...","path":"..."}')
  */
 import mongodb from 'mongodb-legacy'
 import { db } from '../app/src/infrastructure/mongodb.js'
 import ProjectLocator from '../app/src/Features/Project/ProjectLocator.js'
+import minimist from 'minimist'
+import readline from 'node:readline'
+import fs from 'node:fs'
+import logger from '@overleaf/logger'
 
 const { ObjectId } = mongodb
 
+const argv = minimist(process.argv.slice(2), {
+  string: ['logs'],
+})
+
+let gracefulShutdownInitiated = false
+
+process.on('SIGINT', handleSignal)
+process.on('SIGTERM', handleSignal)
+
+function handleSignal() {
+  gracefulShutdownInitiated = true
+  console.warn('graceful shutdown initiated, draining queue')
+}
+
+const STATS = {
+  processedLines: 0,
+  success: 0,
+  alreadyProcessed: 0,
+  hash: 0,
+  failed: 0,
+  unmatched: 0,
+}
+function logStats() {
+  console.log(
+    JSON.stringify({
+      time: new Date(),
+      gracefulShutdownInitiated,
+      ...STATS,
+    })
+  )
+}
+setInterval(logStats, 10_000)
+
 async function main() {
-  const { projectId, mongoPath } = parseArgs()
+  const rl = readline.createInterface({
+    input: fs.createReadStream(argv.logs),
+  })
+  for await (const line of rl) {
+    if (gracefulShutdownInitiated) break
+    STATS.processedLines++
+    if (!line.startsWith('{')) continue
+    try {
+      const { projectId, path, _id } = JSON.parse(line)
+      await processBadPath(projectId, path, _id)
+    } catch (err) {
+      STATS.failed++
+      logger.err({ line, err }, 'failed to fix tree')
+    }
+  }
+}
+
+async function processBadPath(projectId, mongoPath, _id) {
   let modifiedCount
   if (isRootFolder(mongoPath)) {
     modifiedCount = await fixRootFolder(projectId)
@@ -30,27 +85,22 @@ async function main() {
   } else if (isName(mongoPath)) {
     modifiedCount = await fixName(projectId, mongoPath)
   } else if (isHash(mongoPath)) {
-    console.error(`Missing file hash: ${mongoPath}`)
+    console.error(`Missing file hash: ${projectId}/${_id} (${mongoPath})`)
     console.error('SaaS: likely needs filestore restore')
     console.error('Server Pro: please reach out to support')
-    process.exit(1)
+    STATS.hash++
+    return
   } else {
     console.error(`Unexpected mongo path: ${mongoPath}`)
-    process.exit(1)
+    STATS.unmatched++
+    return
   }
 
-  console.log(`${modifiedCount} project(s) modified`)
-  process.exit(0)
-}
-
-function parseArgs() {
-  const args = process.argv.slice(2)
-  if (args.length !== 2) {
-    console.error('Usage: fix_malformed_filetree.js PROJECT_ID MONGO_PATH')
-    process.exit(1)
+  if (modifiedCount === 0) {
+    STATS.alreadyProcessed++
+  } else {
+    STATS.success++
   }
-  const [projectId, mongoPath] = args
-  return { projectId: new ObjectId(projectId), mongoPath }
 }
 
 function isRootFolder(path) {
@@ -182,8 +232,20 @@ function findUniqueName(existingFilenames) {
 }
 
 try {
-  await main()
-  process.exit(0)
+  try {
+    await main()
+  } finally {
+    logStats()
+  }
+  if (STATS.failed > 0) {
+    process.exit(Math.min(STATS.failed, 99))
+  } else if (STATS.hash > 0) {
+    process.exit(100)
+  } else if (STATS.unmatched > 0) {
+    process.exit(101)
+  } else {
+    process.exit(0)
+  }
 } catch (error) {
   console.error(error)
   process.exit(1)