Merge pull request #5915 from overleaf/em-delete-dangling-refs

Add a script for deleting dangling doc and file refs

GitOrigin-RevId: 9de5576e96a6585e72a642324b94744fc605d739

services/web/scripts/cleanup_incorrectly_expired_projects.js

@@ -1,137 +0,0 @@
-/**
- * This script cleans up active projects that were expired. It's meant to be used once.
- *
- * See https://github.com/overleaf/internal/issues/4532
- */
-const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
-const Errors = require('../app/src/Features/Errors/Errors')
-const FileStoreHandler = require('../app/src/Features/FileStore/FileStoreHandler')
-const ProjectEntityMongoUpdateHandler = require('../app/src/Features/Project/ProjectEntityMongoUpdateHandler')
-const { batchedUpdate } = require('./helpers/batchedUpdate')
-
-const DRY_RUN = process.env.DRY_RUN !== 'false'
-
-waitForDb()
-  .then(main)
-  .then(() => {
-    process.exitCode = 0
-  })
-  .catch(error => {
-    console.error({ error })
-    process.exitCode = 1
-  })
-  .finally(() => {
-    process.exit()
-  })
-
-async function main() {
-  await batchedUpdate(
-    'deletedProjects',
-    { project: null },
-    processDeletedProjects,
-    {
-      'deleterData.deletedProjectId': 1,
-    }
-  )
-  if (DRY_RUN) {
-    console.log(
-      '\nThis was a dry run. Re-run with DRY_RUN=false to delete broken refs and projects.'
-    )
-  }
-}
-
-async function processDeletedProjects(collection, deletedProjects) {
-  const projectIds = deletedProjects.map(x => x.deleterData.deletedProjectId)
-  const activeProjects = await db.projects
-    .find(
-      { _id: { $in: projectIds } },
-      { projection: { _id: 1, rootFolder: 1 } }
-    )
-    .toArray()
-  for (const activeProject of activeProjects) {
-    await cleanupProject(activeProject)
-  }
-}
-
-async function cleanupProject(project) {
-  const { docIds, fileIds } = findRefsInFolder(project.rootFolder[0])
-  const badDocIds = []
-  const badFileIds = []
-  for (const docId of docIds) {
-    if (!(await docExists(docId))) {
-      badDocIds.push(docId)
-    }
-  }
-  for (const fileId of fileIds) {
-    if (!(await fileExists(project._id, fileId))) {
-      badFileIds.push(fileId)
-    }
-  }
-
-  if (badDocIds.length === 0 && badFileIds.length === 0) {
-    return
-  }
-
-  if (
-    badDocIds.length === docIds.length &&
-    badFileIds.length === fileIds.length
-  ) {
-    console.log(`Deleting project ${project._id}. All refs are bad.`)
-    if (!DRY_RUN) {
-      await db.projects.removeOne({ _id: project._id })
-    }
-    return
-  }
-
-  console.log(`Cleaning project ${project._id}:`)
-  for (const docId of badDocIds) {
-    console.log(` * Deleting bad doc ref ${docId}`)
-    if (!DRY_RUN) {
-      await ProjectEntityMongoUpdateHandler.promises.deleteEntity(
-        project._id,
-        docId,
-        'doc'
-      )
-    }
-  }
-
-  for (const fileId of badFileIds) {
-    console.log(` * Deleting bad file ref ${fileId}`)
-    if (!DRY_RUN) {
-      await ProjectEntityMongoUpdateHandler.promises.deleteEntity(
-        project._id,
-        fileId,
-        'file'
-      )
-    }
-  }
-}
-
-function findRefsInFolder(folder) {
-  let docIds = folder.docs.map(doc => doc._id)
-  let fileIds = folder.fileRefs.map(file => file._id)
-  for (const subfolder of folder.folders) {
-    const subrefs = findRefsInFolder(subfolder)
-    docIds = docIds.concat(subrefs.docIds)
-    fileIds = fileIds.concat(subrefs.fileIds)
-  }
-  return { docIds, fileIds }
-}
-
-async function docExists(docId) {
-  const doc = await db.docs.findOne({ _id: docId })
-  return doc != null
-}
-
-async function fileExists(projectId, fileId) {
-  try {
-    // Getting the file size to avoid downloading the whole file
-    await FileStoreHandler.promises.getFileSize(projectId, fileId)
-  } catch (err) {
-    if (err instanceof Errors.NotFoundError) {
-      return false
-    }
-    throw err
-  }
-  return true
-}

services/web/scripts/delete_dangling_file_refs.js

@@ -0,0 +1,128 @@
+/**
+ * This script deletes dangling doc and file refs in projects
+ */
+
+const minimist = require('minimist')
+const { ObjectId } = require('mongodb')
+
+const { db, waitForDb } = require('../app/src/infrastructure/mongodb')
+const Errors = require('../app/src/Features/Errors/Errors')
+const FileStoreHandler = require('../app/src/Features/FileStore/FileStoreHandler')
+const ProjectEntityMongoUpdateHandler = require('../app/src/Features/Project/ProjectEntityMongoUpdateHandler')
+
+const OPTIONS = parseArgs()
+
+function parseArgs() {
+  const argv = minimist(process.argv.slice(2), {
+    boolean: ['dry-run'],
+    default: { 'dry-run': true },
+  })
+  const dryRun = argv['dry-run']
+  const projectIds = argv._
+  if (projectIds.length === 0) {
+    console.log(`Usage: ${process.argv[1]} [--no-dry-run] PROJECT_ID ...`)
+    process.exit(0)
+  }
+  return { projectIds, dryRun }
+}
+
+async function main() {
+  await waitForDb()
+  const projects = await getProjects()
+
+  for (const project of projects) {
+    await processProject(project)
+  }
+
+  if (OPTIONS.dryRun) {
+    console.log(
+      '\nThis was a dry run. Re-run with --no-dry-run to delete broken refs.'
+    )
+  }
+}
+
+async function getProjects() {
+  const projectIds = OPTIONS.projectIds.map(id => ObjectId(id))
+  const projects = await db.projects
+    .find(
+      { _id: { $in: projectIds } },
+      { projection: { _id: 1, rootFolder: 1 } }
+    )
+    .toArray()
+  return projects
+}
+
+async function processProject(project) {
+  console.log(`Processing project ${project._id}`)
+  const { docIds, fileIds } = findRefsInFolder(project.rootFolder[0])
+  for (const docId of docIds) {
+    if (!(await docExists(docId))) {
+      await deleteDoc(project._id, docId)
+    }
+  }
+  for (const fileId of fileIds) {
+    if (!(await fileExists(project._id, fileId))) {
+      await deleteFile(project._id, fileId)
+    }
+  }
+}
+
+function findRefsInFolder(folder) {
+  let docIds = folder.docs.map(doc => doc._id)
+  let fileIds = folder.fileRefs.map(file => file._id)
+  for (const subfolder of folder.folders) {
+    const subrefs = findRefsInFolder(subfolder)
+    docIds = docIds.concat(subrefs.docIds)
+    fileIds = fileIds.concat(subrefs.fileIds)
+  }
+  return { docIds, fileIds }
+}
+
+async function docExists(docId) {
+  const doc = await db.docs.findOne({ _id: docId })
+  return doc != null
+}
+
+async function fileExists(projectId, fileId) {
+  try {
+    // Getting the file size to avoid downloading the whole file
+    await FileStoreHandler.promises.getFileSize(projectId, fileId)
+  } catch (err) {
+    if (err instanceof Errors.NotFoundError) {
+      return false
+    }
+    throw err
+  }
+  return true
+}
+
+async function deleteDoc(projectId, docId) {
+  console.log(` * Deleting bad doc ref ${docId}`)
+  if (!OPTIONS.dryRun) {
+    await ProjectEntityMongoUpdateHandler.promises.deleteEntity(
+      projectId,
+      docId,
+      'doc'
+    )
+  }
+}
+
+async function deleteFile(projectId, fileId) {
+  console.log(` * Deleting bad file ref ${fileId}`)
+  if (!OPTIONS.dryRun) {
+    await ProjectEntityMongoUpdateHandler.promises.deleteEntity(
+      projectId,
+      fileId,
+      'file'
+    )
+  }
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error({ error })
+    process.exit(1)
+  })