Merge pull request #8551 from overleaf/bg-docupdater-flush-script-for-track-changes

add script to flush to track-changes data from project-history redis

GitOrigin-RevId: c2cad02e5b167528d32061984fa143bbe331a32c

services/document-updater/scripts/flush_track_changes.js

@@ -0,0 +1,99 @@
+// Flush docs found in docsWithHistory sets in the project-history redis.
+// We normally write docsWithHistory in the main redis, but some keys
+// got into the project-history redis.
+// Usage:  DRY_RUN=false node scripts/flush_track_changes.js
+
+const Settings = require('@overleaf/settings')
+const logger = require('@overleaf/logger')
+const rclientProjectHistory = require('@overleaf/redis-wrapper').createClient(
+  Settings.redis.documentupdater
+)
+const ProjectFlusher = require('../app/js/ProjectFlusher')
+const keys = Settings.redis.history.key_schema
+const request = require('request')
+const util = require('util')
+
+async function findAndProcessDocs(options) {
+  logger.info({ options }, 'flushing docs with pending track-changes')
+  let cursor = 0
+  do {
+    const [newCursor, projectKeys] = await rclientProjectHistory.scan(
+      cursor,
+      'MATCH',
+      keys.docsWithHistoryOps({ project_id: '*' }),
+      'COUNT',
+      options.limit
+    )
+    await flushProjects(projectKeys, options)
+    cursor = newCursor
+  } while (cursor !== '0')
+}
+
+async function flushProjects(projectKeys, options) {
+  const projectIds = ProjectFlusher._extractIds(projectKeys)
+  for (const projectId of projectIds) {
+    const docIds = await rclientProjectHistory.smembers(
+      keys.docsWithHistoryOps({ project_id: projectId })
+    )
+    logger.debug({ projectId, docIds }, 'checking docs')
+    if (docIds) {
+      for (const docId of docIds) {
+        try {
+          await flushDoc(projectId, docId, options)
+        } catch (err) {
+          logger.error({ projectId, docId, err }, 'error flushing doc')
+        }
+      }
+    }
+  }
+}
+
+function makeRequest(url, callback) {
+  request.post(url, function (err, res, body) {
+    if (err) {
+      return callback(err)
+    } else {
+      return callback(err, { res, body })
+    }
+  })
+}
+const makeRequestPromise = util.promisify(makeRequest)
+
+async function flushDoc(projectId, docId, options) {
+  const url = `${Settings.apis.trackchanges.url}/project/${projectId}/doc/${docId}/flush`
+  logger.debug(
+    { projectId, docId, url, options },
+    'flushing doc in track changes api'
+  )
+  if (options.dryRun) {
+    return
+  }
+  const { res, body } = await makeRequestPromise(url)
+  if (res && res.statusCode === 204) {
+    logger.debug(
+      { projectId, docId, statusCode: res.statusCode },
+      'successful flush in track changes api'
+    )
+    // on a successful flush the doc id is removed from docsWithHistoryOps by track-changes
+    // in deleteAppliedDocUpdates BUT it will do it for the main redis, not the project history redis.
+    // So we need to remove the docId here:
+    await rclientProjectHistory.srem(
+      keys.docsWithHistoryOps({ project_id: projectId }),
+      docId
+    )
+  } else {
+    logger.debug(
+      { projectId, docId, res, body },
+      'problem flushing in track changes api'
+    )
+  }
+}
+findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
+  .then(result => {
+    rclientProjectHistory.quit()
+    console.log('DONE')
+  })
+  .catch(function (error) {
+    console.error(error)
+    process.exit(1)
+  })