Overleaf/services/document-updater/scripts/flush_track_changes.js

// Flush docs found in docsWithHistory sets in the project-history redis.
// We normally write docsWithHistory in the main redis, but some keys
// got into the project-history redis.
// Usage:  DRY_RUN=false node scripts/flush_track_changes.js

const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const rclientProjectHistory = require('@overleaf/redis-wrapper').createClient(
  Settings.redis.history
)
const ProjectFlusher = require('../app/js/ProjectFlusher')
const keys = Settings.redis.history.key_schema
const request = require('request')
const util = require('util')

async function findAndProcessDocs(options) {
  logger.info({ options }, 'flushing docs with pending track-changes')
  let cursor = 0
  do {
    const [newCursor, projectKeys] = await rclientProjectHistory.scan(
      cursor,
      'MATCH',
      keys.docsWithHistoryOps({ project_id: '*' }),
      'COUNT',
      options.limit
    )
    await flushProjects(projectKeys, options)
    cursor = newCursor
  } while (cursor !== '0')
}

async function flushProjects(projectKeys, options) {
  const projectIds = ProjectFlusher._extractIds(projectKeys)
  for (const projectId of projectIds) {
    const docIds = await rclientProjectHistory.smembers(
      keys.docsWithHistoryOps({ project_id: projectId })
    )
    logger.debug({ projectId, docIds }, 'checking docs')
    if (docIds) {
      for (const docId of docIds) {
        try {
          await flushDoc(projectId, docId, options)
        } catch (err) {
          logger.error({ projectId, docId, err }, 'error flushing doc')
        }
      }
    }
  }
}

function makeRequest(url, callback) {
  request.post(url, function (err, res, body) {
    if (err) {
      return callback(err)
    } else {
      return callback(err, { res, body })
    }
  })
}
const makeRequestPromise = util.promisify(makeRequest)

async function flushDoc(projectId, docId, options) {
  const url = `${Settings.apis.trackchanges.url}/project/${projectId}/doc/${docId}/flush`
  logger.debug(
    { projectId, docId, url, options },
    'flushing doc in track changes api'
  )
  if (options.dryRun) {
    return
  }
  const { res, body } = await makeRequestPromise(url)
  if (res && res.statusCode === 204) {
    logger.debug(
      { projectId, docId, statusCode: res.statusCode },
      'successful flush in track changes api'
    )
    // on a successful flush the doc id is removed from docsWithHistoryOps by track-changes
    // in deleteAppliedDocUpdates BUT it will do it for the main redis, not the project history redis.
    // So we need to remove the docId here:
    await rclientProjectHistory.srem(
      keys.docsWithHistoryOps({ project_id: projectId }),
      docId
    )
  } else {
    logger.debug(
      { projectId, docId, res, body },
      'problem flushing in track changes api'
    )
  }
}
findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
  .then(result => {
    rclientProjectHistory.quit()
    console.log('DONE')
  })
  .catch(function (error) {
    console.error(error)
    process.exit(1)
  })