overleaf/services/history-v1/storage/scripts/check_chunk.mjs

import commandLineArgs from 'command-line-args'
import {
  loadAtVersion,
  getProjectChunksFromVersion,
} from '../lib/chunk_store/index.js'
import { client } from '../lib/mongodb.js'
import knex from '../lib/knex.js'
import redis from '../lib/redis.js'
import { loadGlobalBlobs, BlobStore } from '../lib/blob_store/index.js'
import { getContentHash } from '../lib/content_hash.js'
import core from 'overleaf-editor-core'
import Events from 'node:events'
Events.setMaxListeners(20)

const { StringFileData, LazyStringFileData } = core

const optionDefinitions = [
  { name: 'historyId', alias: 'p', type: String },
  { name: 'version', alias: 'v', type: Number },
  { name: 'persistedOnly', alias: 'o', type: Boolean },
]

async function ensureFileLoaded(file, blobStore, currentVersion, path) {
  if (
    typeof file.load === 'function' &&
    file.data instanceof LazyStringFileData
  ) {
    if (file.data.rangesHash) {
      console.log(
        `Loading rangesHash ${file.data.rangesHash} for ${path} at version ${currentVersion}`
      )
    } else {
      console.log(
        `No rangesHash found for ${path} at version ${currentVersion}`
      )
    }
    await file.load('eager', blobStore)
    console.log('=> file', file.toRaw())
  }
}

function checkFileTrackedChanges(path, file, currentVersion) {
  let violations = false
  const positions = []
  if (!(file.data instanceof StringFileData)) {
    return { violations, positions }
  }

  let tcList
  try {
    tcList = file.getTrackedChanges()
  } catch (err) {
    return { violations, positions }
  }

  if (!tcList) return { violations, positions }

  const changesArr = Array.from(tcList)
  let prevTc = null

  for (const tc of changesArr) {
    positions.push(`(${tc.range.start}, ${tc.range.end})`)
    if (prevTc) {
      if (prevTc.range.start > tc.range.start) {
        console.error(
          `VIOLATION: Unsorted ranges in ${path} at version ${currentVersion}: [${prevTc.range.start}, ${prevTc.range.end}] comes before [${tc.range.start}, ${tc.range.end}]`
        )
        violations = true
      }
      if (prevTc.range.overlaps(tc.range)) {
        console.error(
          `VIOLATION: Overlapping ranges in ${path} at version ${currentVersion}: [${prevTc.range.start}, ${prevTc.range.end}] overlaps [${tc.range.start}, ${tc.range.end}]`
        )
        violations = true
      }
    }
    prevTc = tc
  }

  return { violations, positions }
}

async function checkSnapshot(snapshot, blobStore, currentVersion) {
  let containsViolations = false
  const pathnames = snapshot.getFilePathnames()
  const diagnostics = []

  for (const path of pathnames) {
    const file = snapshot.getFile(path)
    if (!file) continue

    try {
      await ensureFileLoaded(file, blobStore, currentVersion, path)
    } catch (err) {
      console.error(
        `Failed to load file ${path} at version ${currentVersion}:`,
        err
      )
      continue
    }

    const { violations, positions } = checkFileTrackedChanges(
      path,
      file,
      currentVersion
    )
    if (violations) containsViolations = true
    if (positions.length > 0) {
      diagnostics.push(`  ${path}: changes at [${positions.join(', ')}]`)
    }
  }

  if (diagnostics.length > 0) {
    console.log(`Version ${currentVersion} tracked changes summary:`)
    console.log(diagnostics.join('\n'))
  }

  return containsViolations
}

async function validateContentHash(
  operation,
  snapshot,
  currentVersion,
  blobStore
) {
  if (operation instanceof core.EditFileOperation) {
    const editOperation = operation.getOperation()
    if (
      editOperation instanceof core.TextOperation &&
      editOperation.contentHash != null
    ) {
      const path = operation.getPathname()
      const file = snapshot.getFile(path)
      if (file == null) {
        console.error(
          `VIOLATION: file ${path} not found for hash validation at version ${currentVersion}`
        )
        return true
      }
      try {
        await ensureFileLoaded(file, blobStore, currentVersion, path)
      } catch (err) {
        console.error(
          `Failed to load file ${path} for hash validation at version ${currentVersion}:`,
          err
        )
        return true
      }
      const content = file.getContent({ filterTrackedDeletes: true })
      const expectedHash = editOperation.contentHash
      const actualHash = content != null ? getContentHash(content) : null

      if (actualHash !== expectedHash) {
        console.error(
          `VIOLATION: content hash mismatch in ${path} at version ${currentVersion}: expected ${expectedHash}, got ${actualHash}`
        )
        return true
      }
    }
  }
  return false
}

async function checkChunkChanges(historyId, chunk) {
  const snapshot = chunk.getSnapshot().clone()
  const blobStore = new BlobStore(historyId)

  const changes = chunk.getChanges()
  let currentVersion = chunk.getStartVersion()
  console.log(
    `Checking chunk starting at version ${currentVersion} with ${changes.length} changes.`
  )

  const initialViolations = await checkSnapshot(
    snapshot,
    blobStore,
    currentVersion
  )
  if (initialViolations) {
    console.log(
      `Tracked changes corrupted at initial snapshot version ${currentVersion}`
    )
  }

  for (const change of changes) {
    currentVersion += 1
    let localViolations = false
    if (change?.origin?.kind === 'history-resync') {
      console.log('-'.repeat(16), 'history-resync', '-'.repeat(16))
    }
    console.log(
      `Version ${currentVersion} change:`,
      JSON.stringify(change.toRaw())
    )
    if (change?.origin?.kind === 'history-resync') {
      process.exit()
    }

    try {
      for (const _operation of change.iterativelyApplyTo(snapshot, {
        strict: true,
      })) {
        console.log(
          `Version ${currentVersion} operation:`,
          JSON.stringify(_operation.toRaw())
        )
        const hashErr = await validateContentHash(
          _operation,
          snapshot,
          currentVersion,
          blobStore
        )
        if (hashErr) localViolations = true
      }
    } catch (err) {
      console.error(`Failed to apply change at version ${currentVersion}:`, err)
      continue
    }

    const snapViolations = await checkSnapshot(
      snapshot,
      blobStore,
      currentVersion
    )
    if (snapViolations) localViolations = true

    if (localViolations) {
      console.log(
        `Tracked changes corrupted or hash mismatch at version ${currentVersion}`
      )
      console.log('Change was:', JSON.stringify(change.toRaw(), null, 2))
    }
  }
}

async function main() {
  const options = commandLineArgs(optionDefinitions)
  const { historyId, version, persistedOnly } = options

  if (!historyId) {
    console.error('Error: --historyId is required.')
    process.exit(1)
  }

  await loadGlobalBlobs()

  if (version != null) {
    const chunk = await loadAtVersion(historyId, version, {
      persistedOnly: persistedOnly || false,
    })
    if (!chunk) {
      console.error(`Chunk not found at version ${version}`)
      process.exit(1)
    }
    await checkChunkChanges(historyId, chunk)
  } else {
    let checkedAny = false
    for await (const chunkRecord of getProjectChunksFromVersion(historyId, 0)) {
      const chunk = await loadAtVersion(historyId, chunkRecord.startVersion, {
        persistedOnly: persistedOnly || false,
      })
      if (chunk) {
        checkedAny = true
        await checkChunkChanges(historyId, chunk)
      } else {
        console.error(
          `Failed to load chunk starting at ${chunkRecord.startVersion}`
        )
      }
    }
    if (!checkedAny) {
      console.log(`No chunks found for project ${historyId}`)
    }
  }
}

main()
  .then(() => console.log('Done.'))
  .catch(err => {
    console.error('Error:', err)
    process.exit(1)
  })
  .finally(() => {
    knex.destroy().catch(err => console.error('Error closing Postgres:', err))
    client.close().catch(err => console.error('Error closing MongoDB:', err))
    redis
      .disconnect()
      .catch(err => console.error('Error disconnecting Redis:', err))
  })