verify_backed_up_blobs.mjs 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. // @ts-check
  2. import { ObjectId } from 'mongodb'
  3. import knex from '../lib/knex.js'
  4. import {
  5. batchedUpdate,
  6. objectIdFromInput,
  7. READ_PREFERENCE_SECONDARY,
  8. } from '@overleaf/mongo-utils/batchedUpdate.js'
  9. import {
  10. GLOBAL_BLOBS,
  11. loadGlobalBlobs,
  12. makeProjectKey,
  13. } from '../lib/blob_store/index.js'
  14. import {
  15. backedUpBlobs as backedUpBlobsCollection,
  16. db,
  17. client,
  18. } from '../lib/mongodb.js'
  19. import commandLineArgs from 'command-line-args'
  20. import fs from 'node:fs'
  21. const projectsCollection = db.collection('projects')
  22. // Enable caching for ObjectId.toString()
  23. ObjectId.cacheHexString = true
  24. function parseArgs() {
  25. const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z')
  26. const args = commandLineArgs([
  27. {
  28. name: 'BATCH_RANGE_START',
  29. type: String,
  30. defaultValue: PUBLIC_LAUNCH_DATE.toISOString(),
  31. },
  32. {
  33. name: 'BATCH_RANGE_END',
  34. type: String,
  35. defaultValue: new Date().toISOString(),
  36. },
  37. {
  38. name: 'output',
  39. type: String,
  40. alias: 'o',
  41. },
  42. ])
  43. const BATCH_RANGE_START = objectIdFromInput(
  44. args['BATCH_RANGE_START']
  45. ).toString()
  46. const BATCH_RANGE_END = objectIdFromInput(args['BATCH_RANGE_END']).toString()
  47. if (!args['output']) {
  48. throw new Error('missing --output')
  49. }
  50. const OUTPUT_STREAM = fs.createWriteStream(args['output'])
  51. return {
  52. BATCH_RANGE_START,
  53. BATCH_RANGE_END,
  54. OUTPUT_STREAM,
  55. }
  56. }
  57. const { BATCH_RANGE_START, BATCH_RANGE_END, OUTPUT_STREAM } = parseArgs()
  58. // We need to handle the start and end differently as ids of deleted projects are created at time of deletion.
  59. if (process.env.BATCH_RANGE_START || process.env.BATCH_RANGE_END) {
  60. throw new Error('use --BATCH_RANGE_START and --BATCH_RANGE_END')
  61. }
  62. let gracefulShutdownInitiated = false
  63. process.on('SIGINT', handleSignal)
  64. process.on('SIGTERM', handleSignal)
  65. function handleSignal() {
  66. gracefulShutdownInitiated = true
  67. console.warn('graceful shutdown initiated, draining queue')
  68. }
  69. async function processBatch(batch) {
  70. if (gracefulShutdownInitiated) {
  71. throw new Error('graceful shutdown: aborting batch processing')
  72. }
  73. const N = batch.length
  74. const firstId = batch[0]._id
  75. const lastId = batch[N - 1]._id
  76. const projectCursor = await projectsCollection.find(
  77. { _id: { $gte: firstId, $lte: lastId } },
  78. {
  79. projection: { _id: 1, 'overleaf.history.id': 1, lastUpdated: 1 },
  80. readPreference: READ_PREFERENCE_SECONDARY,
  81. }
  82. )
  83. const projectMap = new Map()
  84. for await (const project of projectCursor) {
  85. projectMap.set(project._id.toString(), project)
  86. }
  87. for (const project of batch) {
  88. const projectId = project._id.toString()
  89. const projectRecord = projectMap.get(projectId)
  90. if (!projectRecord) {
  91. console.error(`project not found: ${projectId}`)
  92. continue
  93. }
  94. if (!projectRecord.overleaf?.history?.id) {
  95. console.error(`project missing history: ${projectId}`)
  96. continue
  97. }
  98. const historyId = projectRecord.overleaf.history.id.toString()
  99. const prefix = `${projectId},${projectRecord.lastUpdated.toISOString()},`
  100. const hashes = project.blobs.map(blob => blob.toString('hex'))
  101. const projectBlobHashes = hashes.filter(hash => !GLOBAL_BLOBS.has(hash))
  102. if (projectBlobHashes.length < hashes.length) {
  103. console.warn(
  104. `project ${projectId} has ${hashes.length - projectBlobHashes.length} global blobs`
  105. )
  106. }
  107. const rows = projectBlobHashes.map(
  108. hash => prefix + makeProjectKey(historyId, hash) + '\n'
  109. )
  110. OUTPUT_STREAM.write(rows.join(''))
  111. }
  112. }
  113. async function main() {
  114. await loadGlobalBlobs()
  115. OUTPUT_STREAM.write('projectId,lastUpdated,path\n')
  116. await batchedUpdate(
  117. backedUpBlobsCollection,
  118. {},
  119. processBatch,
  120. {},
  121. {},
  122. { BATCH_RANGE_START, BATCH_RANGE_END }
  123. )
  124. }
  125. main()
  126. .then(() => console.log('Done.'))
  127. .catch(err => {
  128. console.error('Error:', err)
  129. process.exitCode = 1
  130. })
  131. .finally(() => {
  132. knex.destroy().catch(err => {
  133. console.error('Error closing Postgres connection:', err)
  134. })
  135. client.close().catch(err => console.error('Error closing MongoDB:', err))
  136. })