backupVerifier.test.mjs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375
  1. // @ts-check
  2. import cleanup from '../storage/support/cleanup.js'
  3. import fetch from 'node-fetch'
  4. import testServer from './support/test_backup_verifier_server.mjs'
  5. import { expect } from 'chai'
  6. import testProjects from './support/test_projects.js'
  7. import {
  8. backupPersistor,
  9. chunksBucket,
  10. projectBlobsBucket,
  11. } from '../../../../storage/lib/backupPersistor.mjs'
  12. import {
  13. BlobStore,
  14. makeProjectKey,
  15. } from '../../../../storage/lib/blob_store/index.js'
  16. import Stream from 'node:stream'
  17. import * as zlib from 'node:zlib'
  18. import { promisify } from 'node:util'
  19. import { execFile } from 'node:child_process'
  20. import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
  21. import { chunkStore } from '../../../../storage/index.js'
  22. import { Change, File, Operation } from 'overleaf-editor-core'
  23. import Crypto from 'node:crypto'
  24. import path from 'node:path'
  25. import projectKey from '../../../../storage/lib/project_key.js'
  26. import { historyStore } from '../../../../storage/lib/history_store.js'
  27. /**
  28. * @typedef {import("node-fetch").Response} Response
  29. * @typedef {import("overleaf-editor-core").Blob} Blob
  30. */
  31. async function verifyProjectScript(historyId, expectFail = true) {
  32. try {
  33. const result = await promisify(execFile)(
  34. process.argv0,
  35. ['storage/scripts/verify_project.mjs', `--historyId=${historyId}`],
  36. {
  37. encoding: 'utf-8',
  38. timeout: 5_000,
  39. env: {
  40. ...process.env,
  41. LOG_LEVEL: 'warn',
  42. },
  43. }
  44. )
  45. return { status: 0, stdout: result.stdout, stderr: result.stderr }
  46. } catch (err) {
  47. if (
  48. err &&
  49. typeof err === 'object' &&
  50. 'stdout' in err &&
  51. 'code' in err &&
  52. 'stderr' in err
  53. ) {
  54. if (!expectFail) {
  55. console.log(err)
  56. }
  57. return {
  58. stdout: typeof err.stdout === 'string' ? err.stdout : '',
  59. status: typeof err.code === 'number' ? err.code : -1,
  60. stderr: typeof err.stdout === 'string' ? err.stderr : '',
  61. }
  62. }
  63. throw err
  64. }
  65. }
  66. /**
  67. * @param {string} historyId
  68. * @param {string} hash
  69. * @return {Promise<{stdout: string, status:number }>}
  70. */
  71. async function verifyBlobScript(historyId, hash, expectFail = true) {
  72. try {
  73. const result = await promisify(execFile)(
  74. process.argv0,
  75. [
  76. 'storage/scripts/verify_backup_blob.mjs',
  77. `--historyId=${historyId}`,
  78. hash,
  79. ],
  80. {
  81. encoding: 'utf-8',
  82. timeout: 5_000,
  83. env: {
  84. ...process.env,
  85. LOG_LEVEL: 'warn',
  86. },
  87. }
  88. )
  89. return { status: 0, stdout: result.stdout }
  90. } catch (err) {
  91. if (err && typeof err === 'object' && 'stdout' in err && 'code' in err) {
  92. if (!expectFail) {
  93. console.log(err)
  94. }
  95. return {
  96. stdout: typeof err.stdout === 'string' ? err.stdout : '',
  97. status: typeof err.code === 'number' ? err.code : -1,
  98. }
  99. }
  100. throw err
  101. }
  102. }
  103. /**
  104. * @param {string} historyId
  105. * @param {string} hash
  106. * @return {Promise<Response>}
  107. */
  108. async function verifyBlobHTTP(historyId, hash) {
  109. return await fetch(
  110. testServer.testUrl(`/history/${historyId}/blob/${hash}/verify`),
  111. { method: 'GET' }
  112. )
  113. }
  114. async function backupChunk(historyId) {
  115. const newChunk = await chunkStore.loadLatestRaw(historyId)
  116. const { buffer: chunkBuffer } = await historyStore.loadRawWithBuffer(
  117. historyId,
  118. newChunk.id
  119. )
  120. const md5 = Crypto.createHash('md5').update(chunkBuffer)
  121. await backupPersistor.sendStream(
  122. chunksBucket,
  123. path.join(
  124. projectKey.format(historyId),
  125. projectKey.pad(newChunk.startVersion)
  126. ),
  127. Stream.Readable.from([chunkBuffer]),
  128. {
  129. contentType: 'application/json',
  130. contentEncoding: 'gzip',
  131. contentLength: chunkBuffer.byteLength,
  132. sourceMd5: md5.digest('hex'),
  133. }
  134. )
  135. }
  136. const FIFTEEN_MINUTES_IN_MS = 900_000
  137. async function addFileInNewChunk(
  138. fileContents,
  139. filePath,
  140. historyId,
  141. { creationDate = new Date() }
  142. ) {
  143. const chunk = await chunkStore.loadLatest(historyId)
  144. const operation = Operation.addFile(
  145. `${historyId}.txt`,
  146. File.fromString(fileContents)
  147. )
  148. const changes = [new Change([operation], creationDate, [])]
  149. chunk.pushChanges(changes)
  150. await chunkStore.update(historyId, 0, chunk)
  151. }
  152. /**
  153. * @param {string} historyId
  154. * @param {Object} [backup]
  155. * @return {Promise<string>}
  156. */
  157. async function prepareProjectAndBlob(
  158. historyId,
  159. { shouldBackupBlob, shouldBackupChunk, shouldCreateChunk } = {
  160. shouldBackupBlob: true,
  161. shouldBackupChunk: true,
  162. shouldCreateChunk: true,
  163. }
  164. ) {
  165. await testProjects.createEmptyProject(historyId)
  166. const blobStore = new BlobStore(historyId)
  167. const fileContents = historyId
  168. const blob = await blobStore.putString(fileContents)
  169. if (shouldCreateChunk) {
  170. await addFileInNewChunk(fileContents, `${historyId}.txt`, historyId, {
  171. creationDate: new Date(new Date().getTime() - FIFTEEN_MINUTES_IN_MS),
  172. })
  173. }
  174. if (shouldBackupBlob) {
  175. const gzipped = zlib.gzipSync(Buffer.from(historyId))
  176. await backupPersistor.sendStream(
  177. projectBlobsBucket,
  178. makeProjectKey(historyId, blob.getHash()),
  179. Stream.Readable.from([gzipped]),
  180. { contentLength: gzipped.byteLength, contentEncoding: 'gzip' }
  181. )
  182. await checkDEKExists(historyId)
  183. }
  184. if (shouldCreateChunk && shouldBackupChunk) {
  185. await backupChunk(historyId)
  186. }
  187. return blob.getHash()
  188. }
  189. /**
  190. * @param {string} historyId
  191. * @return {Promise<void>}
  192. */
  193. async function checkDEKExists(historyId) {
  194. await backupPersistor.forProjectRO(
  195. projectBlobsBucket,
  196. makeProjectKey(historyId, '')
  197. )
  198. }
  199. describe('backupVerifier', function () {
  200. const historyIdPostgres = '42'
  201. const historyIdMongo = '000000000000000000000042'
  202. let blobHashPG, blobHashMongo, blobPathPG
  203. beforeEach(cleanup.everything)
  204. beforeEach('create health check projects', async function () {
  205. ;[blobHashPG, blobHashMongo] = await Promise.all([
  206. prepareProjectAndBlob('42'),
  207. prepareProjectAndBlob('000000000000000000000042'),
  208. ])
  209. blobPathPG = makeProjectKey(historyIdPostgres, blobHashPG)
  210. })
  211. beforeEach(testServer.listenOnRandomPort)
  212. it('renders 200 on /status', async function () {
  213. const response = await fetch(testServer.testUrl('/status'))
  214. expect(response.status).to.equal(200)
  215. })
  216. it('renders 200 on /health_check', async function () {
  217. const response = await fetch(testServer.testUrl('/health_check'))
  218. expect(response.status).to.equal(200)
  219. })
  220. describe('storage/scripts/verify_project.mjs', function () {
  221. describe('when the project is appropriately backed up', function () {
  222. it('should return 0', async function () {
  223. const response = await verifyProjectScript(historyIdPostgres, false)
  224. expect(response.status).to.equal(0)
  225. })
  226. })
  227. describe('when the project chunk is not backed up', function () {
  228. let response
  229. beforeEach(async function () {
  230. await prepareProjectAndBlob('000000000000000000000043', {
  231. shouldBackupChunk: false,
  232. shouldBackupBlob: true,
  233. shouldCreateChunk: true,
  234. })
  235. response = await verifyProjectScript('000000000000000000000043')
  236. })
  237. it('should return 1', async function () {
  238. expect(response.status).to.equal(1)
  239. })
  240. it('should emit an error message referring to a missing chunk', async function () {
  241. const stderr = response.stderr
  242. expect(stderr).to.include('BackupRPOViolationChunkNotBackedUpError')
  243. })
  244. })
  245. describe('when a project blob is not backed up', function () {
  246. let response
  247. beforeEach(async function () {
  248. await prepareProjectAndBlob('43', {
  249. shouldBackupChunk: true,
  250. shouldBackupBlob: false,
  251. shouldCreateChunk: true,
  252. })
  253. response = await verifyProjectScript('43')
  254. })
  255. it('should return 1', function () {
  256. expect(response.status).to.equal(1)
  257. })
  258. it('includes a BackupCorruptedError in stderr', function () {
  259. expect(response.stderr).to.include(
  260. 'BackupCorruptedMissingBlobError: missing blob'
  261. )
  262. })
  263. })
  264. })
  265. describe('storage/scripts/verify_backup_blob.mjs', function () {
  266. it('throws and does not create DEK if missing', async function () {
  267. const historyId = '404'
  268. const hash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
  269. const response = await verifyBlobScript(historyId, hash)
  270. expect(response.status).to.equal(1)
  271. expect(response.stdout).to.include('dek does not exist')
  272. await expect(checkDEKExists(historyId)).to.be.rejectedWith(NotFoundError)
  273. })
  274. it('throws when deleted in db', async function () {
  275. const blobStore = new BlobStore(historyIdPostgres)
  276. await blobStore.deleteBlobs()
  277. const response = await verifyBlobScript(historyIdPostgres, blobHashPG)
  278. expect(response.status).to.equal(1)
  279. expect(response.stdout).to.include(`blob ${blobHashPG} not found`)
  280. })
  281. it('throws when not existing', async function () {
  282. await backupPersistor.deleteObject(projectBlobsBucket, blobPathPG)
  283. const result = await verifyBlobScript(historyIdPostgres, blobHashPG)
  284. expect(result.status).to.equal(1)
  285. expect(result.stdout).to.include('missing blob')
  286. })
  287. it('throws when corrupted', async function () {
  288. await backupPersistor.sendStream(
  289. projectBlobsBucket,
  290. blobPathPG,
  291. Stream.Readable.from(['something else']),
  292. { contentLength: 14 }
  293. )
  294. const result = await verifyBlobScript(historyIdPostgres, blobHashPG)
  295. expect(result.status).to.equal(1)
  296. expect(result.stdout).to.include('hash mismatch for backed up blob')
  297. })
  298. it('should successfully verify from postgres', async function () {
  299. const result = await verifyBlobScript(
  300. historyIdPostgres,
  301. blobHashPG,
  302. false
  303. )
  304. expect(result.status).to.equal(0)
  305. expect(result.stdout.split('\n')).to.include('OK')
  306. })
  307. it('should successfully verify from mongo', async function () {
  308. const result = await verifyBlobScript(
  309. historyIdMongo,
  310. blobHashMongo,
  311. false
  312. )
  313. expect(result.status).to.equal(0)
  314. expect(result.stdout.split('\n')).to.include('OK')
  315. })
  316. })
  317. describe('GET /history/:historyId/blob/:hash/verify', function () {
  318. it('returns 404 when deleted in db', async function () {
  319. const blobStore = new BlobStore(historyIdPostgres)
  320. await blobStore.deleteBlobs()
  321. const response = await verifyBlobHTTP(historyIdPostgres, blobHashPG)
  322. expect(response.status).to.equal(404)
  323. expect(await response.text()).to.equal(`blob ${blobHashPG} not found`)
  324. })
  325. it('returns 422 and does not create DEK if missing', async function () {
  326. const historyId = '404'
  327. const hash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
  328. const response = await verifyBlobHTTP(historyId, hash)
  329. expect(response.status).to.equal(422)
  330. expect(await response.text()).to.equal('dek does not exist')
  331. await expect(checkDEKExists(historyId)).to.be.rejectedWith(NotFoundError)
  332. })
  333. it('returns 422 when not existing', async function () {
  334. await backupPersistor.deleteObject(projectBlobsBucket, blobPathPG)
  335. const response = await verifyBlobHTTP(historyIdPostgres, blobHashPG)
  336. expect(response.status).to.equal(422)
  337. expect(await response.text()).to.equal('missing blob')
  338. })
  339. it('returns 422 when corrupted', async function () {
  340. await backupPersistor.sendStream(
  341. projectBlobsBucket,
  342. blobPathPG,
  343. Stream.Readable.from(['something else']),
  344. { contentLength: 14 }
  345. )
  346. const response = await verifyBlobHTTP(historyIdPostgres, blobHashPG)
  347. expect(response.status).to.equal(422)
  348. expect(await response.text()).to.equal('hash mismatch for backed up blob')
  349. })
  350. it('should successfully verify from postgres', async function () {
  351. const response = await verifyBlobHTTP(historyIdPostgres, blobHashPG)
  352. expect(response.status).to.equal(200)
  353. })
  354. it('should successfully verify from mongo', async function () {
  355. const response = await verifyBlobHTTP(historyIdMongo, blobHashMongo)
  356. expect(response.status).to.equal(200)
  357. })
  358. })
  359. })