mongo.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559
  1. // @ts-check
  2. const { ObjectId, ReadPreference, MongoError } = require('mongodb')
  3. const { Chunk } = require('overleaf-editor-core')
  4. const OError = require('@overleaf/o-error')
  5. const config = require('config')
  6. const assert = require('../assert')
  7. const mongodb = require('../mongodb')
  8. const { ChunkVersionConflictError } = require('./errors')
  9. const DUPLICATE_KEY_ERROR_CODE = 11000
  10. /**
  11. * @import { ClientSession } from 'mongodb'
  12. */
  13. /**
  14. * Get the latest chunk's metadata from the database
  15. * @param {string} projectId
  16. * @param {Object} [opts]
  17. * @param {boolean} [opts.readOnly]
  18. */
  19. async function getLatestChunk(projectId, opts = {}) {
  20. assert.mongoId(projectId, 'bad projectId')
  21. const { readOnly = false } = opts
  22. const record = await mongodb.chunks.findOne(
  23. {
  24. projectId: new ObjectId(projectId),
  25. state: { $in: ['active', 'closed'] },
  26. },
  27. {
  28. sort: { startVersion: -1 },
  29. readPreference: readOnly
  30. ? ReadPreference.secondaryPreferred
  31. : ReadPreference.primary,
  32. }
  33. )
  34. if (record == null) {
  35. return null
  36. }
  37. return chunkFromRecord(record)
  38. }
  39. /**
  40. * Get the metadata for the chunk that contains the given version.
  41. *
  42. * @param {string} projectId
  43. * @param {number} version
  44. * @param {object} [opts]
  45. * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
  46. * two chunks, return the newer chunk.
  47. */
  48. async function getChunkForVersion(projectId, version, opts = {}) {
  49. assert.mongoId(projectId, 'bad projectId')
  50. assert.integer(version, 'bad version')
  51. const record = await mongodb.chunks.findOne(
  52. {
  53. projectId: new ObjectId(projectId),
  54. state: { $in: ['active', 'closed'] },
  55. startVersion: { $lte: version },
  56. endVersion: { $gte: version },
  57. },
  58. { sort: { startVersion: opts.preferNewer ? -1 : 1 } }
  59. )
  60. if (record == null) {
  61. throw new Chunk.VersionNotFoundError(projectId, version)
  62. }
  63. return chunkFromRecord(record)
  64. }
  65. /**
  66. * Get the metadata for the chunk that contains the given version before the endTime.
  67. */
  68. async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
  69. assert.mongoId(projectId, 'bad projectId')
  70. assert.date(timestamp, 'bad timestamp')
  71. const recordActive = await getChunkForVersion(projectId, 0)
  72. if (recordActive && recordActive.endTimestamp <= timestamp) {
  73. return recordActive
  74. }
  75. // fallback to deleted chunk
  76. const recordDeleted = await mongodb.chunks.findOne(
  77. {
  78. projectId: new ObjectId(projectId),
  79. state: 'deleted',
  80. startVersion: 0,
  81. updatedAt: { $lte: timestamp }, // indexed for state=deleted
  82. endTimestamp: { $lte: timestamp },
  83. },
  84. { sort: { updatedAt: -1 } }
  85. )
  86. if (recordDeleted) {
  87. return chunkFromRecord(recordDeleted)
  88. }
  89. throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
  90. }
  91. /**
  92. * Get the metadata for the chunk that contains the version that was current at
  93. * the given timestamp.
  94. */
  95. async function getChunkForTimestamp(projectId, timestamp) {
  96. assert.mongoId(projectId, 'bad projectId')
  97. assert.date(timestamp, 'bad timestamp')
  98. const record = await mongodb.chunks.findOne(
  99. {
  100. projectId: new ObjectId(projectId),
  101. state: { $in: ['active', 'closed'] },
  102. endTimestamp: { $gte: timestamp },
  103. },
  104. // We use the index on the startVersion for sorting records. This assumes
  105. // that timestamps go up with each version.
  106. { sort: { startVersion: 1 } }
  107. )
  108. if (record == null) {
  109. // Couldn't find a chunk that had modifications after the given timestamp.
  110. // Fetch the latest chunk instead.
  111. const chunk = await getLatestChunk(projectId)
  112. if (chunk == null) {
  113. throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
  114. }
  115. return chunk
  116. }
  117. return chunkFromRecord(record)
  118. }
  119. /**
  120. * Get the metadata for the chunk that contains the version that was current before
  121. * the given timestamp.
  122. */
  123. async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
  124. assert.mongoId(projectId, 'bad projectId')
  125. assert.date(timestamp, 'bad timestamp')
  126. const record = await mongodb.chunks.findOne(
  127. {
  128. projectId: new ObjectId(projectId),
  129. state: { $in: ['active', 'closed'] },
  130. $or: [
  131. {
  132. endTimestamp: {
  133. $lte: timestamp,
  134. },
  135. },
  136. {
  137. endTimestamp: null,
  138. },
  139. ],
  140. },
  141. // We use the index on the startVersion for sorting records. This assumes
  142. // that timestamps go up with each version.
  143. { sort: { startVersion: -1 } }
  144. )
  145. if (record == null) {
  146. throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
  147. }
  148. return chunkFromRecord(record)
  149. }
  150. /**
  151. * Get all of a project's chunk ids
  152. */
  153. async function getProjectChunkIds(projectId) {
  154. assert.mongoId(projectId, 'bad projectId')
  155. const cursor = mongodb.chunks.find(
  156. {
  157. projectId: new ObjectId(projectId),
  158. state: { $in: ['active', 'closed'] },
  159. },
  160. { projection: { _id: 1 } }
  161. )
  162. return await cursor.map(record => record._id).toArray()
  163. }
  164. /**
  165. * Get all of a projects chunks directly
  166. */
  167. async function getProjectChunks(projectId) {
  168. assert.mongoId(projectId, 'bad projectId')
  169. const cursor = mongodb.chunks
  170. .find(
  171. {
  172. projectId: new ObjectId(projectId),
  173. state: { $in: ['active', 'closed'] },
  174. },
  175. { projection: { state: 0 } }
  176. )
  177. .sort({ startVersion: 1 })
  178. return await cursor.map(chunkFromRecord).toArray()
  179. }
  180. /**
  181. * Insert a pending chunk before sending it to object storage.
  182. */
  183. async function insertPendingChunk(projectId, chunk) {
  184. assert.mongoId(projectId, 'bad projectId')
  185. assert.instance(chunk, Chunk, 'bad chunk')
  186. const chunkId = new ObjectId()
  187. await mongodb.chunks.insertOne({
  188. _id: chunkId,
  189. projectId: new ObjectId(projectId),
  190. startVersion: chunk.getStartVersion(),
  191. endVersion: chunk.getEndVersion(),
  192. endTimestamp: chunk.getEndTimestamp(),
  193. state: 'pending',
  194. updatedAt: new Date(),
  195. })
  196. return chunkId.toString()
  197. }
  198. /**
  199. * Record that a new chunk was created.
  200. *
  201. * @param {string} projectId
  202. * @param {Chunk} chunk
  203. * @param {string} chunkId
  204. * @param {object} opts
  205. * @param {Date} [opts.earliestChangeTimestamp]
  206. * @param {string} [opts.oldChunkId]
  207. */
  208. async function confirmCreate(projectId, chunk, chunkId, opts = {}) {
  209. assert.mongoId(projectId, 'bad projectId')
  210. assert.instance(chunk, Chunk, 'bad newChunk')
  211. assert.mongoId(chunkId, 'bad newChunkId')
  212. await mongodb.client.withSession(async session => {
  213. await session.withTransaction(async () => {
  214. if (opts.oldChunkId != null) {
  215. await closeChunk(projectId, opts.oldChunkId, { session })
  216. }
  217. await activateChunk(projectId, chunkId, { session })
  218. await updateProjectRecord(
  219. projectId,
  220. chunk,
  221. opts.earliestChangeTimestamp,
  222. { session }
  223. )
  224. })
  225. })
  226. }
  227. /**
  228. * Write the metadata to the project record
  229. */
  230. async function updateProjectRecord(
  231. projectId,
  232. chunk,
  233. earliestChangeTimestamp,
  234. mongoOpts = {}
  235. ) {
  236. if (!config.has('backupStore')) {
  237. return
  238. }
  239. // record the end version against the project
  240. await mongodb.projects.updateOne(
  241. {
  242. 'overleaf.history.id': projectId, // string for Object ids, number for postgres ids
  243. },
  244. {
  245. // always store the latest end version and timestamp for the chunk
  246. $max: {
  247. 'overleaf.history.currentEndVersion': chunk.getEndVersion(),
  248. 'overleaf.history.currentEndTimestamp': chunk.getEndTimestamp(),
  249. 'overleaf.history.updatedAt': new Date(),
  250. },
  251. // store the first pending change timestamp for the chunk, this will
  252. // be cleared every time a backup is completed.
  253. $min: {
  254. 'overleaf.backup.pendingChangeAt':
  255. earliestChangeTimestamp || chunk.getEndTimestamp() || new Date(),
  256. },
  257. },
  258. mongoOpts
  259. )
  260. }
  261. /**
  262. * @param {number} historyId
  263. * @return {Promise<string>}
  264. */
  265. async function lookupMongoProjectIdFromHistoryId(historyId) {
  266. const project = await mongodb.projects.findOne(
  267. // string for Object ids, number for postgres ids
  268. { 'overleaf.history.id': historyId },
  269. { projection: { _id: 1 } }
  270. )
  271. if (!project) {
  272. // should not happen: We flush before allowing a project to be soft-deleted.
  273. throw new OError('mongo project not found by history id', { historyId })
  274. }
  275. return project._id.toString()
  276. }
  277. async function resolveHistoryIdToMongoProjectId(projectId) {
  278. return projectId
  279. }
  280. /**
  281. * Record that a chunk was replaced by a new one.
  282. *
  283. * @param {string} projectId
  284. * @param {string} oldChunkId
  285. * @param {Chunk} newChunk
  286. * @param {string} newChunkId
  287. * @param {object} [opts]
  288. * @param {Date} [opts.earliestChangeTimestamp]
  289. */
  290. async function confirmUpdate(
  291. projectId,
  292. oldChunkId,
  293. newChunk,
  294. newChunkId,
  295. opts = {}
  296. ) {
  297. assert.mongoId(projectId, 'bad projectId')
  298. assert.mongoId(oldChunkId, 'bad oldChunkId')
  299. assert.instance(newChunk, Chunk, 'bad newChunk')
  300. assert.mongoId(newChunkId, 'bad newChunkId')
  301. await mongodb.client.withSession(async session => {
  302. await session.withTransaction(async () => {
  303. await deleteActiveChunk(projectId, oldChunkId, { session })
  304. await activateChunk(projectId, newChunkId, { session })
  305. await updateProjectRecord(
  306. projectId,
  307. newChunk,
  308. opts.earliestChangeTimestamp,
  309. { session }
  310. )
  311. })
  312. })
  313. }
  314. /**
  315. * Activate a pending chunk
  316. *
  317. * @param {string} projectId
  318. * @param {string} chunkId
  319. * @param {object} [opts]
  320. * @param {ClientSession} [opts.session]
  321. */
  322. async function activateChunk(projectId, chunkId, opts = {}) {
  323. assert.mongoId(projectId, 'bad projectId')
  324. assert.mongoId(chunkId, 'bad chunkId')
  325. let result
  326. try {
  327. result = await mongodb.chunks.updateOne(
  328. {
  329. _id: new ObjectId(chunkId),
  330. projectId: new ObjectId(projectId),
  331. state: 'pending',
  332. },
  333. { $set: { state: 'active', updatedAt: new Date() } },
  334. opts
  335. )
  336. } catch (err) {
  337. if (err instanceof MongoError && err.code === DUPLICATE_KEY_ERROR_CODE) {
  338. throw new ChunkVersionConflictError('chunk start version is not unique', {
  339. projectId,
  340. chunkId,
  341. })
  342. } else {
  343. throw err
  344. }
  345. }
  346. if (result.matchedCount === 0) {
  347. throw new OError('pending chunk not found', { projectId, chunkId })
  348. }
  349. }
  350. /**
  351. * Close a chunk
  352. *
  353. * A closed chunk is one that can't be extended anymore.
  354. *
  355. * @param {string} projectId
  356. * @param {string} chunkId
  357. * @param {object} [opts]
  358. * @param {ClientSession} [opts.session]
  359. */
  360. async function closeChunk(projectId, chunkId, opts = {}) {
  361. const result = await mongodb.chunks.updateOne(
  362. {
  363. _id: new ObjectId(chunkId),
  364. projectId: new ObjectId(projectId),
  365. state: 'active',
  366. },
  367. { $set: { state: 'closed' } },
  368. opts
  369. )
  370. if (result.matchedCount === 0) {
  371. throw new ChunkVersionConflictError('unable to close chunk', {
  372. projectId,
  373. chunkId,
  374. })
  375. }
  376. }
  377. /**
  378. * Delete an active chunk
  379. *
  380. * This is used to delete chunks that are in the process of being extended. It
  381. * will refuse to delete chunks that are already closed and can therefore not be
  382. * extended.
  383. *
  384. * @param {string} projectId
  385. * @param {string} chunkId
  386. * @param {object} [opts]
  387. * @param {ClientSession} [opts.session]
  388. */
  389. async function deleteActiveChunk(projectId, chunkId, opts = {}) {
  390. const updateResult = await mongodb.chunks.updateOne(
  391. {
  392. _id: new ObjectId(chunkId),
  393. projectId: new ObjectId(projectId),
  394. state: 'active',
  395. },
  396. { $set: { state: 'deleted', updatedAt: new Date() } },
  397. opts
  398. )
  399. if (updateResult.matchedCount === 0) {
  400. throw new ChunkVersionConflictError('unable to delete active chunk', {
  401. projectId,
  402. chunkId,
  403. })
  404. }
  405. }
  406. /**
  407. * Delete a chunk.
  408. *
  409. * @param {string} projectId
  410. * @param {string} chunkId
  411. * @return {Promise}
  412. */
  413. async function deleteChunk(projectId, chunkId, mongoOpts = {}) {
  414. assert.mongoId(projectId, 'bad projectId')
  415. assert.mongoId(chunkId, 'bad chunkId')
  416. await mongodb.chunks.updateOne(
  417. { _id: new ObjectId(chunkId), projectId: new ObjectId(projectId) },
  418. { $set: { state: 'deleted', updatedAt: new Date() } },
  419. mongoOpts
  420. )
  421. }
  422. /**
  423. * Delete all of a project's chunks
  424. */
  425. async function deleteProjectChunks(projectId) {
  426. assert.mongoId(projectId, 'bad projectId')
  427. await mongodb.chunks.updateMany(
  428. {
  429. projectId: new ObjectId(projectId),
  430. state: { $in: ['active', 'closed'] },
  431. },
  432. { $set: { state: 'deleted', updatedAt: new Date() } }
  433. )
  434. }
  435. /**
  436. * Get a batch of old chunks for deletion
  437. */
  438. async function getOldChunksBatch(count, minAgeSecs) {
  439. const maxUpdatedAt = new Date(Date.now() - minAgeSecs * 1000)
  440. const batch = []
  441. // We need to fetch one state at a time to take advantage of the partial
  442. // indexes on the chunks collection.
  443. //
  444. // Mongo 6.0 allows partial indexes that use the $in operator. When we reach
  445. // that Mongo version, we can create a partial index on both the deleted and
  446. // pending states and simplify this logic a bit.
  447. for (const state of ['deleted', 'pending']) {
  448. if (count === 0) {
  449. // There's no more space in the batch
  450. break
  451. }
  452. const cursor = mongodb.chunks
  453. .find(
  454. { state, updatedAt: { $lt: maxUpdatedAt } },
  455. {
  456. limit: count,
  457. projection: { _id: 1, projectId: 1 },
  458. }
  459. )
  460. .map(record => ({
  461. chunkId: record._id.toString(),
  462. projectId: record.projectId.toString(),
  463. }))
  464. for await (const record of cursor) {
  465. batch.push(record)
  466. count -= 1
  467. }
  468. }
  469. return batch
  470. }
  471. /**
  472. * Delete a batch of old chunks from the database
  473. */
  474. async function deleteOldChunks(chunkIds) {
  475. await mongodb.chunks.deleteMany({
  476. _id: { $in: chunkIds.map(id => new ObjectId(id)) },
  477. state: { $in: ['deleted', 'pending'] },
  478. })
  479. }
  480. /**
  481. * Build a chunk metadata object from the database record
  482. */
  483. function chunkFromRecord(record) {
  484. return {
  485. id: record._id.toString(),
  486. startVersion: record.startVersion,
  487. endVersion: record.endVersion,
  488. endTimestamp: record.endTimestamp,
  489. }
  490. }
  491. module.exports = {
  492. getLatestChunk,
  493. getFirstChunkBeforeTimestamp,
  494. getLastActiveChunkBeforeTimestamp,
  495. getChunkForVersion,
  496. getChunkForTimestamp,
  497. getProjectChunkIds,
  498. getProjectChunks,
  499. insertPendingChunk,
  500. confirmCreate,
  501. confirmUpdate,
  502. updateProjectRecord,
  503. deleteChunk,
  504. deleteProjectChunks,
  505. getOldChunksBatch,
  506. deleteOldChunks,
  507. lookupMongoProjectIdFromHistoryId,
  508. resolveHistoryIdToMongoProjectId,
  509. }