persist_changes.js 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. // @ts-check
  2. 'use strict'
  3. const _ = require('lodash')
  4. const logger = require('@overleaf/logger')
  5. const core = require('overleaf-editor-core')
  6. const Chunk = core.Chunk
  7. const History = core.History
  8. const assert = require('./assert')
  9. const chunkStore = require('./chunk_store')
  10. const chunkBuffer = require('./chunk_buffer')
  11. const { BlobStore } = require('./blob_store')
  12. const { InvalidChangeError } = require('./errors')
  13. const { getContentHash } = require('./content_hash')
  14. function countChangeBytes(change) {
  15. // Note: This is not quite accurate, because the raw change may contain raw
  16. // file info (or conceivably even content) that will not be included in the
  17. // actual stored object.
  18. return Buffer.byteLength(JSON.stringify(change.toRaw()))
  19. }
  20. function totalChangeBytes(changes) {
  21. return changes.length ? _(changes).map(countChangeBytes).sum() : 0
  22. }
  23. // provide a simple timer function
  24. function Timer() {
  25. this.t0 = process.hrtime()
  26. }
  27. Timer.prototype.elapsed = function () {
  28. const dt = process.hrtime(this.t0)
  29. const timeInMilliseconds = (dt[0] + dt[1] * 1e-9) * 1e3
  30. return timeInMilliseconds
  31. }
  32. /**
  33. * Break the given set of changes into zero or more Chunks according to the
  34. * provided limits and store them.
  35. *
  36. * Some other possible improvements:
  37. * 1. This does a lot more JSON serialization than it has to. We may know the
  38. * JSON for the changes before we call this function, so we could in that
  39. * case get the byte size of each change without doing any work. Even if we
  40. * don't know it initially, we could save some computation by caching this
  41. * info rather than recomputing it many times. TBD whether it is worthwhile.
  42. * 2. We don't necessarily have to fetch the latest chunk in order to determine
  43. * that it is full. We could store this in the chunk metadata record. It may
  44. * be worth distinguishing between a Chunk and its metadata record. The
  45. * endVersion may be better suited to the metadata record.
  46. *
  47. * @param {string} projectId
  48. * @param {core.Change[]} allChanges
  49. * @param {Object} limits
  50. * @param {number} clientEndVersion
  51. * @return {Promise.<Object?>}
  52. */
  53. async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
  54. assert.projectId(projectId)
  55. assert.array(allChanges)
  56. assert.maybe.object(limits)
  57. assert.integer(clientEndVersion)
  58. const blobStore = new BlobStore(projectId)
  59. const earliestChangeTimestamp =
  60. allChanges.length > 0 ? allChanges[0].getTimestamp() : null
  61. let currentChunk
  62. /**
  63. * currentSnapshot tracks the latest change that we're applying; we use it to
  64. * check that the changes we are persisting are valid.
  65. *
  66. * @type {core.Snapshot}
  67. */
  68. let currentSnapshot
  69. let originalEndVersion
  70. let changesToPersist
  71. limits = limits || {}
  72. _.defaults(limits, {
  73. changeBucketMinutes: 60,
  74. maxChanges: 2500,
  75. maxChangeBytes: 5 * 1024 * 1024,
  76. maxChunkChanges: 2000,
  77. maxChunkChangeBytes: 5 * 1024 * 1024,
  78. maxChunkChangeTime: 5000, // warn if total time for changes in a chunk takes longer than this
  79. })
  80. function checkElapsedTime(timer) {
  81. const timeTaken = timer.elapsed()
  82. if (timeTaken > limits.maxChunkChangeTime) {
  83. console.log('warning: slow chunk', projectId, timeTaken)
  84. }
  85. }
  86. /**
  87. * Add changes to a chunk until the chunk is full
  88. *
  89. * The chunk is full if it reaches a certain number of changes or a certain
  90. * size in bytes
  91. *
  92. * @param {core.Chunk} chunk
  93. * @param {core.Change[]} changes
  94. */
  95. async function fillChunk(chunk, changes) {
  96. let totalBytes = totalChangeBytes(chunk.getChanges())
  97. let changesPushed = false
  98. while (changes.length > 0) {
  99. if (chunk.getChanges().length >= limits.maxChunkChanges) {
  100. break
  101. }
  102. const change = changes[0]
  103. const changeBytes = countChangeBytes(change)
  104. if (totalBytes + changeBytes > limits.maxChunkChangeBytes) {
  105. break
  106. }
  107. for (const operation of change.iterativelyApplyTo(currentSnapshot, {
  108. strict: true,
  109. })) {
  110. await validateContentHash(operation)
  111. }
  112. chunk.pushChanges([change])
  113. changes.shift()
  114. totalBytes += changeBytes
  115. changesPushed = true
  116. }
  117. return changesPushed
  118. }
  119. /**
  120. * Check that the operation is valid and can be incorporated to the history.
  121. *
  122. * For now, this checks content hashes when they are provided.
  123. *
  124. * @param {core.Operation} operation
  125. */
  126. async function validateContentHash(operation) {
  127. if (operation instanceof core.EditFileOperation) {
  128. const editOperation = operation.getOperation()
  129. if (
  130. editOperation instanceof core.TextOperation &&
  131. editOperation.contentHash != null
  132. ) {
  133. const path = operation.getPathname()
  134. const file = currentSnapshot.getFile(path)
  135. if (file == null) {
  136. throw new InvalidChangeError('file not found for hash validation', {
  137. projectId,
  138. path,
  139. })
  140. }
  141. await file.load('eager', blobStore)
  142. const content = file.getContent({ filterTrackedDeletes: true })
  143. const expectedHash = editOperation.contentHash
  144. const actualHash = content != null ? getContentHash(content) : null
  145. logger.debug({ expectedHash, actualHash }, 'validating content hash')
  146. if (actualHash !== expectedHash) {
  147. throw new InvalidChangeError('content hash mismatch', {
  148. projectId,
  149. path,
  150. expectedHash,
  151. actualHash,
  152. })
  153. }
  154. // Remove the content hash from the change before storing it in the chunk.
  155. // It was only useful for validation.
  156. editOperation.contentHash = null
  157. }
  158. }
  159. }
  160. async function extendLastChunkIfPossible() {
  161. const latestChunk = await chunkBuffer.loadLatest(projectId)
  162. currentChunk = latestChunk
  163. originalEndVersion = latestChunk.getEndVersion()
  164. if (originalEndVersion !== clientEndVersion) {
  165. throw new Chunk.ConflictingEndVersion(
  166. clientEndVersion,
  167. originalEndVersion
  168. )
  169. }
  170. currentSnapshot = latestChunk.getSnapshot().clone()
  171. const timer = new Timer()
  172. currentSnapshot.applyAll(latestChunk.getChanges())
  173. const changesPushed = await fillChunk(currentChunk, changesToPersist)
  174. if (!changesPushed) {
  175. return
  176. }
  177. checkElapsedTime(timer)
  178. await chunkStore.update(
  179. projectId,
  180. originalEndVersion,
  181. currentChunk,
  182. earliestChangeTimestamp
  183. )
  184. }
  185. async function createNewChunksAsNeeded() {
  186. while (changesToPersist.length > 0) {
  187. const endVersion = currentChunk.getEndVersion()
  188. const history = new History(currentSnapshot.clone(), [])
  189. const chunk = new Chunk(history, endVersion)
  190. const timer = new Timer()
  191. const changesPushed = await fillChunk(chunk, changesToPersist)
  192. if (changesPushed) {
  193. checkElapsedTime(timer)
  194. currentChunk = chunk
  195. await chunkStore.create(projectId, chunk, earliestChangeTimestamp)
  196. } else {
  197. throw new Error('failed to fill empty chunk')
  198. }
  199. }
  200. }
  201. function isOlderThanMinChangeTimestamp(change) {
  202. return change.getTimestamp().getTime() < limits.minChangeTimestamp
  203. }
  204. function isOlderThanMaxChangeTimestamp(change) {
  205. return change.getTimestamp().getTime() < limits.maxChangeTimestamp
  206. }
  207. const oldChanges = _.filter(allChanges, isOlderThanMinChangeTimestamp)
  208. const anyTooOld = _.some(oldChanges, isOlderThanMaxChangeTimestamp)
  209. const tooManyChanges = oldChanges.length > limits.maxChanges
  210. const tooManyBytes = totalChangeBytes(oldChanges) > limits.maxChangeBytes
  211. if (anyTooOld || tooManyChanges || tooManyBytes) {
  212. changesToPersist = oldChanges
  213. const numberOfChangesToPersist = oldChanges.length
  214. await extendLastChunkIfPossible()
  215. await createNewChunksAsNeeded()
  216. return {
  217. numberOfChangesPersisted: numberOfChangesToPersist,
  218. originalEndVersion,
  219. currentChunk,
  220. }
  221. } else {
  222. return null
  223. }
  224. }
  225. module.exports = persistChanges