UpdatesProcessor.js 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635
  1. import { promisify } from 'node:util'
  2. import logger from '@overleaf/logger'
  3. import async from 'async'
  4. import metrics from '@overleaf/metrics'
  5. import Settings from '@overleaf/settings'
  6. import OError from '@overleaf/o-error'
  7. import * as HistoryStoreManager from './HistoryStoreManager.js'
  8. import * as UpdateTranslator from './UpdateTranslator.js'
  9. import * as BlobManager from './BlobManager.js'
  10. import * as RedisManager from './RedisManager.js'
  11. import * as ErrorRecorder from './ErrorRecorder.js'
  12. import * as LockManager from './LockManager.js'
  13. import * as UpdateCompressor from './UpdateCompressor.js'
  14. import * as WebApiManager from './WebApiManager.js'
  15. import * as SyncManager from './SyncManager.js'
  16. import * as Versions from './Versions.js'
  17. import * as Errors from './Errors.js'
  18. import * as Metrics from './Metrics.js'
  19. import { Profiler } from './Profiler.js'
  20. const keys = Settings.redis.lock.key_schema
  21. export const REDIS_READ_BATCH_SIZE = 500
  22. /**
  23. * Container for functions that need to be mocked in tests
  24. *
  25. * TODO: Rewrite tests in terms of exported functions only
  26. */
  27. export const _mocks = {}
  28. export function getRawUpdates(projectId, batchSize, callback) {
  29. RedisManager.getRawUpdatesBatch(projectId, batchSize, (error, batch) => {
  30. if (error != null) {
  31. return callback(OError.tag(error))
  32. }
  33. let updates
  34. try {
  35. updates = RedisManager.parseDocUpdates(batch.rawUpdates)
  36. } catch (error) {
  37. return callback(OError.tag(error))
  38. }
  39. _getHistoryId(projectId, updates, (error, historyId) => {
  40. if (error != null) {
  41. return callback(OError.tag(error))
  42. }
  43. HistoryStoreManager.getMostRecentChunk(
  44. projectId,
  45. historyId,
  46. (error, chunk) => {
  47. if (error != null) {
  48. return callback(OError.tag(error))
  49. }
  50. callback(null, { project_id: projectId, chunk, updates })
  51. }
  52. )
  53. })
  54. })
  55. }
  56. // Process all updates for a project, only check project-level information once
  57. export function processUpdatesForProject(projectId, callback) {
  58. const startTimeMs = Date.now()
  59. LockManager.runWithLock(
  60. keys.projectHistoryLock({ project_id: projectId }),
  61. (extendLock, releaseLock) => {
  62. _countAndProcessUpdates(
  63. projectId,
  64. extendLock,
  65. REDIS_READ_BATCH_SIZE,
  66. releaseLock
  67. )
  68. },
  69. (error, queueSize) => {
  70. if (error) {
  71. OError.tag(error)
  72. }
  73. ErrorRecorder.record(projectId, queueSize, error, callback)
  74. if (queueSize > 0) {
  75. const duration = (Date.now() - startTimeMs) / 1000
  76. Metrics.historyFlushDurationSeconds.observe(duration)
  77. Metrics.historyFlushQueueSize.observe(queueSize)
  78. }
  79. // clear the timestamp in the background if the queue is now empty
  80. RedisManager.clearDanglingFirstOpTimestamp(projectId, () => {})
  81. }
  82. )
  83. }
  84. export function processUpdatesForProjectUsingBisect(
  85. projectId,
  86. amountToProcess,
  87. callback
  88. ) {
  89. LockManager.runWithLock(
  90. keys.projectHistoryLock({ project_id: projectId }),
  91. (extendLock, releaseLock) => {
  92. _countAndProcessUpdates(
  93. projectId,
  94. extendLock,
  95. amountToProcess,
  96. releaseLock
  97. )
  98. },
  99. (error, queueSize) => {
  100. if (amountToProcess === 0 || queueSize === 0) {
  101. // no further processing possible
  102. if (error != null) {
  103. ErrorRecorder.record(
  104. projectId,
  105. queueSize,
  106. OError.tag(error),
  107. callback
  108. )
  109. } else {
  110. callback()
  111. }
  112. } else {
  113. if (error != null) {
  114. // decrease the batch size when we hit an error
  115. processUpdatesForProjectUsingBisect(
  116. projectId,
  117. Math.floor(amountToProcess / 2),
  118. callback
  119. )
  120. } else {
  121. // otherwise continue processing with the same batch size
  122. processUpdatesForProjectUsingBisect(
  123. projectId,
  124. amountToProcess,
  125. callback
  126. )
  127. }
  128. }
  129. }
  130. )
  131. }
  132. export function processSingleUpdateForProject(projectId, callback) {
  133. LockManager.runWithLock(
  134. keys.projectHistoryLock({ project_id: projectId }),
  135. (
  136. extendLock,
  137. releaseLock // set the batch size to 1 for single-stepping
  138. ) => {
  139. _countAndProcessUpdates(projectId, extendLock, 1, releaseLock)
  140. },
  141. (
  142. error,
  143. queueSize // no need to clear the flush marker when single stepping
  144. ) => {
  145. // it will be cleared up on the next background flush if
  146. // the queue is empty
  147. ErrorRecorder.record(projectId, queueSize, error, callback)
  148. }
  149. )
  150. }
  151. _mocks._countAndProcessUpdates = (
  152. projectId,
  153. extendLock,
  154. batchSize,
  155. callback
  156. ) => {
  157. RedisManager.countUnprocessedUpdates(projectId, (error, queueSize) => {
  158. if (error != null) {
  159. return callback(OError.tag(error))
  160. }
  161. if (queueSize > 0) {
  162. logger.debug({ projectId, queueSize }, 'processing uncompressed updates')
  163. RedisManager.getUpdatesInBatches(
  164. projectId,
  165. batchSize,
  166. (updates, cb) => {
  167. _processUpdatesBatch(projectId, updates, extendLock, cb)
  168. },
  169. error => {
  170. // Unconventional callback signature. The caller needs the queue size
  171. // even when an error is thrown in order to record the queue size in
  172. // the projectHistoryFailures collection. We'll have to find another
  173. // way to achieve this when we promisify.
  174. callback(error, queueSize)
  175. }
  176. )
  177. } else {
  178. logger.debug({ projectId }, 'no updates to process')
  179. callback(null, queueSize)
  180. }
  181. })
  182. }
  183. function _countAndProcessUpdates(...args) {
  184. _mocks._countAndProcessUpdates(...args)
  185. }
  186. function _processUpdatesBatch(projectId, updates, extendLock, callback) {
  187. // If the project doesn't have a history then we can bail out here
  188. _getHistoryId(projectId, updates, (error, historyId) => {
  189. if (error != null) {
  190. return callback(OError.tag(error))
  191. }
  192. if (historyId == null) {
  193. logger.debug(
  194. { projectId },
  195. 'discarding updates as project does not use history'
  196. )
  197. return callback()
  198. }
  199. _processUpdates(projectId, historyId, updates, extendLock, error => {
  200. if (error != null) {
  201. return callback(OError.tag(error))
  202. }
  203. callback()
  204. })
  205. })
  206. }
  207. export function _getHistoryId(projectId, updates, callback) {
  208. let idFromUpdates = null
  209. // check that all updates have the same history id
  210. for (const update of updates) {
  211. if (update.projectHistoryId != null) {
  212. if (idFromUpdates == null) {
  213. idFromUpdates = update.projectHistoryId.toString()
  214. } else if (idFromUpdates !== update.projectHistoryId.toString()) {
  215. metrics.inc('updates.batches.project-history-id.inconsistent-update')
  216. logger.warn(
  217. {
  218. projectId,
  219. updates,
  220. idFromUpdates,
  221. currentId: update.projectHistoryId,
  222. },
  223. 'inconsistent project history id between updates'
  224. )
  225. return callback(
  226. new OError('inconsistent project history id between updates')
  227. )
  228. }
  229. }
  230. }
  231. WebApiManager.getHistoryId(projectId, (error, idFromWeb) => {
  232. if (error != null && idFromUpdates != null) {
  233. // present only on updates
  234. // 404s from web are an error
  235. metrics.inc('updates.batches.project-history-id.from-updates')
  236. return callback(null, idFromUpdates)
  237. } else if (error != null) {
  238. return callback(OError.tag(error))
  239. }
  240. if (idFromWeb == null && idFromUpdates == null) {
  241. // present on neither web nor updates
  242. callback(null, null)
  243. } else if (idFromWeb != null && idFromUpdates == null) {
  244. // present only on web
  245. metrics.inc('updates.batches.project-history-id.from-web')
  246. callback(null, idFromWeb)
  247. } else if (idFromWeb == null && idFromUpdates != null) {
  248. // present only on updates
  249. metrics.inc('updates.batches.project-history-id.from-updates')
  250. callback(null, idFromUpdates)
  251. } else if (idFromWeb.toString() !== idFromUpdates.toString()) {
  252. // inconsistent between web and updates
  253. metrics.inc('updates.batches.project-history-id.inconsistent-with-web')
  254. logger.warn(
  255. {
  256. projectId,
  257. idFromWeb,
  258. idFromUpdates,
  259. updates,
  260. },
  261. 'inconsistent project history id between updates and web'
  262. )
  263. callback(
  264. new OError('inconsistent project history id between updates and web')
  265. )
  266. } else {
  267. // the same on web and updates
  268. metrics.inc('updates.batches.project-history-id.from-updates')
  269. callback(null, idFromWeb)
  270. }
  271. })
  272. }
  273. function _handleOpsOutOfOrderError(projectId, projectHistoryId, err, ...rest) {
  274. const adjustedLength = Math.max(rest.length, 1)
  275. const results = rest.slice(0, adjustedLength - 1)
  276. const callback = rest[adjustedLength - 1]
  277. ErrorRecorder.getFailureRecord(projectId, (error, failureRecord) => {
  278. if (error != null) {
  279. return callback(error)
  280. }
  281. // Bypass ops-out-of-order errors in the stored chunk when in forceDebug mode
  282. if (failureRecord != null && failureRecord.forceDebug === true) {
  283. logger.warn(
  284. { projectId, projectHistoryId },
  285. 'ops out of order in chunk, forced continue'
  286. )
  287. callback(null, ...results) // return results without error
  288. } else {
  289. logger.warn(
  290. { projectId, projectHistoryId },
  291. 'ops out of order in chunk, returning error'
  292. )
  293. callback(err, ...results)
  294. }
  295. })
  296. }
  297. function _getMostRecentVersionWithDebug(projectId, projectHistoryId, callback) {
  298. HistoryStoreManager.getMostRecentVersion(
  299. projectId,
  300. projectHistoryId,
  301. (err, ...results) => {
  302. if (err instanceof Errors.OpsOutOfOrderError) {
  303. _handleOpsOutOfOrderError(
  304. projectId,
  305. projectHistoryId,
  306. err,
  307. ...results,
  308. callback
  309. )
  310. } else {
  311. callback(err, ...results)
  312. }
  313. }
  314. )
  315. }
  316. export function _processUpdates(
  317. projectId,
  318. projectHistoryId,
  319. updates,
  320. extendLock,
  321. callback
  322. ) {
  323. const profile = new Profiler('_processUpdates', {
  324. project_id: projectId,
  325. projectHistoryId,
  326. })
  327. // skip updates first if we're in a sync, we might not need to do anything else
  328. SyncManager.skipUpdatesDuringSync(
  329. projectId,
  330. updates,
  331. (error, filteredUpdates, newSyncState) => {
  332. profile.log('skipUpdatesDuringSync')
  333. if (error != null) {
  334. return callback(error)
  335. }
  336. if (filteredUpdates.length === 0) {
  337. // return early if there are no updates to apply
  338. return SyncManager.setResyncState(projectId, newSyncState, callback)
  339. }
  340. // only make request to history service if we have actual updates to process
  341. _getMostRecentVersionWithDebug(
  342. projectId,
  343. projectHistoryId,
  344. (
  345. error,
  346. baseVersion,
  347. projectStructureAndDocVersions,
  348. _lastChange,
  349. mostRecentChunk
  350. ) => {
  351. if (projectStructureAndDocVersions == null) {
  352. projectStructureAndDocVersions = { project: null, docs: {} }
  353. }
  354. profile.log('getMostRecentVersion')
  355. if (error != null) {
  356. return callback(error)
  357. }
  358. async.waterfall(
  359. [
  360. cb => {
  361. cb = profile.wrap('expandSyncUpdates', cb)
  362. SyncManager.expandSyncUpdates(
  363. projectId,
  364. projectHistoryId,
  365. mostRecentChunk,
  366. filteredUpdates,
  367. extendLock,
  368. cb
  369. )
  370. },
  371. (expandedUpdates, cb) => {
  372. let unappliedUpdates
  373. try {
  374. unappliedUpdates = _skipAlreadyAppliedUpdates(
  375. projectId,
  376. expandedUpdates,
  377. projectStructureAndDocVersions
  378. )
  379. } catch (err) {
  380. return cb(err)
  381. }
  382. profile.log('skipAlreadyAppliedUpdates')
  383. const compressedUpdates =
  384. UpdateCompressor.compressRawUpdates(unappliedUpdates)
  385. const timeTaken = profile
  386. .log('compressRawUpdates')
  387. .getTimeDelta()
  388. if (timeTaken >= 1000) {
  389. logger.debug(
  390. { projectId, updates: unappliedUpdates, timeTaken },
  391. 'slow compression of raw updates'
  392. )
  393. }
  394. cb = profile.wrap('createBlobs', cb)
  395. BlobManager.createBlobsForUpdates(
  396. projectId,
  397. projectHistoryId,
  398. compressedUpdates,
  399. extendLock,
  400. cb
  401. )
  402. },
  403. (updatesWithBlobs, cb) => {
  404. let changes
  405. try {
  406. changes = UpdateTranslator.convertToChanges(
  407. projectId,
  408. updatesWithBlobs
  409. ).map(change => change.toRaw())
  410. } catch (err) {
  411. return cb(err)
  412. } finally {
  413. profile.log('convertToChanges')
  414. }
  415. cb(null, changes)
  416. },
  417. (changes, cb) => {
  418. let change
  419. const numChanges = changes.length
  420. const byteLength = Buffer.byteLength(
  421. JSON.stringify(changes),
  422. 'utf8'
  423. )
  424. let numOperations = 0
  425. for (change of changes) {
  426. if (change.operations != null) {
  427. numOperations += change.operations.length
  428. }
  429. }
  430. metrics.timing('history-store.request.changes', numChanges, 1)
  431. metrics.timing('history-store.request.bytes', byteLength, 1)
  432. metrics.timing(
  433. 'history-store.request.operations',
  434. numOperations,
  435. 1
  436. )
  437. // thresholds taken from write_latex/main/lib/history_exporter.rb
  438. if (numChanges > 1000) {
  439. metrics.inc('history-store.request.exceeds-threshold.changes')
  440. }
  441. if (byteLength > Math.pow(1024, 2)) {
  442. metrics.inc('history-store.request.exceeds-threshold.bytes')
  443. const changeLengths = changes.map(change =>
  444. Buffer.byteLength(JSON.stringify(change), 'utf8')
  445. )
  446. logger.warn(
  447. { projectId, byteLength, changeLengths },
  448. 'change size exceeds limit'
  449. )
  450. }
  451. cb = profile.wrap('sendChanges', cb)
  452. // this is usually the longest request, so extend the lock before starting it
  453. extendLock(error => {
  454. if (error != null) {
  455. return cb(error)
  456. }
  457. if (changes.length === 0) {
  458. return cb()
  459. } // avoid unnecessary requests to history service
  460. HistoryStoreManager.sendChanges(
  461. projectId,
  462. projectHistoryId,
  463. changes,
  464. baseVersion,
  465. cb
  466. )
  467. })
  468. },
  469. cb => {
  470. cb = profile.wrap('setResyncState', cb)
  471. SyncManager.setResyncState(projectId, newSyncState, cb)
  472. },
  473. ],
  474. error => {
  475. profile.end()
  476. callback(error)
  477. }
  478. )
  479. }
  480. )
  481. }
  482. )
  483. }
  484. _mocks._skipAlreadyAppliedUpdates = (
  485. projectId,
  486. updates,
  487. projectStructureAndDocVersions
  488. ) => {
  489. function alreadySeenProjectVersion(previousProjectStructureVersion, update) {
  490. return (
  491. UpdateTranslator.isProjectStructureUpdate(update) &&
  492. previousProjectStructureVersion != null &&
  493. update.version != null &&
  494. Versions.gte(previousProjectStructureVersion, update.version)
  495. )
  496. }
  497. function alreadySeenDocVersion(previousDocVersions, update) {
  498. if (UpdateTranslator.isTextUpdate(update) && update.v != null) {
  499. const docId = update.doc
  500. return (
  501. previousDocVersions[docId] != null &&
  502. previousDocVersions[docId].v != null &&
  503. Versions.gte(previousDocVersions[docId].v, update.v)
  504. )
  505. } else {
  506. return false
  507. }
  508. }
  509. // check that the incoming updates are in the correct order (we do not
  510. // want to send out of order updates to the history service)
  511. let incomingProjectStructureVersion = null
  512. const incomingDocVersions = {}
  513. for (const update of updates) {
  514. if (alreadySeenProjectVersion(incomingProjectStructureVersion, update)) {
  515. logger.warn(
  516. { projectId, update, incomingProjectStructureVersion },
  517. 'incoming project structure updates are out of order'
  518. )
  519. throw new Errors.OpsOutOfOrderError(
  520. 'project structure version out of order on incoming updates'
  521. )
  522. } else if (alreadySeenDocVersion(incomingDocVersions, update)) {
  523. logger.warn(
  524. { projectId, update, incomingDocVersions },
  525. 'incoming doc updates are out of order'
  526. )
  527. throw new Errors.OpsOutOfOrderError(
  528. 'doc version out of order on incoming updates'
  529. )
  530. }
  531. // update the current project structure and doc versions
  532. if (UpdateTranslator.isProjectStructureUpdate(update)) {
  533. incomingProjectStructureVersion = update.version
  534. } else if (UpdateTranslator.isTextUpdate(update)) {
  535. incomingDocVersions[update.doc] = { v: update.v }
  536. }
  537. }
  538. // discard updates already applied
  539. const updatesToApply = []
  540. const previousProjectStructureVersion = projectStructureAndDocVersions.project
  541. const previousDocVersions = projectStructureAndDocVersions.docs
  542. if (projectStructureAndDocVersions != null) {
  543. const updateProjectVersions = []
  544. for (const update of updates) {
  545. if (update != null && update.version != null) {
  546. updateProjectVersions.push(update.version)
  547. }
  548. }
  549. logger.debug(
  550. { projectId, projectStructureAndDocVersions, updateProjectVersions },
  551. 'comparing updates with existing project versions'
  552. )
  553. }
  554. for (const update of updates) {
  555. if (alreadySeenProjectVersion(previousProjectStructureVersion, update)) {
  556. metrics.inc('updates.discarded_project_structure_version')
  557. logger.debug(
  558. { projectId, update, previousProjectStructureVersion },
  559. 'discarding previously applied project structure update'
  560. )
  561. continue
  562. }
  563. if (alreadySeenDocVersion(previousDocVersions, update)) {
  564. metrics.inc('updates.discarded_doc_version')
  565. logger.debug(
  566. { projectId, update, previousDocVersions },
  567. 'discarding previously applied doc update'
  568. )
  569. continue
  570. }
  571. // remove non-BMP characters from resync updates that have bypassed the normal docupdater flow
  572. _sanitizeUpdate(update)
  573. // if all checks above are ok then accept the update
  574. updatesToApply.push(update)
  575. }
  576. return updatesToApply
  577. }
  578. export function _skipAlreadyAppliedUpdates(...args) {
  579. return _mocks._skipAlreadyAppliedUpdates(...args)
  580. }
  581. function _sanitizeUpdate(update) {
  582. // adapted from docupdater's UpdateManager, we should clean these in docupdater
  583. // too but we already have queues with this problem so we will handle it here
  584. // too for robustness.
  585. // Replace high and low surrogate characters with 'replacement character' (\uFFFD)
  586. const removeBadChars = str => str.replace(/[\uD800-\uDFFF]/g, '\uFFFD')
  587. // clean up any bad chars in resync diffs
  588. if (update.op) {
  589. for (const op of update.op) {
  590. if (op.i != null) {
  591. op.i = removeBadChars(op.i)
  592. }
  593. }
  594. }
  595. // clean up any bad chars in resync new docs
  596. if (update.docLines != null) {
  597. update.docLines = removeBadChars(update.docLines)
  598. }
  599. return update
  600. }
  601. export const promises = {
  602. processUpdatesForProject: promisify(processUpdatesForProject),
  603. }