UpdatesManager.coffee 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281
  1. MongoManager = require "./MongoManager"
  2. RedisManager = require "./RedisManager"
  3. UpdateCompressor = require "./UpdateCompressor"
  4. LockManager = require "./LockManager"
  5. WebApiManager = require "./WebApiManager"
  6. UpdateTrimmer = require "./UpdateTrimmer"
  7. logger = require "logger-sharelatex"
  8. async = require "async"
  9. DocArchiveManager = require "./DocArchiveManager"
  10. _ = require "underscore"
  11. Settings = require "settings-sharelatex"
  12. module.exports = UpdatesManager =
  13. compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) ->
  14. length = rawUpdates.length
  15. if length == 0
  16. return callback()
  17. MongoManager.peekLastCompressedUpdate doc_id, (error, lastCompressedUpdate, lastVersion) ->
  18. # lastCompressedUpdate is the most recent update in Mongo.
  19. #
  20. # The peekLastCompressedUpdate method may pass it back as 'null'
  21. # to force the start of a new compressed update, even when there
  22. # was a previous compressed update in Mongo. In this case it
  23. # passes back the lastVersion from the update to check
  24. # consistency.
  25. # when lastVersion is not provided, default to lastCompressedUpdate.v
  26. lastVersion ?= lastCompressedUpdate?.v
  27. return callback(error) if error?
  28. # Ensure that raw updates start where lastVersion left off
  29. if lastVersion?
  30. rawUpdates = rawUpdates.slice(0)
  31. while rawUpdates[0]? and rawUpdates[0].v <= lastVersion
  32. rawUpdates.shift()
  33. if rawUpdates[0]? and rawUpdates[0].v != lastVersion + 1
  34. ts = lastCompressedUpdate?.meta?.end_ts
  35. last_timestamp = if ts? then new Date(ts) else 'unknown time'
  36. error = new Error("Tried to apply raw op at version #{rawUpdates[0].v} to last compressed update with version #{lastVersion} from #{last_timestamp}")
  37. logger.error err: error, doc_id: doc_id, project_id: project_id, prev_end_ts: ts, "inconsistent doc versions"
  38. if Settings.trackchanges?.continueOnError and rawUpdates[0].v > lastVersion + 1
  39. # we have lost some ops - continue to write into the database, we can't recover at this point
  40. lastCompressedUpdate = null
  41. else
  42. return callback error
  43. compressedUpdates = UpdateCompressor.compressRawUpdates lastCompressedUpdate, rawUpdates
  44. if not lastCompressedUpdate?
  45. # no existing update, insert everything
  46. updateToModify = null
  47. updatesToInsert = compressedUpdates
  48. else
  49. # there are existing updates, see what happens when we
  50. # compress them together with the new ones
  51. [firstUpdate, additionalUpdates...] = compressedUpdates
  52. if firstUpdate.v == lastCompressedUpdate.v and _.isEqual(firstUpdate, lastCompressedUpdate)
  53. # first update version hasn't changed, skip it and insert remaining updates
  54. # this is an optimisation, we could update the existing op with itself
  55. updateToModify = null
  56. updatesToInsert = additionalUpdates
  57. else
  58. # first update version did changed, modify it and insert remaining updates
  59. updateToModify = firstUpdate
  60. updatesToInsert = additionalUpdates
  61. MongoManager.modifyCompressedUpdate lastCompressedUpdate, updateToModify, (error, result) ->
  62. return callback(error) if error?
  63. logger.log {project_id, doc_id, orig_v: lastCompressedUpdate.v, new_v: result.v}, "applied update in-place" if result?
  64. MongoManager.insertCompressedUpdates project_id, doc_id, updatesToInsert, temporary,(error) ->
  65. return callback(error) if error?
  66. logger.log project_id: project_id, doc_id: doc_id, rawUpdatesLength: length, compressedUpdatesLength: compressedUpdates.length, "compressed doc updates"
  67. callback()
  68. REDIS_READ_BATCH_SIZE: 100
  69. processUncompressedUpdates: (project_id, doc_id, callback = (error) ->) ->
  70. UpdateTrimmer.shouldTrimUpdates project_id, (error, temporary) ->
  71. return callback(error) if error?
  72. MongoManager.backportProjectId project_id, doc_id, (error) ->
  73. return callback(error) if error?
  74. # get the updates as strings from redis (so we can delete them after they are applied)
  75. RedisManager.getOldestDocUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, docUpdates) ->
  76. return callback(error) if error?
  77. length = docUpdates.length
  78. # parse the redis strings into ShareJs updates
  79. RedisManager.expandDocUpdates docUpdates, (error, rawUpdates) ->
  80. return callback(error) if error?
  81. UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, temporary, (error) ->
  82. return callback(error) if error?
  83. logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates"
  84. # delete the applied updates from redis
  85. RedisManager.deleteAppliedDocUpdates project_id, doc_id, docUpdates, (error) ->
  86. return callback(error) if error?
  87. if length == UpdatesManager.REDIS_READ_BATCH_SIZE
  88. # There might be more updates
  89. logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates"
  90. setTimeout () ->
  91. UpdatesManager.processUncompressedUpdates project_id, doc_id, callback
  92. , 0
  93. else
  94. logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed"
  95. callback()
  96. processUncompressedUpdatesWithLock: (project_id, doc_id, callback = (error) ->) ->
  97. LockManager.runWithLock(
  98. "HistoryLock:#{doc_id}",
  99. (releaseLock) ->
  100. UpdatesManager.processUncompressedUpdates project_id, doc_id, releaseLock
  101. callback
  102. )
  103. processUncompressedUpdatesForProject: (project_id, callback = (error) ->) ->
  104. RedisManager.getDocIdsWithHistoryOps project_id, (error, doc_ids) ->
  105. return callback(error) if error?
  106. jobs = []
  107. for doc_id in doc_ids
  108. do (doc_id) ->
  109. jobs.push (callback) ->
  110. UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, callback
  111. async.parallelLimit jobs, 5, callback
  112. getDocUpdates: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
  113. UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, (error) ->
  114. return callback(error) if error?
  115. MongoManager.getDocUpdates doc_id, options, callback
  116. getDocUpdatesWithUserInfo: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
  117. UpdatesManager.getDocUpdates project_id, doc_id, options, (error, updates) ->
  118. return callback(error) if error?
  119. UpdatesManager.fillUserInfo updates, (error, updates) ->
  120. return callback(error) if error?
  121. callback null, updates
  122. getProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) ->
  123. UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
  124. return callback(error) if error?
  125. MongoManager.getProjectUpdates project_id, options, (error, updates) ->
  126. jobs = []
  127. for update in updates
  128. if update.inS3
  129. do (update) ->
  130. jobs.push (callback) -> DocArchiveManager.unArchiveDocChanges update.project_id, update.doc_id, callback
  131. if jobs.length?
  132. async.series jobs, (err) ->
  133. MongoManager.getProjectUpdates project_id, options, callback
  134. else
  135. callback(error, updates)
  136. getProjectUpdatesWithUserInfo: (project_id, options = {}, callback = (error, updates) ->) ->
  137. UpdatesManager.getProjectUpdates project_id, options, (error, updates) ->
  138. return callback(error) if error?
  139. UpdatesManager.fillUserInfo updates, (error, updates) ->
  140. return callback(error) if error?
  141. callback null, updates
  142. getSummarizedProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) ->
  143. options.min_count ||= 25
  144. summarizedUpdates = []
  145. before = options.before
  146. do fetchNextBatch = () ->
  147. UpdatesManager._extendBatchOfSummarizedUpdates project_id, summarizedUpdates, before, options.min_count, (error, updates, nextBeforeUpdate) ->
  148. return callback(error) if error?
  149. if !nextBeforeUpdate? or updates.length >= options.min_count
  150. callback null, updates, nextBeforeUpdate
  151. else
  152. before = nextBeforeUpdate
  153. summarizedUpdates = updates
  154. fetchNextBatch()
  155. _extendBatchOfSummarizedUpdates: (
  156. project_id,
  157. existingSummarizedUpdates,
  158. before, desiredLength,
  159. callback = (error, summarizedUpdates, endOfDatabase) ->
  160. ) ->
  161. UpdatesManager.getProjectUpdatesWithUserInfo project_id, { before: before, limit: 3 * desiredLength }, (error, updates) ->
  162. return callback(error) if error?
  163. # Suppose in this request we have fetch the solid updates. In the next request we need
  164. # to fetch the dotted updates. These are defined by having an end timestamp less than
  165. # the last update's end timestamp (updates are ordered by descending end_ts). I.e.
  166. # start_ts--v v--end_ts
  167. # doc1: |......| |...| |-------|
  168. # doc2: |------------------|
  169. # ^----- Next time, fetch all updates with an
  170. # end_ts less than this
  171. #
  172. if updates? and updates.length > 0
  173. nextBeforeTimestamp = updates[updates.length - 1].meta.end_ts
  174. if nextBeforeTimestamp >= before
  175. error = new Error("history order is broken")
  176. logger.error err: error, project_id:project_id, nextBeforeTimestamp: nextBeforeTimestamp, before:before, "error in project history"
  177. return callback(error)
  178. else
  179. nextBeforeTimestamp = null
  180. summarizedUpdates = UpdatesManager._summarizeUpdates(
  181. updates, existingSummarizedUpdates
  182. )
  183. callback null,
  184. summarizedUpdates,
  185. nextBeforeTimestamp
  186. fillUserInfo: (updates, callback = (error, updates) ->) ->
  187. users = {}
  188. for update in updates
  189. if UpdatesManager._validUserId(update.meta.user_id)
  190. users[update.meta.user_id] = true
  191. jobs = []
  192. for user_id of users
  193. do (user_id) ->
  194. jobs.push (callback) ->
  195. WebApiManager.getUserInfo user_id, (error, userInfo) ->
  196. return callback(error) if error?
  197. users[user_id] = userInfo
  198. callback()
  199. async.series jobs, (error) ->
  200. return callback(error) if error?
  201. for update in updates
  202. user_id = update.meta.user_id
  203. delete update.meta.user_id
  204. if UpdatesManager._validUserId(user_id)
  205. update.meta.user = users[user_id]
  206. callback null, updates
  207. _validUserId: (user_id) ->
  208. if !user_id?
  209. return false
  210. else
  211. return !!user_id.match(/^[a-f0-9]{24}$/)
  212. TIME_BETWEEN_DISTINCT_UPDATES: fiveMinutes = 5 * 60 * 1000
  213. _summarizeUpdates: (updates, existingSummarizedUpdates = []) ->
  214. summarizedUpdates = existingSummarizedUpdates.slice()
  215. for update in updates
  216. earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1]
  217. if earliestUpdate and earliestUpdate.meta.start_ts - update.meta.end_ts < @TIME_BETWEEN_DISTINCT_UPDATES
  218. # check if the user in this update is already present in the earliest update,
  219. # if not, add them to the users list of the earliest update
  220. userExists = false
  221. for user in earliestUpdate.meta.users
  222. if (!user and !update.meta.user) or (user?.id == update.meta.user?.id)
  223. userExists = true
  224. break
  225. if !userExists
  226. earliestUpdate.meta.users.push update.meta.user
  227. doc_id = update.doc_id.toString()
  228. doc = earliestUpdate.docs[doc_id]
  229. if doc?
  230. doc.fromV = Math.min(doc.fromV, update.v)
  231. doc.toV = Math.max(doc.toV, update.v)
  232. else
  233. earliestUpdate.docs[doc_id] =
  234. fromV: update.v
  235. toV: update.v
  236. earliestUpdate.meta.start_ts = Math.min(earliestUpdate.meta.start_ts, update.meta.start_ts)
  237. earliestUpdate.meta.end_ts = Math.max(earliestUpdate.meta.end_ts, update.meta.end_ts)
  238. else
  239. newUpdate =
  240. meta:
  241. users: []
  242. start_ts: update.meta.start_ts
  243. end_ts: update.meta.end_ts
  244. docs: {}
  245. newUpdate.docs[update.doc_id.toString()] =
  246. fromV: update.v
  247. toV: update.v
  248. newUpdate.meta.users.push update.meta.user
  249. summarizedUpdates.push newUpdate
  250. return summarizedUpdates