UpdatesManager.coffee 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340
  1. MongoManager = require "./MongoManager"
  2. PackManager = require "./PackManager"
  3. RedisManager = require "./RedisManager"
  4. UpdateCompressor = require "./UpdateCompressor"
  5. LockManager = require "./LockManager"
  6. WebApiManager = require "./WebApiManager"
  7. UpdateTrimmer = require "./UpdateTrimmer"
  8. logger = require "logger-sharelatex"
  9. async = require "async"
  10. _ = require "underscore"
  11. Settings = require "settings-sharelatex"
  12. module.exports = UpdatesManager =
  13. compressAndSaveRawUpdates: (project_id, doc_id, rawUpdates, temporary, callback = (error) ->) ->
  14. length = rawUpdates.length
  15. if length == 0
  16. return callback()
  17. # check that ops are in the correct order
  18. for op, i in rawUpdates when i > 0
  19. thisVersion = op?.v
  20. prevVersion = rawUpdates[i-1]?.v
  21. if not (prevVersion < thisVersion)
  22. logger.error project_id: project_id, doc_id: doc_id, rawUpdates:rawUpdates, temporary: temporary, thisVersion:thisVersion, prevVersion:prevVersion, "op versions out of order"
  23. # FIXME: we no longer need the lastCompressedUpdate, so change functions not to need it
  24. # CORRECTION: we do use it to log the time in case of error
  25. MongoManager.peekLastCompressedUpdate doc_id, (error, lastCompressedUpdate, lastVersion) ->
  26. # lastCompressedUpdate is the most recent update in Mongo, and
  27. # lastVersion is its sharejs version number.
  28. #
  29. # The peekLastCompressedUpdate method may pass the update back
  30. # as 'null' (for example if the previous compressed update has
  31. # been archived). In this case it can still pass back the
  32. # lastVersion from the update to allow us to check consistency.
  33. return callback(error) if error?
  34. # Ensure that raw updates start where lastVersion left off
  35. if lastVersion?
  36. discardedUpdates = []
  37. rawUpdates = rawUpdates.slice(0)
  38. while rawUpdates[0]? and rawUpdates[0].v <= lastVersion
  39. discardedUpdates.push rawUpdates.shift()
  40. if discardedUpdates.length
  41. logger.error project_id: project_id, doc_id: doc_id, discardedUpdates: discardedUpdates, temporary: temporary, lastVersion: lastVersion, "discarded updates already present"
  42. if rawUpdates[0]? and rawUpdates[0].v != lastVersion + 1
  43. ts = lastCompressedUpdate?.meta?.end_ts
  44. last_timestamp = if ts? then new Date(ts) else 'unknown time'
  45. error = new Error("Tried to apply raw op at version #{rawUpdates[0].v} to last compressed update with version #{lastVersion} from #{last_timestamp}")
  46. logger.error err: error, doc_id: doc_id, project_id: project_id, prev_end_ts: ts, temporary: temporary, lastCompressedUpdate: lastCompressedUpdate, "inconsistent doc versions"
  47. if Settings.trackchanges?.continueOnError and rawUpdates[0].v > lastVersion + 1
  48. # we have lost some ops - continue to write into the database, we can't recover at this point
  49. lastCompressedUpdate = null
  50. else
  51. return callback error
  52. if rawUpdates.length == 0
  53. return callback()
  54. # some old large ops in redis need to be rejected, they predate
  55. # the size limit that now prevents them going through the system
  56. REJECT_LARGE_OP_SIZE = 4 * 1024 * 1024
  57. for rawUpdate in rawUpdates
  58. opSizes = ((op.i?.length || op.d?.length) for op in rawUpdate?.op or [])
  59. size = _.max opSizes
  60. if size > REJECT_LARGE_OP_SIZE
  61. error = new Error("dropped op exceeding maximum allowed size of #{REJECT_LARGE_OP_SIZE}")
  62. logger.error err: error, doc_id: doc_id, project_id: project_id, size: size, rawUpdate: rawUpdate, "dropped op - too big"
  63. rawUpdate.op = []
  64. compressedUpdates = UpdateCompressor.compressRawUpdates null, rawUpdates
  65. PackManager.insertCompressedUpdates project_id, doc_id, lastCompressedUpdate, compressedUpdates, temporary, (error, result) ->
  66. return callback(error) if error?
  67. logger.log {project_id, doc_id, orig_v: lastCompressedUpdate?.v, new_v: result.v}, "inserted updates into pack" if result?
  68. callback()
  69. # Check whether the updates are temporary (per-project property)
  70. _prepareProjectForUpdates: (project_id, callback = (error, temporary) ->) ->
  71. UpdateTrimmer.shouldTrimUpdates project_id, (error, temporary) ->
  72. return callback(error) if error?
  73. callback(null, temporary)
  74. # Check for project id on document history (per-document property)
  75. _prepareDocForUpdates: (project_id, doc_id, callback = (error) ->) ->
  76. MongoManager.backportProjectId project_id, doc_id, (error) ->
  77. return callback(error) if error?
  78. callback(null)
  79. # Apply updates for specific project/doc after preparing at project and doc level
  80. REDIS_READ_BATCH_SIZE: 100
  81. processUncompressedUpdates: (project_id, doc_id, temporary, callback = (error) ->) ->
  82. # get the updates as strings from redis (so we can delete them after they are applied)
  83. RedisManager.getOldestDocUpdates doc_id, UpdatesManager.REDIS_READ_BATCH_SIZE, (error, docUpdates) ->
  84. return callback(error) if error?
  85. length = docUpdates.length
  86. # parse the redis strings into ShareJs updates
  87. RedisManager.expandDocUpdates docUpdates, (error, rawUpdates) ->
  88. return callback(error) if error?
  89. logger.log project_id: project_id, doc_id: doc_id, rawUpdates: rawUpdates, "retrieved raw updates from redis"
  90. UpdatesManager.compressAndSaveRawUpdates project_id, doc_id, rawUpdates, temporary, (error) ->
  91. return callback(error) if error?
  92. logger.log project_id: project_id, doc_id: doc_id, "compressed and saved doc updates"
  93. # delete the applied updates from redis
  94. RedisManager.deleteAppliedDocUpdates project_id, doc_id, docUpdates, (error) ->
  95. return callback(error) if error?
  96. if length == UpdatesManager.REDIS_READ_BATCH_SIZE
  97. # There might be more updates
  98. logger.log project_id: project_id, doc_id: doc_id, "continuing processing updates"
  99. setTimeout () ->
  100. UpdatesManager.processUncompressedUpdates project_id, doc_id, temporary, callback
  101. , 0
  102. else
  103. logger.log project_id: project_id, doc_id: doc_id, "all raw updates processed"
  104. callback()
  105. # Process updates for a doc when we flush it individually
  106. processUncompressedUpdatesWithLock: (project_id, doc_id, callback = (error) ->) ->
  107. UpdatesManager._prepareProjectForUpdates project_id, (error, temporary) ->
  108. return callback(error) if error?
  109. UpdatesManager._processUncompressedUpdatesForDocWithLock project_id, doc_id, temporary, callback
  110. # Process updates for a doc when the whole project is flushed (internal method)
  111. _processUncompressedUpdatesForDocWithLock: (project_id, doc_id, temporary, callback = (error) ->) ->
  112. UpdatesManager._prepareDocForUpdates project_id, doc_id, (error) ->
  113. return callback(error) if error?
  114. LockManager.runWithLock(
  115. "HistoryLock:#{doc_id}",
  116. (releaseLock) ->
  117. UpdatesManager.processUncompressedUpdates project_id, doc_id, temporary, releaseLock
  118. callback
  119. )
  120. # Process all updates for a project, only check project-level information once
  121. processUncompressedUpdatesForProject: (project_id, callback = (error) ->) ->
  122. RedisManager.getDocIdsWithHistoryOps project_id, (error, doc_ids) ->
  123. return callback(error) if error?
  124. UpdatesManager._prepareProjectForUpdates project_id, (error, temporary) ->
  125. jobs = []
  126. for doc_id in doc_ids
  127. do (doc_id) ->
  128. jobs.push (cb) ->
  129. UpdatesManager._processUncompressedUpdatesForDocWithLock project_id, doc_id, temporary, cb
  130. async.parallelLimit jobs, 5, callback
  131. # flush all outstanding changes
  132. flushAll: (limit, callback = (error, result) ->) ->
  133. RedisManager.getProjectIdsWithHistoryOps (error, project_ids) ->
  134. return callback(error) if error?
  135. logger.log {count: project_ids?.length, project_ids: project_ids}, "found projects"
  136. jobs = []
  137. selectedProjects = if limit < 0 then project_ids else project_ids[0...limit]
  138. for project_id in selectedProjects
  139. do (project_id) ->
  140. jobs.push (cb) ->
  141. UpdatesManager.processUncompressedUpdatesForProject project_id, (err) ->
  142. return cb(null, {failed: err?, project_id: project_id})
  143. async.series jobs, (error, result) ->
  144. return callback(error) if error?
  145. failedProjects = (x.project_id for x in result when x.failed)
  146. succeededProjects = (x.project_id for x in result when not x.failed)
  147. callback(null, {failed: failedProjects, succeeded: succeededProjects, all: project_ids})
  148. getDanglingUpdates: (callback = (error, doc_ids) ->) ->
  149. RedisManager.getAllDocIdsWithHistoryOps (error, all_doc_ids) ->
  150. return callback(error) if error?
  151. RedisManager.getProjectIdsWithHistoryOps (error, all_project_ids) ->
  152. return callback(error) if error?
  153. # function to get doc_ids for each project
  154. task = (cb) -> async.concatSeries all_project_ids, RedisManager.getDocIdsWithHistoryOps, cb
  155. # find the dangling doc ids
  156. task (error, project_doc_ids) ->
  157. dangling_doc_ids = _.difference(all_doc_ids, project_doc_ids)
  158. logger.log {all_doc_ids: all_doc_ids, all_project_ids: all_project_ids, project_doc_ids: project_doc_ids, dangling_doc_ids: dangling_doc_ids}, "checking for dangling doc ids"
  159. callback(null, dangling_doc_ids)
  160. getDocUpdates: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
  161. UpdatesManager.processUncompressedUpdatesWithLock project_id, doc_id, (error) ->
  162. return callback(error) if error?
  163. #console.log "options", options
  164. PackManager.getOpsByVersionRange project_id, doc_id, options.from, options.to, (error, updates) ->
  165. return callback(error) if error?
  166. callback null, updates
  167. getDocUpdatesWithUserInfo: (project_id, doc_id, options = {}, callback = (error, updates) ->) ->
  168. UpdatesManager.getDocUpdates project_id, doc_id, options, (error, updates) ->
  169. return callback(error) if error?
  170. UpdatesManager.fillUserInfo updates, (error, updates) ->
  171. return callback(error) if error?
  172. callback null, updates
  173. getSummarizedProjectUpdates: (project_id, options = {}, callback = (error, updates) ->) ->
  174. options.min_count ||= 25
  175. summarizedUpdates = []
  176. before = options.before
  177. nextBeforeTimestamp = null
  178. UpdatesManager.processUncompressedUpdatesForProject project_id, (error) ->
  179. return callback(error) if error?
  180. PackManager.makeProjectIterator project_id, before, (err, iterator) ->
  181. return callback(err) if err?
  182. # repeatedly get updates and pass them through the summariser to get an final output with user info
  183. async.whilst () ->
  184. #console.log "checking iterator.done", iterator.done()
  185. return summarizedUpdates.length < options.min_count and not iterator.done()
  186. , (cb) ->
  187. iterator.next (err, partialUpdates) ->
  188. return callback(err) if err?
  189. #logger.log {partialUpdates}, 'got partialUpdates'
  190. return cb() if partialUpdates.length is 0 ## FIXME should try to avoid this happening
  191. nextBeforeTimestamp = partialUpdates[partialUpdates.length - 1].meta.end_ts
  192. # add the updates to the summary list
  193. summarizedUpdates = UpdatesManager._summarizeUpdates partialUpdates, summarizedUpdates
  194. cb()
  195. , () ->
  196. # finally done all updates
  197. #console.log 'summarized Updates', summarizedUpdates
  198. UpdatesManager.fillSummarizedUserInfo summarizedUpdates, (err, results) ->
  199. return callback(err) if err?
  200. callback null, results, if not iterator.done() then nextBeforeTimestamp else undefined
  201. fetchUserInfo: (users, callback = (error, fetchedUserInfo) ->) ->
  202. jobs = []
  203. fetchedUserInfo = {}
  204. for user_id of users
  205. do (user_id) ->
  206. jobs.push (callback) ->
  207. WebApiManager.getUserInfo user_id, (error, userInfo) ->
  208. return callback(error) if error?
  209. fetchedUserInfo[user_id] = userInfo
  210. callback()
  211. async.series jobs, (err) ->
  212. return callback(err) if err?
  213. callback(null, fetchedUserInfo)
  214. fillUserInfo: (updates, callback = (error, updates) ->) ->
  215. users = {}
  216. for update in updates
  217. user_id = update.meta.user_id
  218. if UpdatesManager._validUserId(user_id)
  219. users[user_id] = true
  220. UpdatesManager.fetchUserInfo users, (error, fetchedUserInfo) ->
  221. return callback(error) if error?
  222. for update in updates
  223. user_id = update.meta.user_id
  224. delete update.meta.user_id
  225. if UpdatesManager._validUserId(user_id)
  226. update.meta.user = fetchedUserInfo[user_id]
  227. callback null, updates
  228. fillSummarizedUserInfo: (updates, callback = (error, updates) ->) ->
  229. users = {}
  230. for update in updates
  231. user_ids = update.meta.user_ids or []
  232. for user_id in user_ids
  233. if UpdatesManager._validUserId(user_id)
  234. users[user_id] = true
  235. UpdatesManager.fetchUserInfo users, (error, fetchedUserInfo) ->
  236. return callback(error) if error?
  237. for update in updates
  238. user_ids = update.meta.user_ids or []
  239. update.meta.users = []
  240. delete update.meta.user_ids
  241. for user_id in user_ids
  242. if UpdatesManager._validUserId(user_id)
  243. update.meta.users.push fetchedUserInfo[user_id]
  244. else
  245. update.meta.users.push null
  246. callback null, updates
  247. _validUserId: (user_id) ->
  248. if !user_id?
  249. return false
  250. else
  251. return !!user_id.match(/^[a-f0-9]{24}$/)
  252. TIME_BETWEEN_DISTINCT_UPDATES: fiveMinutes = 5 * 60 * 1000
  253. SPLIT_ON_DELETE_SIZE: 16 # characters
  254. _summarizeUpdates: (updates, existingSummarizedUpdates = []) ->
  255. summarizedUpdates = existingSummarizedUpdates.slice()
  256. previousUpdateWasBigDelete = false
  257. for update in updates
  258. earliestUpdate = summarizedUpdates[summarizedUpdates.length - 1]
  259. shouldConcat = false
  260. # If a user inserts some text, then deletes a big chunk including that text,
  261. # the update we show might concat the insert and delete, and there will be no sign
  262. # of that insert having happened, or be able to restore to it (restoring after a big delete is common).
  263. # So, we split the summary on 'big' deletes. However, we've stepping backwards in time with
  264. # most recent changes considered first, so if this update is a big delete, we want to start
  265. # a new summarized update next timge, hence we monitor the previous update.
  266. if previousUpdateWasBigDelete
  267. shouldConcat = false
  268. else if earliestUpdate and earliestUpdate.meta.end_ts - update.meta.start_ts < @TIME_BETWEEN_DISTINCT_UPDATES
  269. # We're going backwards in time through the updates, so only combine if this update starts less than 5 minutes before
  270. # the end of current summarized block, so no block spans more than 5 minutes.
  271. shouldConcat = true
  272. isBigDelete = false
  273. for op in update.op or []
  274. if op.d? and op.d.length > @SPLIT_ON_DELETE_SIZE
  275. isBigDelete = true
  276. previousUpdateWasBigDelete = isBigDelete
  277. if shouldConcat
  278. # check if the user in this update is already present in the earliest update,
  279. # if not, add them to the users list of the earliest update
  280. earliestUpdate.meta.user_ids = _.union earliestUpdate.meta.user_ids, [update.meta.user_id]
  281. doc_id = update.doc_id.toString()
  282. doc = earliestUpdate.docs[doc_id]
  283. if doc?
  284. doc.fromV = Math.min(doc.fromV, update.v)
  285. doc.toV = Math.max(doc.toV, update.v)
  286. else
  287. earliestUpdate.docs[doc_id] =
  288. fromV: update.v
  289. toV: update.v
  290. earliestUpdate.meta.start_ts = Math.min(earliestUpdate.meta.start_ts, update.meta.start_ts)
  291. earliestUpdate.meta.end_ts = Math.max(earliestUpdate.meta.end_ts, update.meta.end_ts)
  292. else
  293. newUpdate =
  294. meta:
  295. user_ids: []
  296. start_ts: update.meta.start_ts
  297. end_ts: update.meta.end_ts
  298. docs: {}
  299. newUpdate.docs[update.doc_id.toString()] =
  300. fromV: update.v
  301. toV: update.v
  302. newUpdate.meta.user_ids.push update.meta.user_id
  303. summarizedUpdates.push newUpdate
  304. return summarizedUpdates