PackWorker.coffee 3.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107
  1. Settings = require "settings-sharelatex"
  2. async = require "async"
  3. _ = require "underscore"
  4. {db, ObjectId, BSON} = require "./mongojs"
  5. logger = require "logger-sharelatex"
  6. logger.initialize("track-changes-packworker")
  7. if Settings.sentry?.dsn?
  8. logger.initializeErrorReporting(Settings.sentry.dsn)
  9. DAYS = 24 * 3600 * 1000
  10. LockManager = require "./LockManager"
  11. PackManager = require "./PackManager"
  12. # this worker script is forked by the main process to look for
  13. # document histories which can be archived
  14. LIMIT = Number(process.argv[2]) || 1000
  15. DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000
  16. TIMEOUT = Number(process.argv[4]) || 30*60*1000
  17. shutDownRequested = false
  18. shutDownTimer = setTimeout () ->
  19. logger.log "pack timed out, requesting shutdown"
  20. # start the shutdown on the next pack
  21. shutDownRequested = true
  22. # do a hard shutdown after a further 5 minutes
  23. setTimeout () ->
  24. logger.error "HARD TIMEOUT in pack archive worker"
  25. process.exit()
  26. , 5*60*1000
  27. , TIMEOUT
  28. logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}"
  29. # work around for https://github.com/mafintosh/mongojs/issues/224
  30. db.close = (callback) ->
  31. this._getServer (err, server) ->
  32. return callback(err) if err?
  33. server = if server.destroy? then server else server.topology
  34. server.destroy(true, true)
  35. callback()
  36. finish = () ->
  37. if shutDownTimer?
  38. logger.log 'cancelling timeout'
  39. clearTimeout shutDownTimer
  40. logger.log 'closing db'
  41. db.close () ->
  42. logger.log 'closing LockManager Redis Connection'
  43. LockManager.close () ->
  44. logger.log 'ready to exit from pack archive worker'
  45. hardTimeout = setTimeout () ->
  46. logger.error 'hard exit from pack archive worker'
  47. process.exit(1)
  48. , 5*1000
  49. hardTimeout.unref()
  50. process.on 'exit', (code) ->
  51. logger.log {code}, 'pack archive worker exited'
  52. processUpdates = (pending) ->
  53. async.eachSeries pending, (result, callback) ->
  54. {_id, project_id, doc_id} = result
  55. if not project_id? or not doc_id?
  56. logger.log {project_id, doc_id}, "skipping pack, missing project/doc id"
  57. return callback()
  58. PackManager.processOldPack project_id, doc_id, _id, (err, result) ->
  59. if err?
  60. logger.error {err, result}, "error in pack archive worker"
  61. return callback(err)
  62. if shutDownRequested
  63. logger.error "shutting down pack archive worker"
  64. return callback(new Error("shutdown"))
  65. setTimeout () ->
  66. callback(err, result)
  67. , DOCUMENT_PACK_DELAY
  68. , (err, results) ->
  69. if err? and err.message != "shutdown"
  70. logger.error {err}, 'error in pack archive worker processUpdates'
  71. finish()
  72. # find the packs which can be archived
  73. ObjectIdFromDate = (date) ->
  74. id = Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
  75. return ObjectId(id)
  76. # new approach, two passes
  77. # find packs to be marked as finalised:true, those which have a newer pack present
  78. # then only consider finalised:true packs for archiving
  79. db.docHistory.find({
  80. expiresAt: {$exists: false}
  81. project_id: {$exists: true}
  82. v_end: {$exists: true}
  83. _id: {$lt: ObjectIdFromDate(new Date(Date.now() - 7 * DAYS))}
  84. }, {_id:1, doc_id:1, project_id:1}).sort({
  85. last_checked:1
  86. }).limit LIMIT, (err, results) ->
  87. if err?
  88. logger.log {err}, 'error checking for updates'
  89. finish()
  90. return
  91. pending = _.uniq results, false, (result) -> result.doc_id.toString()
  92. logger.log "found #{pending.length} documents to archive"
  93. processUpdates pending