PackWorker.coffee 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127
  1. Settings = require "settings-sharelatex"
  2. async = require "async"
  3. _ = require "underscore"
  4. {db, ObjectId, BSON} = require "./mongojs"
  5. fs = require "fs"
  6. logger = require "logger-sharelatex"
  7. logger.initialize("track-changes-packworker")
  8. if Settings.sentry?.dsn?
  9. logger.initializeErrorReporting(Settings.sentry.dsn)
  10. DAYS = 24 * 3600 * 1000
  11. LockManager = require "./LockManager"
  12. PackManager = require "./PackManager"
  13. # this worker script is forked by the main process to look for
  14. # document histories which can be archived
  15. source = process.argv[2]
  16. DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000
  17. TIMEOUT = Number(process.argv[4]) || 30*60*1000
  18. if source.match(/[^0-9]/)
  19. file = fs.readFileSync source
  20. result = for line in file.toString().split('\n')
  21. [project_id, doc_id] = line.split(' ')
  22. {doc_id, project_id}
  23. pending = _.filter result, (row) -> row?.doc_id?.match(/^[a-f0-9]{24}$/)
  24. else
  25. LIMIT = Number(process.argv[2]) || 1000
  26. shutDownRequested = false
  27. shutDownTimer = setTimeout () ->
  28. logger.log "pack timed out, requesting shutdown"
  29. # start the shutdown on the next pack
  30. shutDownRequested = true
  31. # do a hard shutdown after a further 5 minutes
  32. setTimeout () ->
  33. logger.error "HARD TIMEOUT in pack archive worker"
  34. process.exit()
  35. , 5*60*1000
  36. , TIMEOUT
  37. logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}"
  38. # work around for https://github.com/mafintosh/mongojs/issues/224
  39. db.close = (callback) ->
  40. this._getServer (err, server) ->
  41. return callback(err) if err?
  42. server = if server.destroy? then server else server.topology
  43. server.destroy(true, true)
  44. callback()
  45. finish = () ->
  46. if shutDownTimer?
  47. logger.log 'cancelling timeout'
  48. clearTimeout shutDownTimer
  49. logger.log 'closing db'
  50. db.close () ->
  51. logger.log 'closing LockManager Redis Connection'
  52. LockManager.close () ->
  53. logger.log 'ready to exit from pack archive worker'
  54. hardTimeout = setTimeout () ->
  55. logger.error 'hard exit from pack archive worker'
  56. process.exit(1)
  57. , 5*1000
  58. hardTimeout.unref()
  59. process.on 'exit', (code) ->
  60. logger.log {code}, 'pack archive worker exited'
  61. processUpdates = (pending) ->
  62. async.eachSeries pending, (result, callback) ->
  63. {_id, project_id, doc_id} = result
  64. logger.log {project_id, doc_id}, "processing"
  65. if not project_id? or not doc_id?
  66. logger.log {project_id, doc_id}, "skipping pack, missing project/doc id"
  67. return callback()
  68. handler = (err, result) ->
  69. if err?
  70. logger.error {err, result}, "error in pack archive worker"
  71. return callback(err)
  72. if shutDownRequested
  73. logger.error "shutting down pack archive worker"
  74. return callback(new Error("shutdown"))
  75. setTimeout () ->
  76. callback(err, result)
  77. , DOCUMENT_PACK_DELAY
  78. if not _id?
  79. PackManager.pushOldPacks project_id, doc_id, handler
  80. else
  81. PackManager.processOldPack project_id, doc_id, _id, handler
  82. , (err, results) ->
  83. if err? and err.message != "shutdown"
  84. logger.error {err}, 'error in pack archive worker processUpdates'
  85. finish()
  86. # find the packs which can be archived
  87. ObjectIdFromDate = (date) ->
  88. id = Math.floor(date.getTime() / 1000).toString(16) + "0000000000000000";
  89. return ObjectId(id)
  90. # new approach, two passes
  91. # find packs to be marked as finalised:true, those which have a newer pack present
  92. # then only consider finalised:true packs for archiving
  93. if pending?
  94. logger.log "got #{pending.length} entries from #{source}"
  95. processUpdates pending
  96. else
  97. db.docHistory.find({
  98. expiresAt: {$exists: false}
  99. project_id: {$exists: true}
  100. v_end: {$exists: true}
  101. $or: [ {n:512}, {n:1024} ]
  102. _id: {$lt: ObjectIdFromDate(new Date(Date.now() - 7 * DAYS))}
  103. }, {_id:1, doc_id:1, project_id:1}).sort({
  104. last_checked:1
  105. }).limit LIMIT, (err, results) ->
  106. if err?
  107. logger.log {err}, 'error checking for updates'
  108. finish()
  109. return
  110. pending = _.uniq results, false, (result) -> result.doc_id.toString()
  111. logger.log "found #{pending.length} documents to archive"
  112. processUpdates pending