LockManager.coffee 2.8 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182
  1. metrics = require('metrics-sharelatex')
  2. Settings = require('settings-sharelatex')
  3. RedisWrapper = require("./RedisWrapper")
  4. rclient = RedisWrapper.client("lock")
  5. logger = require "logger-sharelatex"
  6. module.exports = LockManager =
  7. LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock
  8. MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock
  9. REDIS_LOCK_EXPIRY: 30 # seconds. Time until lock auto expires in redis
  10. SLOW_EXECUTION_THRESHOLD: 5000 # 5s, if execution takes longer than this then log
  11. runWithLock: (namespace, id, runner = ( (releaseLock = (error) ->) -> ), callback = ( (error) -> )) ->
  12. # The lock can expire in redis but the process carry on. This setTimout call
  13. # is designed to log if this happens.
  14. #
  15. # error is defined here so we get a useful stacktrace
  16. lockReleased = false
  17. slowExecutionError = new Error "slow execution during lock"
  18. countIfExceededLockTimeout = () ->
  19. if !lockReleased
  20. metrics.inc "lock.#{namespace}.exceeded_lock_timeout"
  21. logger.log "exceeded lock timeout", { namespace, id, slowExecutionError }
  22. setTimeout countIfExceededLockTimeout, LockManager.REDIS_LOCK_EXPIRY * 1000
  23. timer = new metrics.Timer("lock.#{namespace}")
  24. key = "lock:web:#{namespace}:#{id}"
  25. LockManager._getLock key, (error) ->
  26. return callback(error) if error?
  27. runner (error1, values...) ->
  28. LockManager._releaseLock key, (error2) ->
  29. lockReleased = true
  30. timeTaken = new Date - timer.start
  31. if timeTaken > LockManager.SLOW_EXECUTION_THRESHOLD
  32. logger.log "slow execution during lock", { namespace, id, timeTaken, slowExecutionError }
  33. timer.done()
  34. error = error1 or error2
  35. return callback(error) if error?
  36. callback null, values...
  37. _tryLock : (key, callback = (err, isFree)->)->
  38. rclient.set key, "locked", "EX", LockManager.REDIS_LOCK_EXPIRY, "NX", (err, gotLock)->
  39. return callback(err) if err?
  40. if gotLock == "OK"
  41. metrics.inc "lock-not-blocking"
  42. callback err, true
  43. else
  44. metrics.inc "lock-blocking"
  45. logger.log key: key, redis_response: gotLock, "lock is locked"
  46. callback err, false
  47. _getLock: (key, callback = (error) ->) ->
  48. startTime = Date.now()
  49. do attempt = () ->
  50. if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME
  51. return callback(new Error("Timeout"))
  52. LockManager._tryLock key, (error, gotLock) ->
  53. return callback(error) if error?
  54. if gotLock
  55. callback(null)
  56. else
  57. setTimeout attempt, LockManager.LOCK_TEST_INTERVAL
  58. _checkLock: (key, callback = (err, isFree)->)->
  59. multi = rclient.multi()
  60. multi.exists key
  61. multi.exec (err, replys)->
  62. return callback(err) if err?
  63. exists = parseInt replys[0]
  64. if exists == 1
  65. metrics.inc "lock-blocking"
  66. callback err, false
  67. else
  68. metrics.inc "lock-not-blocking"
  69. callback err, true
  70. _releaseLock: (key, callback)->
  71. rclient.del key, callback