app.coffee 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. CompileController = require "./app/js/CompileController"
  2. Settings = require "settings-sharelatex"
  3. logger = require "logger-sharelatex"
  4. logger.initialize("clsi")
  5. if Settings.sentry?.dsn?
  6. logger.initializeErrorReporting(Settings.sentry.dsn)
  7. smokeTest = require "smoke-test-sharelatex"
  8. ContentTypeMapper = require "./app/js/ContentTypeMapper"
  9. Errors = require './app/js/Errors'
  10. Path = require "path"
  11. fs = require "fs"
  12. Metrics = require "metrics-sharelatex"
  13. Metrics.initialize("clsi")
  14. Metrics.open_sockets.monitor(logger)
  15. Metrics.memory.monitor(logger)
  16. ProjectPersistenceManager = require "./app/js/ProjectPersistenceManager"
  17. OutputCacheManager = require "./app/js/OutputCacheManager"
  18. require("./app/js/db").sync()
  19. express = require "express"
  20. bodyParser = require "body-parser"
  21. app = express()
  22. app.use Metrics.http.monitor(logger)
  23. # Compile requests can take longer than the default two
  24. # minutes (including file download time), so bump up the
  25. # timeout a bit.
  26. TIMEOUT = 6 * 60 * 1000
  27. app.use (req, res, next) ->
  28. req.setTimeout TIMEOUT
  29. res.setTimeout TIMEOUT
  30. res.removeHeader("X-Powered-By")
  31. next()
  32. app.param 'project_id', (req, res, next, project_id) ->
  33. if project_id?.match /^[a-zA-Z0-9_-]+$/
  34. next()
  35. else
  36. next new Error("invalid project id")
  37. app.param 'user_id', (req, res, next, user_id) ->
  38. if user_id?.match /^[0-9a-f]{24}$/
  39. next()
  40. else
  41. next new Error("invalid user id")
  42. app.param 'build_id', (req, res, next, build_id) ->
  43. if build_id?.match OutputCacheManager.BUILD_REGEX
  44. next()
  45. else
  46. next new Error("invalid build id #{build_id}")
  47. app.post "/project/:project_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile
  48. app.post "/project/:project_id/compile/stop", CompileController.stopCompile
  49. app.delete "/project/:project_id", CompileController.clearCache
  50. app.get "/project/:project_id/sync/code", CompileController.syncFromCode
  51. app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf
  52. app.get "/project/:project_id/wordcount", CompileController.wordcount
  53. app.get "/project/:project_id/status", CompileController.status
  54. # Per-user containers
  55. app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile
  56. app.post "/project/:project_id/user/:user_id/compile/stop", CompileController.stopCompile
  57. app.delete "/project/:project_id/user/:user_id", CompileController.clearCache
  58. app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode
  59. app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf
  60. app.get "/project/:project_id/user/:user_id/wordcount", CompileController.wordcount
  61. ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks"
  62. # create a static server which does not allow access to any symlinks
  63. # avoids possible mismatch of root directory between middleware check
  64. # and serving the files
  65. staticServer = ForbidSymlinks express.static, Settings.path.compilesDir, setHeaders: (res, path, stat) ->
  66. if Path.basename(path) == "output.pdf"
  67. # Calculate an etag in the same way as nginx
  68. # https://github.com/tj/send/issues/65
  69. etag = (path, stat) ->
  70. '"' + Math.ceil(+stat.mtime / 1000).toString(16) +
  71. '-' + Number(stat.size).toString(16) + '"'
  72. res.set("Etag", etag(path, stat))
  73. res.set("Content-Type", ContentTypeMapper.map(path))
  74. app.get "/project/:project_id/user/:user_id/build/:build_id/output/*", (req, res, next) ->
  75. # for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
  76. req.url = "/#{req.params.project_id}-#{req.params.user_id}/" + OutputCacheManager.path(req.params.build_id, "/#{req.params[0]}")
  77. staticServer(req, res, next)
  78. app.get "/project/:project_id/build/:build_id/output/*", (req, res, next) ->
  79. # for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
  80. req.url = "/#{req.params.project_id}/" + OutputCacheManager.path(req.params.build_id, "/#{req.params[0]}")
  81. staticServer(req, res, next)
  82. app.get "/project/:project_id/user/:user_id/output/*", (req, res, next) ->
  83. # for specific user get the path to the top level file
  84. req.url = "/#{req.params.project_id}-#{req.params.user_id}/#{req.params[0]}"
  85. staticServer(req, res, next)
  86. app.get "/project/:project_id/output/*", (req, res, next) ->
  87. if req.query?.build? && req.query.build.match(OutputCacheManager.BUILD_REGEX)
  88. # for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
  89. req.url = "/#{req.params.project_id}/" + OutputCacheManager.path(req.query.build, "/#{req.params[0]}")
  90. else
  91. req.url = "/#{req.params.project_id}/#{req.params[0]}"
  92. staticServer(req, res, next)
  93. app.get "/oops", (req, res, next) ->
  94. logger.error {err: "hello"}, "test error"
  95. res.send "error\n"
  96. app.get "/status", (req, res, next) ->
  97. res.send "CLSI is alive\n"
  98. resCacher =
  99. contentType:(@setContentType)->
  100. send:(@code, @body)->
  101. #default the server to be down
  102. code:500
  103. body:{}
  104. setContentType:"application/json"
  105. if Settings.smokeTest
  106. do runSmokeTest = ->
  107. logger.log("running smoke tests")
  108. smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))({}, resCacher)
  109. setTimeout(runSmokeTest, 30 * 1000)
  110. app.get "/health_check", (req, res)->
  111. res.contentType(resCacher?.setContentType)
  112. res.status(resCacher?.code).send(resCacher?.body)
  113. app.get "/smoke_test_force", (req, res)->
  114. smokeTest.run(require.resolve(__dirname + "/test/smoke/js/SmokeTests.js"))(req, res)
  115. profiler = require "v8-profiler"
  116. app.get "/profile", (req, res) ->
  117. time = parseInt(req.query.time || "1000")
  118. profiler.startProfiling("test")
  119. setTimeout () ->
  120. profile = profiler.stopProfiling("test")
  121. res.json(profile)
  122. , time
  123. app.get "/heapdump", (req, res)->
  124. require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.clsi.heapsnapshot', (err, filename)->
  125. res.send filename
  126. app.use (error, req, res, next) ->
  127. if error instanceof Errors.NotFoundError
  128. logger.warn {err: error, url: req.url}, "not found error"
  129. return res.sendStatus(404)
  130. else
  131. logger.error {err: error, url: req.url}, "server error"
  132. res.sendStatus(error?.statusCode || 500)
  133. net = require "net"
  134. os = require "os"
  135. STATE = "up"
  136. loadTcpServer = net.createServer (socket) ->
  137. socket.on "error", (err)->
  138. if err.code == "ECONNRESET"
  139. # this always comes up, we don't know why
  140. return
  141. logger.err err:err, "error with socket on load check"
  142. socket.destroy()
  143. if STATE == "up" and Settings.internal.load_balancer_agent.report_load
  144. currentLoad = os.loadavg()[0]
  145. # staging clis's have 1 cpu core only
  146. if os.cpus().length == 1
  147. availableWorkingCpus = 1
  148. else
  149. availableWorkingCpus = os.cpus().length - 1
  150. freeLoad = availableWorkingCpus - currentLoad
  151. freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
  152. if freeLoadPercentage <= 0
  153. freeLoadPercentage = 1 # when its 0 the server is set to drain and will move projects to different servers
  154. socket.write("up, #{freeLoadPercentage}%\n", "ASCII")
  155. socket.end()
  156. else
  157. socket.write("#{STATE}\n", "ASCII")
  158. socket.end()
  159. loadHttpServer = express()
  160. loadHttpServer.post "/state/up", (req, res, next) ->
  161. STATE = "up"
  162. logger.info "getting message to set server to down"
  163. res.sendStatus 204
  164. loadHttpServer.post "/state/down", (req, res, next) ->
  165. STATE = "down"
  166. logger.info "getting message to set server to down"
  167. res.sendStatus 204
  168. loadHttpServer.post "/state/maint", (req, res, next) ->
  169. STATE = "maint"
  170. logger.info "getting message to set server to maint"
  171. res.sendStatus 204
  172. port = (Settings.internal?.clsi?.port or 3013)
  173. host = (Settings.internal?.clsi?.host or "localhost")
  174. load_tcp_port = Settings.internal.load_balancer_agent.load_port
  175. load_http_port = Settings.internal.load_balancer_agent.local_port
  176. if !module.parent # Called directly
  177. app.listen port, host, (error) ->
  178. logger.info "CLSI starting up, listening on #{host}:#{port}"
  179. loadTcpServer.listen load_tcp_port, host, (error) ->
  180. throw error if error?
  181. logger.info "Load tcp agent listening on load port #{load_tcp_port}"
  182. loadHttpServer.listen load_http_port, host, (error) ->
  183. throw error if error?
  184. logger.info "Load http agent listening on load port #{load_http_port}"
  185. module.exports = app
  186. setInterval () ->
  187. ProjectPersistenceManager.clearExpiredProjects()
  188. , tenMinutes = 10 * 60 * 1000