app.js 7.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232
  1. const Metrics = require('metrics-sharelatex')
  2. const Settings = require('settings-sharelatex')
  3. Metrics.initialize(Settings.appName || 'real-time')
  4. const async = require('async')
  5. const logger = require('logger-sharelatex')
  6. logger.initialize('real-time')
  7. Metrics.event_loop.monitor(logger)
  8. const express = require('express')
  9. const session = require('express-session')
  10. const redis = require('redis-sharelatex')
  11. if (Settings.sentry && Settings.sentry.dsn) {
  12. logger.initializeErrorReporting(Settings.sentry.dsn)
  13. }
  14. const sessionRedisClient = redis.createClient(Settings.redis.websessions)
  15. const RedisStore = require('connect-redis')(session)
  16. const SessionSockets = require('./app/js/SessionSockets')
  17. const CookieParser = require('cookie-parser')
  18. const DrainManager = require('./app/js/DrainManager')
  19. const HealthCheckManager = require('./app/js/HealthCheckManager')
  20. // work around frame handler bug in socket.io v0.9.16
  21. require('./socket.io.patch.js')
  22. // Set up socket.io server
  23. const app = express()
  24. const server = require('http').createServer(app)
  25. const io = require('socket.io').listen(server)
  26. // Bind to sessions
  27. const sessionStore = new RedisStore({ client: sessionRedisClient })
  28. const cookieParser = CookieParser(Settings.security.sessionSecret)
  29. const sessionSockets = new SessionSockets(
  30. io,
  31. sessionStore,
  32. cookieParser,
  33. Settings.cookieName
  34. )
  35. Metrics.injectMetricsRoute(app)
  36. app.use(Metrics.http.monitor(logger))
  37. io.configure(function () {
  38. io.enable('browser client minification')
  39. io.enable('browser client etag')
  40. // Fix for Safari 5 error of "Error during WebSocket handshake: location mismatch"
  41. // See http://answers.dotcloud.com/question/578/problem-with-websocket-over-ssl-in-safari-with
  42. io.set('match origin protocol', true)
  43. // gzip uses a Node 0.8.x method of calling the gzip program which
  44. // doesn't work with 0.6.x
  45. // io.enable('browser client gzip')
  46. io.set('transports', [
  47. 'websocket',
  48. 'flashsocket',
  49. 'htmlfile',
  50. 'xhr-polling',
  51. 'jsonp-polling'
  52. ])
  53. io.set('log level', 1)
  54. })
  55. app.get('/', (req, res) => res.send('real-time-sharelatex is alive'))
  56. app.get('/status', function (req, res) {
  57. if (Settings.shutDownInProgress) {
  58. res.send(503) // Service unavailable
  59. } else {
  60. res.send('real-time-sharelatex is alive')
  61. }
  62. })
  63. app.get('/debug/events', function (req, res) {
  64. Settings.debugEvents = parseInt(req.query.count, 10) || 20
  65. logger.log({ count: Settings.debugEvents }, 'starting debug mode')
  66. res.send(`debug mode will log next ${Settings.debugEvents} events`)
  67. })
  68. const rclient = require('redis-sharelatex').createClient(
  69. Settings.redis.realtime
  70. )
  71. function healthCheck(req, res) {
  72. rclient.healthCheck(function (error) {
  73. if (error) {
  74. logger.err({ err: error }, 'failed redis health check')
  75. res.sendStatus(500)
  76. } else if (HealthCheckManager.isFailing()) {
  77. const status = HealthCheckManager.status()
  78. logger.err({ pubSubErrors: status }, 'failed pubsub health check')
  79. res.sendStatus(500)
  80. } else {
  81. res.sendStatus(200)
  82. }
  83. })
  84. }
  85. app.get('/health_check', healthCheck)
  86. app.get('/health_check/redis', healthCheck)
  87. const Router = require('./app/js/Router')
  88. Router.configure(app, io, sessionSockets)
  89. const WebsocketLoadBalancer = require('./app/js/WebsocketLoadBalancer')
  90. WebsocketLoadBalancer.listenForEditorEvents(io)
  91. const DocumentUpdaterController = require('./app/js/DocumentUpdaterController')
  92. DocumentUpdaterController.listenForUpdatesFromDocumentUpdater(io)
  93. const { port } = Settings.internal.realTime
  94. const { host } = Settings.internal.realTime
  95. server.listen(port, host, function (error) {
  96. if (error) {
  97. throw error
  98. }
  99. logger.info(`realtime starting up, listening on ${host}:${port}`)
  100. })
  101. // Stop huge stack traces in logs from all the socket.io parsing steps.
  102. Error.stackTraceLimit = 10
  103. function shutdownCleanly(signal) {
  104. const connectedClients = io.sockets.clients().length
  105. if (connectedClients === 0) {
  106. logger.warn('no clients connected, exiting')
  107. process.exit()
  108. } else {
  109. logger.warn(
  110. { connectedClients },
  111. 'clients still connected, not shutting down yet'
  112. )
  113. setTimeout(() => shutdownCleanly(signal), 30 * 1000)
  114. }
  115. }
  116. function drainAndShutdown(signal) {
  117. if (Settings.shutDownInProgress) {
  118. logger.warn({ signal }, 'shutdown already in progress, ignoring signal')
  119. } else {
  120. Settings.shutDownInProgress = true
  121. const { statusCheckInterval } = Settings
  122. if (statusCheckInterval) {
  123. logger.warn(
  124. { signal },
  125. `received interrupt, delay drain by ${statusCheckInterval}ms`
  126. )
  127. }
  128. setTimeout(function () {
  129. logger.warn(
  130. { signal },
  131. `received interrupt, starting drain over ${shutdownDrainTimeWindow} mins`
  132. )
  133. DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
  134. shutdownCleanly(signal)
  135. }, statusCheckInterval)
  136. }
  137. }
  138. Settings.shutDownInProgress = false
  139. const shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10)
  140. if (Settings.shutdownDrainTimeWindow) {
  141. logger.log({ shutdownDrainTimeWindow }, 'shutdownDrainTimeWindow enabled')
  142. for (const signal of [
  143. 'SIGINT',
  144. 'SIGHUP',
  145. 'SIGQUIT',
  146. 'SIGUSR1',
  147. 'SIGUSR2',
  148. 'SIGTERM',
  149. 'SIGABRT'
  150. ]) {
  151. process.on(signal, drainAndShutdown)
  152. } // signal is passed as argument to event handler
  153. // global exception handler
  154. if (Settings.errors && Settings.errors.catchUncaughtErrors) {
  155. process.removeAllListeners('uncaughtException')
  156. process.on('uncaughtException', function (error) {
  157. if (['EPIPE', 'ECONNRESET'].includes(error.code)) {
  158. Metrics.inc('disconnected_write', 1, { status: error.code })
  159. return logger.warn(
  160. { err: error },
  161. 'attempted to write to disconnected client'
  162. )
  163. }
  164. logger.error({ err: error }, 'uncaught exception')
  165. if (Settings.errors && Settings.errors.shutdownOnUncaughtError) {
  166. drainAndShutdown('SIGABRT')
  167. }
  168. })
  169. }
  170. }
  171. if (Settings.continualPubsubTraffic) {
  172. logger.warn('continualPubsubTraffic enabled')
  173. const pubsubClient = redis.createClient(Settings.redis.pubsub)
  174. const clusterClient = redis.createClient(Settings.redis.websessions)
  175. const publishJob = function (channel, callback) {
  176. const checker = new HealthCheckManager(channel)
  177. logger.debug({ channel }, 'sending pub to keep connection alive')
  178. const json = JSON.stringify({
  179. health_check: true,
  180. key: checker.id,
  181. date: new Date().toString()
  182. })
  183. Metrics.summary(`redis.publish.${channel}`, json.length)
  184. pubsubClient.publish(channel, json, function (err) {
  185. if (err) {
  186. logger.err({ err, channel }, 'error publishing pubsub traffic to redis')
  187. }
  188. const blob = JSON.stringify({ keep: 'alive' })
  189. Metrics.summary('redis.publish.cluster-continual-traffic', blob.length)
  190. clusterClient.publish('cluster-continual-traffic', blob, callback)
  191. })
  192. }
  193. const runPubSubTraffic = () =>
  194. async.map(['applied-ops', 'editor-events'], publishJob, () =>
  195. setTimeout(runPubSubTraffic, 1000 * 20)
  196. )
  197. runPubSubTraffic()
  198. }