Merge pull request #84 from overleaf/spd-catch-errors

Bump to Node 10 and add error handlers for socket.io

services/real-time/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:6.15.1 as app
+FROM node:10.16.3 as app
 
 WORKDIR /app
 
@@ -12,7 +12,7 @@ COPY . /app
 
 RUN npm run compile:all
 
-FROM node:6.15.1
+FROM node:10.16.3
 
 COPY --from=app /app /app
 

services/real-time/app.coffee

@@ -119,22 +119,32 @@ shutdownCleanly = (signal) ->
 			shutdownCleanly(signal)
 		, 10000
 
+drainAndShutdown = (signal) ->
+	if Settings.shutDownInProgress
+		logger.log signal: signal, "shutdown already in progress, ignoring signal"
+		return
+	else
+		Settings.shutDownInProgress = true
+		logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins"
+		DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
+		shutdownCleanly(signal)
+
+
 Settings.shutDownInProgress = false
 if Settings.shutdownDrainTimeWindow?
 	shutdownDrainTimeWindow = parseInt(Settings.shutdownDrainTimeWindow, 10)
 	logger.log shutdownDrainTimeWindow: shutdownDrainTimeWindow,"shutdownDrainTimeWindow enabled"
 	for signal in ['SIGINT', 'SIGHUP', 'SIGQUIT', 'SIGUSR1', 'SIGUSR2', 'SIGTERM', 'SIGABRT']
 		process.on signal, ->
-			if Settings.shutDownInProgress
-				logger.log signal: signal, "shutdown already in progress, ignoring signal"
-				return
-			else
-				Settings.shutDownInProgress = true
-				logger.warn signal: signal, "received interrupt, starting drain over #{shutdownDrainTimeWindow} mins"
-				DrainManager.startDrainTimeWindow(io, shutdownDrainTimeWindow)
-				shutdownCleanly(signal)
-
-
+			drainAndShutdown(signal)
+
+	# global exception handler
+	if Settings.errors?.catchUncaughtErrors
+		process.removeAllListeners('uncaughtException')
+		process.on 'uncaughtException', (error) ->
+			logger.error err: error, 'uncaught exception'
+			if Settings.errors?.shutdownOnUncaughtError
+				drainAndShutdown('SIGABRT')
 
 if Settings.continualPubsubTraffic
 	console.log "continualPubsubTraffic enabled"

services/real-time/app/coffee/DrainManager.coffee

@@ -36,4 +36,4 @@ module.exports = DrainManager =
 			if haveDrainedNClients
 				break
 		if drainedCount < N
-			logger.log "All clients have been told to reconnectGracefully"
+			logger.log "All clients have been told to reconnectGracefully"

services/real-time/app/coffee/Router.coffee

@@ -42,6 +42,12 @@ module.exports = Router =
 		app.post "/drain", httpAuth, HttpApiController.startDrain
 
 		session.on 'connection', (error, client, session) ->
+			client?.on "error", (err) ->
+				logger.err { clientErr: err }, "socket.io client error"
+				if client.connected
+					client.emit("reconnectGracefully")
+					client.disconnect()
+
 			if settings.shutDownInProgress
 				client.emit("connectionRejected", {message: "retry"})
 				client.disconnect()

services/real-time/config/settings.defaults.coffee

@@ -62,6 +62,10 @@ settings =
 
 	sentry:
 		dsn: process.env.SENTRY_DSN
+
+	errors:
+		catchUncaughtErrors: true
+		shutdownOnUncaughtError: true
 	
 # console.log settings.redis
 module.exports = settings