Merge pull request #210 from overleaf/bg-real-time-status-file

add blue-green deployment support with load-balancer health-checks

services/real-time/app.js

@@ -22,6 +22,7 @@ const CookieParser = require('cookie-parser')
 
 const DrainManager = require('./app/js/DrainManager')
 const HealthCheckManager = require('./app/js/HealthCheckManager')
+const DeploymentManager = require('./app/js/DeploymentManager')
 
 // NOTE: debug is invoked for every blob that is put on the wire
 const socketIoLogger = {
@@ -36,6 +37,9 @@ const socketIoLogger = {
   log() {}
 }
 
+// monitor status file to take dark deployments out of the load-balancer
+DeploymentManager.initialise()
+
 // Set up socket.io server
 const app = express()
 
@@ -79,13 +83,20 @@ io.configure(function () {
 })
 
 // a 200 response on '/' is required for load balancer health checks
-app.get('/', (req, res) => res.send('real-time-sharelatex is alive'))
+// these operate separately from kubernetes readiness checks
+app.get('/', function (req, res) {
+  if (Settings.shutDownInProgress || DeploymentManager.deploymentIsClosed()) {
+    res.sendStatus(503) // Service unavailable
+  } else {
+    res.send('real-time is open')
+  }
+})
 
 app.get('/status', function (req, res) {
   if (Settings.shutDownInProgress) {
     res.sendStatus(503) // Service unavailable
   } else {
-    res.send('real-time-sharelatex is alive')
+    res.send('real-time is alive')
   }
 })
 

services/real-time/app/js/DeploymentManager.js

@@ -0,0 +1,59 @@
+const logger = require('logger-sharelatex')
+const settings = require('settings-sharelatex')
+const fs = require('fs')
+
+// Monitor a status file (e.g. /etc/real_time_status) periodically and close the
+// service if the file contents don't contain the matching deployment colour.
+
+const FILE_CHECK_INTERVAL = 5000
+const statusFile = settings.deploymentFile
+const deploymentColour = settings.deploymentColour
+
+var serviceCloseTime
+
+function updateDeploymentStatus(fileContent) {
+  const closed = fileContent && !fileContent.includes(deploymentColour)
+  if (closed && !settings.serviceIsClosed) {
+    settings.serviceIsClosed = true
+    serviceCloseTime = Date.now() + 60 * 1000 // delay closing by 1 minute
+    logger.warn({ fileContent }, 'closing service')
+  } else if (!closed && settings.serviceIsClosed) {
+    settings.serviceIsClosed = false
+    logger.warn({ fileContent }, 'opening service')
+  }
+}
+
+function pollStatusFile() {
+  fs.readFile(statusFile, { encoding: 'utf8' }, (err, fileContent) => {
+    if (err) {
+      logger.error(
+        { file: statusFile, fsErr: err },
+        'error reading service status file'
+      )
+      return
+    }
+    updateDeploymentStatus(fileContent)
+  })
+}
+
+function checkStatusFileSync() {
+  // crash on start up if file does not exist
+  const content = fs.readFileSync(statusFile, { encoding: 'utf8' })
+  updateDeploymentStatus(content)
+}
+
+module.exports = {
+  initialise() {
+    if (statusFile && deploymentColour) {
+      logger.log(
+        { statusFile, deploymentColour, interval: FILE_CHECK_INTERVAL },
+        'monitoring deployment status file'
+      )
+      checkStatusFileSync() // perform an initial synchronous check at start up
+      setInterval(pollStatusFile, FILE_CHECK_INTERVAL) // continue checking periodically
+    }
+  },
+  deploymentIsClosed() {
+    return settings.serviceIsClosed && Date.now() > serviceCloseTime
+  }
+}

services/real-time/config/settings.defaults.js

@@ -148,6 +148,12 @@ const settings = {
 
   statusCheckInterval: parseInt(process.env.STATUS_CHECK_INTERVAL || '0'),
 
+  // The deployment colour for this app (if any). Used for blue green deploys.
+  deploymentColour: process.env.DEPLOYMENT_COLOUR,
+  // Load balancer health checks will return 200 only when this file contains
+  // the deployment colour for this app.
+  deploymentFile: process.env.DEPLOYMENT_FILE,
+
   sentry: {
     dsn: process.env.SENTRY_DSN
   },