Merge pull request #16554 from overleaf/bg-clsi-timeout-filestore-graceful-shutdown

Add graceful shutdown to filestore

GitOrigin-RevId: a940fe1c68a32aa281b4be866e350e3e6b184c16

services/filestore/app.js

@@ -39,6 +39,19 @@ app.use(function (req, res, next) {
   next()
 })
 
+// Handle requests that come in after we've started shutting down
+app.use((req, res, next) => {
+  if (settings.shuttingDown) {
+    logger.warn(
+      { req, timeSinceShutdown: Date.now() - settings.shutDownTime },
+      'request received after shutting down'
+    )
+    // We don't want keep-alive connections to be kept open when the server is shutting down.
+    res.set('Connection', 'close')
+  }
+  next()
+})
+
 Metrics.injectMetricsRoute(app)
 
 app.head(
@@ -134,7 +147,11 @@ app.get(
 )
 
 app.get('/status', function (req, res) {
-  res.send('filestore sharelatex up')
+  if (settings.shuttingDown) {
+    res.sendStatus(503) // Service unavailable
+  } else {
+    res.send('filestore is up')
+  }
 })
 
 app.get('/health_check', healthCheckController.check)
@@ -164,4 +181,20 @@ process
     process.exit(1)
   })
 
+function handleShutdownSignal(signal) {
+  logger.info({ signal }, 'received interrupt, cleaning up')
+  if (settings.shuttingDown) {
+    logger.warn({ signal }, 'already shutting down, ignoring interrupt')
+    return
+  }
+  settings.shuttingDown = true
+  settings.shutDownTime = Date.now()
+  setTimeout(() => {
+    logger.info({ signal }, 'shutting down')
+    process.exit()
+  }, settings.delayShutdownMs)
+}
+
+process.on('SIGTERM', handleShutdownSignal)
+
 module.exports = app

services/filestore/config/settings.defaults.js

@@ -112,6 +112,8 @@ const settings = {
   sentry: {
     dsn: process.env.SENTRY_DSN,
   },
+
+  delayShutdownMs: parseInt(process.env.DELAY_SHUTDOWN_MS || '30000', 10),
 }
 
 // Filestore health check