Merge pull request #155 from overleaf/ho-limit-lifespan

limit clsi lifespan via health checks and PROCESS_LIFE_SPAN_LIMIT_MS

services/clsi/app.js

@@ -22,7 +22,6 @@ const ContentTypeMapper = require('./app/js/ContentTypeMapper')
 const Errors = require('./app/js/Errors')
 
 const Path = require('path')
-const fs = require('fs')
 
 Metrics.open_sockets.monitor(logger)
 Metrics.memory.monitor(logger)
@@ -208,23 +207,41 @@ const resCacher = {
   setContentType: 'application/json'
 }
 
+let shutdownTime
+if (Settings.processLifespanLimitMs) {
+  Settings.processLifespanLimitMs +=
+    Settings.processLifespanLimitMs * (Math.random() / 10)
+  shutdownTime = Date.now() + Settings.processLifespanLimitMs
+  logger.info('Lifespan limited to ', shutdownTime)
+}
+
+const checkIfProcessIsTooOld = function(cont) {
+  if (shutdownTime && shutdownTime < Date.now()) {
+    logger.log('shutting down, process is too old')
+    resCacher.send = function() {}
+    resCacher.code = 500
+    resCacher.body = { processToOld: true }
+  } else {
+    cont()
+  }
+}
+
 if (Settings.smokeTest) {
-  let runSmokeTest
-  ;(runSmokeTest = function() {
-    logger.log('running smoke tests')
-    smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))(
-      {},
-      resCacher
-    )
-    return setTimeout(runSmokeTest, 30 * 1000)
-  })()
+  const runSmokeTest = function() {
+    checkIfProcessIsTooOld(function() {
+      logger.log('running smoke tests')
+      smokeTest.run(
+        require.resolve(__dirname + '/test/smoke/js/SmokeTests.js')
+      )({}, resCacher)
+      return setTimeout(runSmokeTest, 30 * 1000)
+    })
+  }
+  runSmokeTest()
 }
 
 app.get('/health_check', function(req, res) {
-  res.contentType(resCacher != null ? resCacher.setContentType : undefined)
-  return res
-    .status(resCacher != null ? resCacher.code : undefined)
-    .send(resCacher != null ? resCacher.body : undefined)
+  res.contentType(resCacher.setContentType)
+  return res.status(resCacher.code).send(resCacher.body)
 })
 
 app.get('/smoke_test_force', (req, res) =>
@@ -234,23 +251,6 @@ app.get('/smoke_test_force', (req, res) =>
   )
 )
 
-const profiler = require('v8-profiler-node8')
-app.get('/profile', function(req, res) {
-  const time = parseInt(req.query.time || '1000')
-  profiler.startProfiling('test')
-  return setTimeout(function() {
-    const profile = profiler.stopProfiling('test')
-    return res.json(profile)
-  }, time)
-})
-
-app.get('/heapdump', (req, res) =>
-  require('heapdump').writeSnapshot(
-    `/tmp/${Date.now()}.clsi.heapsnapshot`,
-    (err, filename) => res.send(filename)
-  )
-)
-
 app.use(function(error, req, res, next) {
   if (error instanceof Errors.NotFoundError) {
     logger.warn({ err: error, url: req.url }, 'not found error')

services/clsi/config/settings.defaults.js

@@ -22,6 +22,9 @@ module.exports = {
 
   compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
 
+  processLifespanLimitMs:
+    parseInt(process.env.PROCESS_LIFE_SPAN_LIMIT_MS) || 60 * 60 * 24 * 1000 * 2,
+
   path: {
     compilesDir: Path.resolve(__dirname + '/../compiles'),
     clsiCacheDir: Path.resolve(__dirname + '/../cache'),
@@ -65,8 +68,7 @@ if (process.env.DOCKER_RUNNER) {
     dockerRunner: process.env.DOCKER_RUNNER === 'true',
     docker: {
       image:
-        process.env.TEXLIVE_IMAGE ||
-        'quay.io/sharelatex/texlive-full:2017.1',
+        process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1',
       env: {
         HOME: '/tmp'
       },
@@ -93,8 +95,7 @@ if (process.env.DOCKER_RUNNER) {
 
   module.exports.path.synctexBaseDir = () => '/compile'
 
-  module.exports.path.sandboxedCompilesHostDir =
-    process.env.COMPILES_HOST_DIR
+  module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR
 
   module.exports.path.synctexBinHostPath = process.env.SYNCTEX_BIN_HOST_PATH
 }