limit clsi lifespan via health checks and PROCESS_LIFE_SPAN_LIMIT_MS

services/clsi/app.js

@@ -22,7 +22,6 @@ const ContentTypeMapper = require('./app/js/ContentTypeMapper')
 const Errors = require('./app/js/Errors')
 
 const Path = require('path')
-const fs = require('fs')
 
 Metrics.open_sockets.monitor(logger)
 Metrics.memory.monitor(logger)
@@ -208,23 +207,35 @@ const resCacher = {
   setContentType: 'application/json'
 }
 
+const startupTime = Date.now()
+const checkIfProcessIsTooOld = function() {
+  if (
+    Settings.processLifespanLimitMs &&
+    startupTime + Settings.processLifespanLimitMs < Date.now()
+  ) {
+    logger.log('shutting down, process is too old')
+    resCacher.send = function() {}
+    resCacher.statusCode = 500
+    resCacher.body = { processToOld: true }
+  }
+}
+
 if (Settings.smokeTest) {
-  let runSmokeTest
-  ;(runSmokeTest = function() {
+  const runSmokeTest = function() {
+    checkIfProcessIsTooOld()
     logger.log('running smoke tests')
     smokeTest.run(require.resolve(__dirname + '/test/smoke/js/SmokeTests.js'))(
       {},
       resCacher
     )
     return setTimeout(runSmokeTest, 30 * 1000)
-  })()
+  }
+  runSmokeTest()
 }
 
 app.get('/health_check', function(req, res) {
-  res.contentType(resCacher != null ? resCacher.setContentType : undefined)
-  return res
-    .status(resCacher != null ? resCacher.code : undefined)
-    .send(resCacher != null ? resCacher.body : undefined)
+  res.contentType(resCacher.setContentType)
+  return res.status(resCacher.code).send(resCacher.body)
 })
 
 app.get('/smoke_test_force', (req, res) =>
@@ -234,23 +245,6 @@ app.get('/smoke_test_force', (req, res) =>
   )
 )
 
-const profiler = require('v8-profiler-node8')
-app.get('/profile', function(req, res) {
-  const time = parseInt(req.query.time || '1000')
-  profiler.startProfiling('test')
-  return setTimeout(function() {
-    const profile = profiler.stopProfiling('test')
-    return res.json(profile)
-  }, time)
-})
-
-app.get('/heapdump', (req, res) =>
-  require('heapdump').writeSnapshot(
-    `/tmp/${Date.now()}.clsi.heapsnapshot`,
-    (err, filename) => res.send(filename)
-  )
-)
-
 app.use(function(error, req, res, next) {
   if (error instanceof Errors.NotFoundError) {
     logger.warn({ err: error, url: req.url }, 'not found error')

services/clsi/config/settings.defaults.js

@@ -22,6 +22,9 @@ module.exports = {
 
   compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
 
+  processLifespanLimitMs:
+    process.env.PROCESS_LIFE_SPAN_LIMIT_MS || 60 * 60 * 24 * 1000 * 2,
+
   path: {
     compilesDir: Path.resolve(__dirname + '/../compiles'),
     clsiCacheDir: Path.resolve(__dirname + '/../cache'),
@@ -65,8 +68,7 @@ if (process.env.DOCKER_RUNNER) {
     dockerRunner: process.env.DOCKER_RUNNER === 'true',
     docker: {
       image:
-        process.env.TEXLIVE_IMAGE ||
-        'quay.io/sharelatex/texlive-full:2017.1',
+        process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1',
       env: {
         HOME: '/tmp'
       },
@@ -93,8 +95,7 @@ if (process.env.DOCKER_RUNNER) {
 
   module.exports.path.synctexBaseDir = () => '/compile'
 
-  module.exports.path.sandboxedCompilesHostDir =
-    process.env.COMPILES_HOST_DIR
+  module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR
 
   module.exports.path.synctexBinHostPath = process.env.SYNCTEX_BIN_HOST_PATH
 }