Merge pull request #11733 from overleaf/msm-close-site-sp-shutdwn

[CE/SP] close site on shutdown before flushing

GitOrigin-RevId: e36c80f2f4c6f10eda3a536319a2fcc5dfda4fa4

server-ce/Dockerfile

@@ -70,6 +70,11 @@ COPY server-ce/config/production.json /overleaf/services/history-v1/config/produ
 ADD server-ce/bin/grunt /usr/local/bin/grunt
 RUN chmod +x /usr/local/bin/grunt
 
+# File that controls open|closed status of the site
+# -------------------------------------------------
+ENV SITE_MAINTENANCE_FILE "/etc/sharelatex/site_status"
+RUN touch $SITE_MAINTENANCE_FILE
+
 # Set Environment Variables
 # --------------------------------
 ENV SHARELATEX_CONFIG /etc/sharelatex/settings.js

server-ce/init_preshutdown_scripts/00_close_site

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown
+
+mv "${SITE_MAINTENANCE_FILE}" "${SITE_MAINTENANCE_FILE_BAK}"
+echo "closed" > "${SITE_MAINTENANCE_FILE}"
+
+# status file is polled every 5 seconds
+sleep 5
+
+# giving a grace period of 5 seconds for users before disconnecting them and start shutting down
+cd /overleaf/services/web && node scripts/disconnect_all_users.js 5 >> /var/log/sharelatex/web.log 2>&1
+
+EXIT_CODE="$?"
+if [ $EXIT_CODE -ne 0 ]
+then
+  echo "scripts/disconnect_all_users.js failed with exit code $EXIT_CODE"
+fi
+
+# wait for disconnection
+sleep 5
+
+exit 0

server-ce/init_scripts/00_restore_site_status

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# pre-shutdown scripts close the site by overriding the content of SITE_MAINTENANCE_FILE,
+# this script restores the original value on container restart
+SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown"
+
+if [ -f "${SITE_MAINTENANCE_FILE_BAK}" ]; then
+    mv -f "${SITE_MAINTENANCE_FILE_BAK}" "${SITE_MAINTENANCE_FILE}"
+    rm -f "${SITE_MAINTENANCE_FILE_BAK}"
+fi

services/web/app.js

@@ -14,6 +14,7 @@ metrics.initialize(process.env.METRICS_APP_NAME || 'web')
 const Settings = require('@overleaf/settings')
 const logger = require('@overleaf/logger')
 const PlansLocator = require('./app/src/Features/Subscription/PlansLocator')
+const SiteAdminHandler = require('./app/src/infrastructure/SiteAdminHandler')
 logger.initialize(process.env.METRICS_APP_NAME || 'web')
 logger.logger.serializers.user =
   require('./app/src/infrastructure/LoggerSerializers').user
@@ -78,6 +79,9 @@ if (!module.parent) {
     })
 }
 
+// monitor site maintenance file
+SiteAdminHandler.initialise()
+
 // handle SIGTERM for graceful shutdown in kubernetes
 process.on('SIGTERM', function (signal) {
   triggerGracefulShutdown(Server.server, signal)

services/web/app/src/infrastructure/SiteAdminHandler.js

@@ -0,0 +1,65 @@
+const logger = require('@overleaf/logger')
+const settings = require('@overleaf/settings')
+const fs = require('fs')
+const {
+  addOptionalCleanupHandlerAfterDrainingConnections,
+} = require('./GracefulShutdown')
+
+// Monitor a site maintenance file (e.g. /etc/site_status) periodically and
+// close the site if the file contents contain the string "closed".
+
+const STATUS_FILE_CHECK_INTERVAL = 5000
+const statusFile = settings.siteMaintenanceFile
+
+function updateSiteMaintenanceStatus(fileContent) {
+  const isClosed = !settings.siteIsOpen
+  const shouldBeClosed = fileContent && fileContent.indexOf('closed') >= 0
+  if (!isClosed && shouldBeClosed) {
+    settings.siteIsOpen = false
+    logger.warn({ fileContent }, 'putting site into maintenance mode')
+  } else if (isClosed && !shouldBeClosed) {
+    settings.siteIsOpen = true
+    logger.warn({ fileContent }, 'taking site out of maintenance mode')
+  }
+}
+
+function pollSiteMaintenanceFile() {
+  fs.readFile(statusFile, { encoding: 'utf8' }, (err, fileContent) => {
+    if (err) {
+      logger.error(
+        { file: statusFile, fsErr: err },
+        'error reading site maintenance file'
+      )
+      return
+    }
+    updateSiteMaintenanceStatus(fileContent)
+  })
+}
+
+function checkSiteMaintenanceFileSync() {
+  // crash on start up if file does not exist
+  const content = fs.readFileSync(statusFile, { encoding: 'utf8' })
+  updateSiteMaintenanceStatus(content)
+}
+
+module.exports = {
+  initialise() {
+    if (settings.enabledServices.includes('web') && statusFile) {
+      logger.debug(
+        { statusFile, interval: STATUS_FILE_CHECK_INTERVAL },
+        'monitoring site maintenance file'
+      )
+      checkSiteMaintenanceFileSync() // perform an initial synchronous check at start up
+      const intervalHandle = setInterval(
+        pollSiteMaintenanceFile,
+        STATUS_FILE_CHECK_INTERVAL
+      ) // continue checking periodically
+      addOptionalCleanupHandlerAfterDrainingConnections(
+        'poll site maintenance file',
+        () => {
+          clearInterval(intervalHandle)
+        }
+      )
+    }
+  },
+}