Merge pull request #19220 from overleaf/jpa-precompile-pug

[web] precompile pug templates in CI

GitOrigin-RevId: 6ec2b85a357fa3d5c35d8e7eb1a2e81ac5f3b447

server-ce/genScript.js

@@ -23,6 +23,8 @@ switch (process.argv.pop()) {
           console.log('rm -rf node_modules/.cache')
           // uninstall webpack and frontend dependencies
           console.log('npm install --omit=dev')
+          // precompile pug
+          console.log('npm run precompile-pug')
           break
         default:
           console.log(`echo ${service.name} does not require a compilation`)

services/web/.gitignore

@@ -72,6 +72,9 @@ config/*.js
 
 modules/**/Makefile
 
+# Precompiled pug files
+**/app/views/**/*.js
+
 # Sentry secrets file (injected by CI)
 .sentryclirc
 

services/web/Dockerfile

@@ -20,6 +20,7 @@ RUN mkdir -p /overleaf/services/web/data/dumpFolder \
     &&  chmod -R 0755 /overleaf/services/web/data \
     &&  chown -R node:node /overleaf/services/web/data
 
+
 # the deps image is used for caching npm ci
 FROM base as deps-prod
 
@@ -37,13 +38,6 @@ ENV CYPRESS_INSTALL_BINARY=0
 COPY tsconfig.backend.json /overleaf/
 RUN cd /overleaf && npm install
 
-# the web image with only production dependencies but no webpack production build, for development
-FROM deps-prod as app-only
-
-COPY services/web /overleaf/services/web
-USER node
-
-CMD ["node", "--expose-gc", "app.js"]
 
 # the dev is suitable for running tests
 FROM deps as dev
@@ -60,22 +54,33 @@ USER node
 
 # the webpack image has deps+src+webpack artifacts
 FROM dev as webpack
-
 USER root
 RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
 
 
-# the final production image without webpack source maps
+# intermediate image for removing source maps ahead of copying into final production image
 FROM webpack as webpack-no-sourcemaps
 RUN find /overleaf/services/web/public -name '*.js.map' -delete
 
-FROM deps-prod as app
 
-ARG SENTRY_RELEASE
-ENV SENTRY_RELEASE=$SENTRY_RELEASE
+# copy source code and precompile pug images
+FROM deps-prod as pug
 COPY services/web /overleaf/services/web
-COPY --from=webpack-no-sourcemaps /overleaf/services/web/public /overleaf/services/web/public
+# Omit Server Pro/CE specific scripts from SaaS image
 RUN rm /overleaf/services/web/modules/server-ce-scripts -rf
+RUN OVERLEAF_CONFIG=/overleaf/services/web/config/settings.overrides.saas.js npm run precompile-pug
+
+
+# the web image with only production dependencies but no webpack production build, for development
+FROM pug as app-only
 USER node
+CMD ["node", "--expose-gc", "app.js"]
 
+
+# the final production image, with webpack production build but without source maps
+FROM pug as app
+ARG SENTRY_RELEASE
+ENV SENTRY_RELEASE=$SENTRY_RELEASE
+COPY --from=webpack-no-sourcemaps /overleaf/services/web/public /overleaf/services/web/public
+USER node
 CMD ["node", "--expose-gc", "app.js"]

services/web/Makefile

@@ -492,10 +492,20 @@ build_webpack_once:
 		--file Dockerfile \
 		../..
 
+build_pug:
+	docker build \
+		--build-arg SENTRY_RELEASE \
+		--cache-from $(IMAGE_CI)-dev \
+		--tag $(IMAGE_CI)-pug \
+		--target pug \
+		--file Dockerfile \
+		../..
+
 build:
 	docker build \
 		--build-arg SENTRY_RELEASE \
 		--cache-from $(IMAGE_CI)-webpack \
+		--cache-from $(IMAGE_CI)-pug \
 		--cache-from $(IMAGE_REPO_FINAL) \
 		--tag $(IMAGE_REPO_FINAL) \
 		--target app \

services/web/app/src/infrastructure/ExpressLocals.js

@@ -314,7 +314,7 @@ module.exports = function (webRouter, privateApiRouter, publicApiRouter) {
 
   webRouter.use(function (req, res, next) {
     if (Settings.reloadModuleViewsOnEachRequest) {
-      Modules.loadViewIncludes()
+      Modules.loadViewIncludes(req.app)
     }
     res.locals.moduleIncludes = Modules.moduleIncludes
     res.locals.moduleIncludesAvailable = Modules.moduleIncludesAvailable

services/web/app/src/infrastructure/Modules.js

@@ -1,9 +1,9 @@
 const fs = require('fs')
 const Path = require('path')
-const pug = require('pug')
 const async = require('async')
 const { promisify } = require('util')
 const Settings = require('@overleaf/settings')
+const Views = require('./Views')
 
 const MODULE_BASE_PATH = Path.join(__dirname, '/../../../modules')
 
@@ -35,6 +35,11 @@ function loadModules() {
     )
     loadedModule.name = moduleName
     _modules.push(loadedModule)
+    if (loadedModule.viewIncludes) {
+      throw new Error(
+        `${moduleName}: module.viewIncludes moved into Settings.viewIncludes`
+      )
+    }
   }
   _modulesLoaded = true
   attachHooks()
@@ -64,28 +69,7 @@ function applyNonCsrfRouter(webRouter, privateApiRouter, publicApiRouter) {
 }
 
 function loadViewIncludes(app) {
-  _viewIncludes = {}
-  for (const module of modules()) {
-    const object = module.viewIncludes || {}
-    for (const view in object) {
-      const partial = object[view]
-      if (!_viewIncludes[view]) {
-        _viewIncludes[view] = []
-      }
-      const filePath = Path.join(
-        MODULE_BASE_PATH,
-        module.name,
-        'app/views',
-        partial + '.pug'
-      )
-      _viewIncludes[view].push(
-        pug.compileFile(filePath, {
-          doctype: 'html',
-          compileDebug: Settings.debugPugTemplates,
-        })
-      )
-    }
-  }
+  _viewIncludes = Views.compileViewIncludes(app)
 }
 
 function registerMiddleware(appOrRouter, middlewareName, options) {

services/web/app/src/infrastructure/Server.js

@@ -129,7 +129,18 @@ webRouter.use(
 )
 app.set('views', Path.join(__dirname, '/../../views'))
 app.set('view engine', 'pug')
-Modules.loadViewIncludes(app)
+
+if (Settings.enabledServices.includes('web')) {
+  if (app.get('env') !== 'development') {
+    logger.debug('enabling view cache for production or acceptance tests')
+    app.enable('view cache')
+  }
+  if (Settings.precompilePugTemplatesAtBootTime) {
+    logger.debug('precompiling views for web in production environment')
+    Views.precompileViews(app)
+  }
+  Modules.loadViewIncludes(app)
+}
 
 app.use(metrics.http.monitor(logger))
 
@@ -334,16 +345,6 @@ if (Settings.enabledServices.includes('api')) {
 
 if (Settings.enabledServices.includes('web')) {
   logger.debug('providing web router')
-
-  if (Settings.precompilePugTemplatesAtBootTime) {
-    logger.debug('precompiling views for web in production environment')
-    Views.precompileViews(app)
-  }
-  if (app.get('env') === 'test') {
-    logger.debug('enabling view cache for acceptance tests')
-    app.enable('view cache')
-  }
-
   app.use(publicApiRouter) // public API goes with web router for public access
   app.use(Validation.errorMiddleware)
   app.use(ErrorController.handleApiError)

services/web/app/src/infrastructure/Views.js

@@ -2,52 +2,129 @@ const logger = require('@overleaf/logger')
 const pug = require('pug')
 const globby = require('globby')
 const Settings = require('@overleaf/settings')
-const path = require('path')
+const fs = require('fs')
+const Path = require('path')
 
 // Generate list of view names from app/views
-
-const viewList = globby
-  .sync('app/views/**/*.pug', {
-    onlyFiles: true,
-    concurrency: 1,
-    ignore: '**/_*.pug',
-  })
-  .concat(
-    globby.sync('modules/*/app/views/**/*.pug', {
+function buildViewList() {
+  return globby
+    .sync('app/views/**/*.pug', {
       onlyFiles: true,
       concurrency: 1,
-      ignore: '**/_*.pug',
+      ignore: [
+        // Ignore includes
+        '**/_*.pug',
+        '**/_*/**',
+        // Ignore shared layout files
+        'app/views/layout*',
+        'app/views/layout/*',
+      ],
     })
+    .concat(
+      globby.sync('modules/*/app/views/**/*.pug', {
+        onlyFiles: true,
+        concurrency: 1,
+        // Ignore includes
+        ignore: ['**/_*.pug', '**/_*/**'],
+      })
+    )
+    .concat(Object.values(Settings.viewIncludes).flat())
+    .map(x => Path.resolve(x))
+}
+
+const PUG_COMPILE_ARGUMENTS = {
+  doctype: 'html',
+  cache: true,
+  compileDebug: Settings.debugPugTemplates,
+  inlineRuntimeFunctions: false,
+  module: true,
+}
+
+function precompileViewsAndCacheToDisk() {
+  const startTime = Date.now()
+  let success = 0
+  let precompiled = 0
+  for (const filename of buildViewList()) {
+    const precompiledFilename = filename.replace(/\.pug$/, '.js')
+    try {
+      const src = pug.compileFileClient(filename, PUG_COMPILE_ARGUMENTS)
+      try {
+        if (fs.readFileSync(precompiledFilename, 'utf-8') === src) {
+          precompiled++
+          continue
+        }
+      } catch {}
+      fs.writeFileSync(precompiledFilename, src, {
+        encoding: 'utf-8',
+        mode: 0o644,
+      })
+      success++
+    } catch (err) {
+      logger.err({ err, filename }, 'failed to precompile pug template')
+      throw err
+    }
+  }
+  logger.info(
+    { timeTaken: Date.now() - startTime, success, precompiled },
+    'compiled pug templates'
   )
-  .map(x => {
-    return x.replace(/\.pug$/, '') // strip trailing .pug extension
-  })
-  .filter(x => {
-    return !/^_/.test(x)
-  })
+}
 
 module.exports = {
+  compileViewIncludes(app) {
+    const viewIncludes = {}
+    for (const [view, paths] of Object.entries(Settings.viewIncludes)) {
+      viewIncludes[view] = []
+      for (const filePath of paths) {
+        viewIncludes[view].push(
+          pug.compileFile(filePath, {
+            ...PUG_COMPILE_ARGUMENTS,
+            cache: app.enabled('view cache'),
+          })
+        )
+      }
+    }
+    return viewIncludes
+  },
+
   precompileViews(app) {
     const startTime = Date.now()
     let success = 0
+    let precompiled = 0
     let failures = 0
-    viewList.forEach(view => {
-      const filename = path.resolve(view + '.pug') // express views are cached using the absolute path
+    for (const filename of buildViewList()) {
+      const precompiledFilename = filename.replace(/\.pug$/, '.js')
+      if (fs.existsSync(precompiledFilename)) {
+        logger.debug({ filename }, 'loading precompiled pug template')
+        try {
+          pug.cache[filename] = require(precompiledFilename)
+          precompiled++
+          continue
+        } catch (err) {
+          logger.error(
+            { filename, err },
+            'error loading precompiled pug template'
+          )
+          failures++
+        }
+      }
       try {
-        pug.compileFile(filename, {
-          cache: true,
-          compileDebug: Settings.debugPugTemplates,
-        })
-        logger.debug({ filename }, 'compiled')
+        logger.warn({ filename }, 'compiling pug template at boot time')
+        pug.compileFile(filename, PUG_COMPILE_ARGUMENTS)
         success++
       } catch (err) {
-        logger.error({ filename, err: err.message }, 'error compiling')
+        logger.error({ filename, err }, 'error compiling pug template')
         failures++
       }
-    })
+    }
     logger.debug(
-      { timeTaken: Date.now() - startTime, failures, success },
-      'compiled templates'
+      { timeTaken: Date.now() - startTime, failures, success, precompiled },
+      'compiled pug templates'
     )
   },
 }
+
+if (require.main === module) {
+  precompileViewsAndCacheToDisk()
+  process.exit(0)
+}

services/web/config/settings.defaults.js

@@ -900,6 +900,7 @@ module.exports = {
     'server-ce-scripts',
     'user-activate',
   ],
+  viewIncludes: {},
 
   csp: {
     enabled: process.env.CSP_ENABLED === 'true',

services/web/package.json

@@ -43,6 +43,7 @@
     "routes": "bin/routes",
     "storybook": "storybook dev -p 6006",
     "build-storybook": "storybook build",
+    "precompile-pug": "node app/src/infrastructure/Views",
     "local:nodemon": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; echo $OVERLEAF_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon",
     "local:webpack": "set -a;. ../../config/dev-environment.env;. ./docker-compose.common.env;. ../../config/local-dev.env;. ./local-dev.env;. ../../config/local.env; set +a; PORT=13808 OVERLEAF_CONFIG=$(pwd)/config/settings.webpack.js npm run webpack",
     "local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",