[clsi] run SyncTeX in specific output dir rather than compile dir (#24690)

* [clsi] drop support for docker-in-docker

* [clsi] run SyncTeX in specific output dir rather than compile dir

* [clsi] store output.synctex.gz outside of tar-ball in clsi-cache

* [clsi] add documentation for rewriting of docker bind-mounts

* [server-pro] update env vars for sandboxed compiles in sample config

GitOrigin-RevId: 8debd7102ac612544961f237aa4ff1c530aa3da3

develop/docker-compose.yml

@@ -29,6 +29,7 @@ services:
       - DOCKER_RUNNER=true
       - TEXLIVE_IMAGE=texlive-full # docker build texlive -t texlive-full
       - COMPILES_HOST_DIR=${PWD}/compiles
+      - OUTPUT_HOST_DIR=${PWD}/output
     user: root
     volumes:
       - ${PWD}/compiles:/overleaf/services/clsi/compiles

docker-compose.yml

@@ -77,7 +77,9 @@ services:
             SANDBOXED_COMPILES: 'true'
             SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true'
             ### Bind-mount source for /var/lib/overleaf/data/compiles inside the container.
-            SANDBOXED_COMPILES_HOST_DIR: '/home/user/sharelatex_data/data/compiles'
+            SANDBOXED_COMPILES_HOST_DIR_COMPILES: '/home/user/sharelatex_data/data/compiles'
+            ### Bind-mount source for /var/lib/overleaf/data/output inside the container.
+            SANDBOXED_COMPILES_HOST_DIR_OUTPUT: '/home/user/sharelatex_data/data/output'
 
             ## Works with test LDAP server shown at bottom of docker compose
             # OVERLEAF_LDAP_URL: 'ldap://ldap:389'

services/clsi/README.md

@@ -20,6 +20,7 @@ The CLSI can be configured through the following environment variables:
 * `CATCH_ERRORS` - Set to `true` to log uncaught exceptions
 * `COMPILE_GROUP_DOCKER_CONFIGS` - JSON string of Docker configs for compile groups
 * `COMPILES_HOST_DIR` - Working directory for LaTeX compiles
+* `OUTPUT_HOST_DIR` - Output directory for LaTeX compiles
 * `COMPILE_SIZE_LIMIT` - Sets the body-parser [limit](https://github.com/expressjs/body-parser#limit)
 * `DOCKER_RUNNER` - Set to true to use sibling containers
 * `DOCKER_RUNTIME` -

services/clsi/app/js/CLSICacheHandler.js

@@ -1,3 +1,4 @@
+const crypto = require('node:crypto')
 const fs = require('node:fs')
 const Path = require('node:path')
 const { pipeline } = require('node:stream/promises')
@@ -71,6 +72,7 @@ function notifyCLSICacheAboutBuild({
         f =>
           f.path === 'output.pdf' ||
           f.path === 'output.log' ||
+          f.path === 'output.synctex.gz' ||
           f.path.endsWith('.blg')
       )
       .map(f => {
@@ -110,9 +112,7 @@ async function buildTarball({ projectId, userId, buildId, outputFiles }) {
     buildId
   )
 
-  const files = outputFiles.filter(
-    f => f.path === 'output.synctex.gz' || !isExtraneousFile(f.path)
-  )
+  const files = outputFiles.filter(f => !isExtraneousFile(f.path))
   if (files.length > MAX_ENTRIES_IN_OUTPUT_TAR) {
     Metrics.inc('clsi_cache_build_too_many_entries')
     throw new Error('too many output files for output.tar.gz')
@@ -136,59 +136,80 @@ async function buildTarball({ projectId, userId, buildId, outputFiles }) {
   }
 }
 
-/**
- * @param {string} projectId
- * @param {string} userId
- * @param {string} compileDir
- * @return {Promise<boolean>}
- */
-async function downloadLatestCompileCache(projectId, userId, compileDir) {
-  return await _downloadCompileCache(
-    `${Settings.apis.clsiCache.url}/project/${projectId}/${
-      userId ? `user/${userId}/` : ''
-    }latest/output/output.tar.gz`,
-    compileDir,
-    'tar'
-  )
-}
-
 /**
  * @param {string} projectId
  * @param {string} userId
  * @param {string} editorId
  * @param {string} buildId
- * @param {string} compileDir
+ * @param {string} outputDir
  * @return {Promise<boolean>}
  */
-async function downloadOldCompileCache(
+async function downloadOutputDotSynctexFromCompileCache(
   projectId,
   userId,
   editorId,
   buildId,
-  compileDir
+  outputDir
 ) {
-  return await _downloadCompileCache(
-    `${Settings.apis.clsiCache.url}/project/${projectId}/${
-      userId ? `user/${userId}/` : ''
-    }build/${editorId}-${buildId}/search/output/output.tar.gz`,
-    compileDir,
-    'synctex'
+  if (!Settings.apis.clsiCache.enabled) return false
+
+  const timer = new Metrics.Timer(
+    'clsi_cache_download',
+    1,
+    { method: 'synctex' },
+    TIMING_BUCKETS
   )
+  let stream
+  try {
+    stream = await fetchStream(
+      `${Settings.apis.clsiCache.url}/project/${projectId}/${
+        userId ? `user/${userId}/` : ''
+      }build/${editorId}-${buildId}/search/output/output.synctex.gz`,
+      {
+        method: 'GET',
+        signal: AbortSignal.timeout(10_000),
+      }
+    )
+  } catch (err) {
+    if (err instanceof RequestFailedError && err.response.status === 404) {
+      timer.done({ status: 'not-found' })
+      return false
+    }
+    timer.done({ status: 'error' })
+    throw err
+  }
+  await fs.promises.mkdir(outputDir, { recursive: true })
+  const dst = Path.join(outputDir, 'output.synctex.gz')
+  const tmp = dst + crypto.randomUUID()
+  try {
+    await pipeline(stream, fs.createWriteStream(tmp))
+    await fs.promises.rename(tmp, dst)
+  } catch (err) {
+    try {
+      await fs.promises.unlink(tmp)
+    } catch {}
+    throw err
+  }
+  timer.done({ status: 'success' })
+  return true
 }
 
 /**
- * @param {string} url
+ * @param {string} projectId
+ * @param {string} userId
  * @param {string} compileDir
- * @param {string} method
  * @return {Promise<boolean>}
  */
-async function _downloadCompileCache(url, compileDir, method) {
+async function downloadLatestCompileCache(projectId, userId, compileDir) {
   if (!Settings.apis.clsiCache.enabled) return false
 
+  const url = `${Settings.apis.clsiCache.url}/project/${projectId}/${
+    userId ? `user/${userId}/` : ''
+  }latest/output/output.tar.gz`
   const timer = new Metrics.Timer(
     'clsi_cache_download',
     1,
-    { status: 'success', method },
+    { method: 'tar' },
     TIMING_BUCKETS
   )
   let stream
@@ -199,12 +220,10 @@ async function _downloadCompileCache(url, compileDir, method) {
     })
   } catch (err) {
     if (err instanceof RequestFailedError && err.response.status === 404) {
-      timer.labels.status = 'not-found'
-      timer.done()
+      timer.done({ status: 'not-found' })
       return false
     }
-    timer.labels.status = 'error'
-    timer.done()
+    timer.done({ status: 'error' })
     throw err
   }
   let n = 0
@@ -224,7 +243,6 @@ async function _downloadCompileCache(url, compileDir, method) {
             {
               url,
               compileDir,
-              method,
             },
             'too many entries in tar-ball from clsi-cache'
           )
@@ -234,7 +252,6 @@ async function _downloadCompileCache(url, compileDir, method) {
             {
               url,
               compileDir,
-              method,
               entryType: header.type,
             },
             'unexpected entry in tar-ball from clsi-cache'
@@ -245,12 +262,12 @@ async function _downloadCompileCache(url, compileDir, method) {
     })
   )
   Metrics.count('clsi_cache_download_entries', n)
-  timer.done()
+  timer.done({ status: 'success' })
   return !abort
 }
 
 module.exports = {
   notifyCLSICacheAboutBuild,
   downloadLatestCompileCache,
-  downloadOldCompileCache,
+  downloadOutputDotSynctexFromCompileCache,
 }

services/clsi/app/js/CompileManager.js

@@ -21,7 +21,7 @@ const { emitPdfStats } = require('./ContentCacheMetrics')
 const SynctexOutputParser = require('./SynctexOutputParser')
 const {
   downloadLatestCompileCache,
-  downloadOldCompileCache,
+  downloadOutputDotSynctexFromCompileCache,
 } = require('./CLSICacheHandler')
 
 const COMPILE_TIME_BUCKETS = [
@@ -509,58 +509,72 @@ async function _runSynctex(projectId, userId, command, opts) {
     throw new Errors.InvalidParameter('invalid buildId')
   }
 
-  const directory = getCompileDir(projectId, userId)
+  const outputDir = getOutputDir(projectId, userId)
+  const runInOutputDir = buildId && CommandRunner.canRunSyncTeXInOutputDir()
+
+  const directory = runInOutputDir
+    ? Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR, buildId)
+    : getCompileDir(projectId, userId)
   const timeout = 60 * 1000 // increased to allow for large projects
   const compileName = getCompileName(projectId, userId)
-  const compileGroup = 'synctex'
+  const compileGroup = runInOutputDir ? 'synctex-output' : 'synctex'
   const defaultImageName =
     Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.image
-  try {
-    await _checkFileExists(directory, 'output.synctex.gz')
-  } catch (err) {
-    if (
-      err instanceof Errors.NotFoundError &&
-      compileFromClsiCache &&
-      editorId &&
-      buildId
-    ) {
+  // eslint-disable-next-line @typescript-eslint/return-await
+  return await OutputCacheManager.promises.queueDirOperation(
+    outputDir,
+    /**
+     * @return {Promise<string>}
+     */
+    async () => {
       try {
-        await downloadOldCompileCache(
-          projectId,
-          userId,
-          editorId,
-          buildId,
-          directory
-        )
+        await _checkFileExists(directory, 'output.synctex.gz')
       } catch (err) {
-        logger.warn(
-          { err, projectId, userId, editorId, buildId },
-          'failed to populate compile dir for synctex using old output'
+        if (
+          err instanceof Errors.NotFoundError &&
+          compileFromClsiCache &&
+          editorId &&
+          buildId
+        ) {
+          try {
+            await downloadOutputDotSynctexFromCompileCache(
+              projectId,
+              userId,
+              editorId,
+              buildId,
+              directory
+            )
+          } catch (err) {
+            logger.warn(
+              { err, projectId, userId, editorId, buildId },
+              'failed to download output.synctex.gz from clsi-cache'
+            )
+          }
+          await _checkFileExists(directory, 'output.synctex.gz')
+        } else {
+          throw err
+        }
+      }
+      try {
+        const output = await CommandRunner.promises.run(
+          compileName,
+          command,
+          directory,
+          imageName || defaultImageName,
+          timeout,
+          {},
+          compileGroup
         )
+        return output.stdout
+      } catch (error) {
+        throw OError.tag(error, 'error running synctex', {
+          command,
+          projectId,
+          userId,
+        })
       }
-      await _checkFileExists(directory, 'output.synctex.gz')
-    } else {
-      throw err
     }
-  }
-  try {
-    const output = await CommandRunner.promises.run(
-      compileName,
-      command,
-      directory,
-      imageName || defaultImageName,
-      timeout,
-      {},
-      compileGroup
-    )
-    return output.stdout
-  } catch (error) {
-    throw OError.tag(error, 'error running synctex', {
-      command,
-      projectId,
-      userId,
-    })
-  }
+  )
 }
 
 async function wordcount(projectId, userId, filename, image) {

services/clsi/app/js/DockerRunner.js

@@ -6,21 +6,12 @@ const dockerode = new Docker()
 const crypto = require('node:crypto')
 const async = require('async')
 const LockManager = require('./DockerLockManager')
-const fs = require('node:fs')
 const Path = require('node:path')
 const _ = require('lodash')
 
 const ONE_HOUR_IN_MS = 60 * 60 * 1000
 logger.debug('using docker runner')
 
-function usingSiblingContainers() {
-  return (
-    Settings != null &&
-    Settings.path != null &&
-    Settings.path.sandboxedCompilesHostDir != null
-  )
-}
-
 let containerMonitorTimeout
 let containerMonitorInterval
 
@@ -35,24 +26,6 @@ const DockerRunner = {
     compileGroup,
     callback
   ) {
-    if (usingSiblingContainers()) {
-      const _newPath = Settings.path.sandboxedCompilesHostDir
-      logger.debug(
-        { path: _newPath },
-        'altering bind path for sibling containers'
-      )
-      // Server Pro, example:
-      //   '/var/lib/overleaf/data/compiles/<project-id>'
-      //   ... becomes ...
-      //   '/opt/overleaf_data/data/compiles/<project-id>'
-      directory = Path.join(
-        Settings.path.sandboxedCompilesHostDir,
-        Path.basename(directory)
-      )
-    }
-
-    const volumes = { [directory]: '/compile' }
-
     command = command.map(arg =>
       arg.toString().replace('$COMPILE_DIR', '/compile')
     )
@@ -72,7 +45,32 @@ const DockerRunner = {
       image = `${Settings.texliveImageNameOveride}/${img[2]}`
     }
 
-    if (compileGroup === 'synctex' || compileGroup === 'wordcount') {
+    if (compileGroup === 'synctex-output') {
+      // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId'
+      //             directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId'
+      //  sandboxedCompilesHostDirOutput = '/host/output'
+      // Out:                  directory = '/host/output/projectId-userId/generated-files/buildId'
+      directory = Path.join(
+        Settings.path.sandboxedCompilesHostDirOutput,
+        ...directory.split('/').slice(-3)
+      )
+    } else {
+      // In:   directory = '/overleaf/services/clsi/compiles/projectId-userId'
+      //                       Path.basename(directory) === 'projectId-userId'
+      //  sandboxedCompilesHostDirCompiles = '/host/compiles'
+      // Out:                    directory = '/host/compiles/projectId-userId'
+      directory = Path.join(
+        Settings.path.sandboxedCompilesHostDirCompiles,
+        Path.basename(directory)
+      )
+    }
+
+    const volumes = { [directory]: '/compile' }
+    if (
+      compileGroup === 'synctex' ||
+      compileGroup === 'synctex-output' ||
+      compileGroup === 'wordcount'
+    ) {
       volumes[directory] += ':ro'
     }
 
@@ -309,50 +307,17 @@ const DockerRunner = {
     LockManager.runWithLock(
       options.name,
       releaseLock =>
-        // Check that volumes exist before starting the container.
-        // When a container is started with volume pointing to a
-        // non-existent directory then docker creates the directory but
-        // with root ownership.
-        DockerRunner._checkVolumes(options, volumes, err => {
-          if (err != null) {
-            return releaseLock(err)
-          }
-          DockerRunner._startContainer(
-            options,
-            volumes,
-            attachStreamHandler,
-            releaseLock
-          )
-        }),
-
+        DockerRunner._startContainer(
+          options,
+          volumes,
+          attachStreamHandler,
+          releaseLock
+        ),
       callback
     )
   },
 
   // Check that volumes exist and are directories
-  _checkVolumes(options, volumes, callback) {
-    if (usingSiblingContainers()) {
-      // Server Pro, with sibling-containers active, skip checks
-      return callback(null)
-    }
-
-    const checkVolume = (path, cb) =>
-      fs.stat(path, (err, stats) => {
-        if (err != null) {
-          return cb(err)
-        }
-        if (!stats.isDirectory()) {
-          return cb(new Error('not a directory'))
-        }
-        cb()
-      })
-    const jobs = []
-    for (const vol in volumes) {
-      jobs.push(cb => checkVolume(vol, cb))
-    }
-    async.series(jobs, callback)
-  },
-
   _startContainer(options, volumes, attachStreamHandler, callback) {
     callback = _.once(callback)
     const { name } = options
@@ -617,6 +582,10 @@ const DockerRunner = {
       containerMonitorInterval = undefined
     }
   },
+
+  canRunSyncTeXInOutputDir() {
+    return Boolean(Settings.path.sandboxedCompilesHostDirOutput)
+  },
 }
 
 DockerRunner.startContainerMonitor()

services/clsi/app/js/LocalCommandRunner.js

@@ -99,6 +99,10 @@ module.exports = CommandRunner = {
     }
     return callback()
   },
+
+  canRunSyncTeXInOutputDir() {
+    return true
+  },
 }
 
 module.exports.promises = {

services/clsi/app/js/OutputCacheManager.js

@@ -83,6 +83,13 @@ async function cleanupDirectory(dir, options) {
   })
 }
 
+/**
+ * @template T
+ *
+ * @param {string} dir
+ * @param {() => Promise<T>} fn
+ * @return {Promise<T>}
+ */
 async function queueDirOperation(dir, fn) {
   const pending = PENDING_PROJECT_ACTIONS.get(dir) || Promise.resolve()
   const p = pending.then(fn, fn).finally(() => {
@@ -677,4 +684,5 @@ OutputCacheManager.promises = {
   saveOutputFilesInBuildDir: promisify(
     OutputCacheManager.saveOutputFilesInBuildDir
   ),
+  queueDirOperation,
 }

services/clsi/buildscript.txt

@@ -2,7 +2,7 @@ clsi
 --data-dirs=cache,compiles,output
 --dependencies=
 --docker-repos=gcr.io/overleaf-ops,us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles
+--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles,OUTPUT_HOST_DIR=$PWD/output
 --env-pass-through=
 --esmock-loader=False
 --node-version=20.18.2

services/clsi/config/settings.defaults.js

@@ -131,6 +131,7 @@ if (process.env.DOCKER_RUNNER) {
     const defaultCompileGroupConfig = {
       wordcount: { 'HostConfig.AutoRemove': true },
       synctex: { 'HostConfig.AutoRemove': true },
+      'synctex-output': { 'HostConfig.AutoRemove': true },
     }
     module.exports.clsi.docker.compileGroupConfig = Object.assign(
       defaultCompileGroupConfig,
@@ -175,5 +176,8 @@ if (process.env.DOCKER_RUNNER) {
 
   module.exports.path.synctexBaseDir = () => '/compile'
 
-  module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR
+  module.exports.path.sandboxedCompilesHostDirCompiles =
+    process.env.COMPILES_HOST_DIR
+  module.exports.path.sandboxedCompilesHostDirOutput =
+    process.env.OUTPUT_HOST_DIR
 }

services/clsi/docker-compose.ci.yml

@@ -31,6 +31,7 @@ services:
       TEXLIVE_IMAGE_USER: "tex"
       DOCKER_RUNNER: "true"
       COMPILES_HOST_DIR: $PWD/compiles
+      OUTPUT_HOST_DIR: $PWD/output
     volumes:
       - ./compiles:/overleaf/services/clsi/compiles
       - /var/run/docker.sock:/var/run/docker.sock

services/clsi/docker-compose.yml

@@ -49,5 +49,6 @@ services:
       TEXLIVE_IMAGE_USER: "tex"
       DOCKER_RUNNER: "true"
       COMPILES_HOST_DIR: $PWD/compiles
+      OUTPUT_HOST_DIR: $PWD/output
     command: npm run --silent test:acceptance
 

services/clsi/test/unit/js/CompileControllerTests.js

@@ -33,7 +33,7 @@ describe('CompileController', function () {
         './CLSICacheHandler': {
           notifyCLSICacheAboutBuild: sinon.stub(),
           downloadLatestCompileCache: sinon.stub().resolves(),
-          downloadOldCompileCache: sinon.stub().resolves(),
+          downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(),
         },
         './Errors': (this.Erros = Errors),
       },

services/clsi/test/unit/js/CompileManagerTests.js

@@ -62,6 +62,7 @@ describe('CompileManager', function () {
     }
     this.OutputCacheManager = {
       promises: {
+        queueDirOperation: sinon.stub().callsArg(1),
         saveOutputFiles: sinon
           .stub()
           .resolves({ outputFiles: this.buildFiles, buildId: this.buildId }),
@@ -163,7 +164,7 @@ describe('CompileManager', function () {
         './CLSICacheHandler': {
           notifyCLSICacheAboutBuild: sinon.stub(),
           downloadLatestCompileCache: sinon.stub().resolves(),
-          downloadOldCompileCache: sinon.stub().resolves(),
+          downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(),
         },
       },
     })

services/clsi/test/unit/js/DockerRunnerTests.js

@@ -76,8 +76,11 @@ describe('DockerRunner', function () {
     this.env = {}
     this.callback = sinon.stub()
     this.project_id = 'project-id-123'
-    this.volumes = { '/local/compile/directory': '/compile' }
+    this.volumes = { '/some/host/dir/compiles/directory': '/compile' }
     this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
+    this.Settings.path.sandboxedCompilesHostDirCompiles =
+      '/some/host/dir/compiles'
+    this.Settings.path.sandboxedCompilesHostDirOutput = '/some/host/dir/output'
     this.compileGroup = 'compile-group'
     return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
   })
@@ -151,9 +154,8 @@ describe('DockerRunner', function () {
       })
     })
 
-    describe('when path.sandboxedCompilesHostDir is set', function () {
+    describe('standard compile', function () {
       beforeEach(function () {
-        this.Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles'
         this.directory = '/var/lib/overleaf/data/compiles/xyz'
         this.DockerRunner._runAndWaitForContainer = sinon
           .stub()
@@ -183,6 +185,99 @@ describe('DockerRunner', function () {
       })
     })
 
+    describe('synctex-output', function () {
+      beforeEach(function () {
+        this.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id'
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+        this.DockerRunner.run(
+          this.project_id,
+          this.command,
+          this.directory,
+          this.image,
+          this.timeout,
+          this.env,
+          'synctex-output',
+          this.callback
+        )
+      })
+
+      it('should re-write the bind directory and set ro flag', function () {
+        const volumes =
+          this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
+        expect(volumes).to.deep.equal({
+          '/some/host/dir/output/xyz/generated-files/id': '/compile:ro',
+        })
+      })
+
+      it('should call the callback', function () {
+        this.callback.calledWith(null, this.output).should.equal(true)
+      })
+    })
+
+    describe('synctex', function () {
+      beforeEach(function () {
+        this.directory = '/var/lib/overleaf/data/compile/xyz'
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+        this.DockerRunner.run(
+          this.project_id,
+          this.command,
+          this.directory,
+          this.image,
+          this.timeout,
+          this.env,
+          'synctex',
+          this.callback
+        )
+      })
+
+      it('should re-write the bind directory', function () {
+        const volumes =
+          this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
+        expect(volumes).to.deep.equal({
+          '/some/host/dir/compiles/xyz': '/compile:ro',
+        })
+      })
+
+      it('should call the callback', function () {
+        this.callback.calledWith(null, this.output).should.equal(true)
+      })
+    })
+
+    describe('wordcount', function () {
+      beforeEach(function () {
+        this.directory = '/var/lib/overleaf/data/compile/xyz'
+        this.DockerRunner._runAndWaitForContainer = sinon
+          .stub()
+          .callsArgWith(3, null, (this.output = 'mock-output'))
+        this.DockerRunner.run(
+          this.project_id,
+          this.command,
+          this.directory,
+          this.image,
+          this.timeout,
+          this.env,
+          'wordcount',
+          this.callback
+        )
+      })
+
+      it('should re-write the bind directory', function () {
+        const volumes =
+          this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
+        expect(volumes).to.deep.equal({
+          '/some/host/dir/compiles/xyz': '/compile:ro',
+        })
+      })
+
+      it('should call the callback', function () {
+        this.callback.calledWith(null, this.output).should.equal(true)
+      })
+    })
+
     describe('when the run throws an error', function () {
       beforeEach(function () {
         let firstTime = true
@@ -390,7 +485,7 @@ describe('DockerRunner', function () {
         const options =
           this.DockerRunner._runAndWaitForContainer.lastCall.args[0]
         return expect(options.HostConfig).to.deep.include({
-          Binds: ['/local/compile/directory:/compile:rw'],
+          Binds: ['/some/host/dir/compiles/directory:/compile:rw'],
           LogConfig: { Type: 'none', Config: {} },
           CapDrop: 'ALL',
           SecurityOpt: ['no-new-privileges'],
@@ -562,82 +657,6 @@ describe('DockerRunner', function () {
       })
     })
 
-    describe('when a volume does not exist', function () {
-      beforeEach(function () {
-        this.fs.stat = sinon.stub().yields(new Error('no such path'))
-        return this.DockerRunner.startContainer(
-          this.options,
-          this.volumes,
-          this.attachStreamHandler,
-          this.callback
-        )
-      })
-
-      it('should not try to create the container', function () {
-        return this.createContainer.called.should.equal(false)
-      })
-
-      it('should call the callback with an error', function () {
-        this.callback.calledWith(sinon.match(Error)).should.equal(true)
-      })
-    })
-
-    describe('when a volume exists but is not a directory', function () {
-      beforeEach(function () {
-        this.fs.stat = sinon.stub().yields(null, {
-          isDirectory() {
-            return false
-          },
-        })
-        return this.DockerRunner.startContainer(
-          this.options,
-          this.volumes,
-          this.attachStreamHandler,
-          this.callback
-        )
-      })
-
-      it('should not try to create the container', function () {
-        return this.createContainer.called.should.equal(false)
-      })
-
-      it('should call the callback with an error', function () {
-        this.callback.calledWith(sinon.match(Error)).should.equal(true)
-      })
-    })
-
-    describe('when a volume does not exist, but sibling-containers are used', function () {
-      beforeEach(function () {
-        this.fs.stat = sinon.stub().yields(new Error('no such path'))
-        this.Settings.path.sandboxedCompilesHostDir = '/some/path'
-        this.container.start = sinon.stub().yields()
-        return this.DockerRunner.startContainer(
-          this.options,
-          this.volumes,
-          () => {},
-          this.callback
-        )
-      })
-
-      afterEach(function () {
-        return delete this.Settings.path.sandboxedCompilesHostDir
-      })
-
-      it('should start the container with the given name', function () {
-        this.getContainer.calledWith(this.options.name).should.equal(true)
-        return this.container.start.called.should.equal(true)
-      })
-
-      it('should not try to create the container', function () {
-        return this.createContainer.called.should.equal(false)
-      })
-
-      return it('should call the callback', function () {
-        this.callback.called.should.equal(true)
-        return this.callback.calledWith(new Error()).should.equal(false)
-      })
-    })
-
     return describe('when the container tries to be created, but already has been (race condition)', function () {})
   })
 

services/web/app/src/Features/Compile/ClsiCacheHandler.js

@@ -15,6 +15,7 @@ function validateFilename(filename) {
       'output.blg',
       'output.log',
       'output.pdf',
+      'output.synctex.gz',
       'output.overleaf.json',
       'output.tar.gz',
     ].includes(filename) ||