Merge pull request #13510 from overleaf/em-promisify-clsi-manager

Clean up of ClsiManager and ClsiCookieManager

GitOrigin-RevId: e5047b253613e87fd6cb4f12855b821028fcaf8e

services/web/app/src/Features/Compile/ClsiCookieManager.js

@@ -1,32 +1,19 @@
-/* eslint-disable
-    n/handle-callback-err,
-    max-len,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
-let rclientSecondary
 const { URL, URLSearchParams } = require('url')
 const OError = require('@overleaf/o-error')
 const Settings = require('@overleaf/settings')
 const request = require('request').defaults({ timeout: 30 * 1000 })
 const RedisWrapper = require('../../infrastructure/RedisWrapper')
-const rclient = RedisWrapper.client('clsi_cookie')
-if (Settings.redis.clsi_cookie_secondary != null) {
-  rclientSecondary = RedisWrapper.client('clsi_cookie_secondary')
-}
 const Cookie = require('cookie')
 const logger = require('@overleaf/logger')
 const Metrics = require('@overleaf/metrics')
 
-const clsiCookiesEnabled =
-  (Settings.clsiCookie != null ? Settings.clsiCookie.key : undefined) != null &&
-  Settings.clsiCookie.key.length !== 0
+const clsiCookiesEnabled = (Settings.clsiCookie?.key ?? '') !== ''
+
+const rclient = RedisWrapper.client('clsi_cookie')
+let rclientSecondary
+if (Settings.redis.clsi_cookie_secondary != null) {
+  rclientSecondary = RedisWrapper.client('clsi_cookie_secondary')
+}
 
 module.exports = function (backendGroup) {
   return {
@@ -45,15 +32,12 @@ module.exports = function (backendGroup) {
       compileBackendClass,
       callback
     ) {
-      if (callback == null) {
-        callback = function () {}
-      }
-      return rclient.get(this.buildKey(projectId, userId), (err, serverId) => {
-        if (err != null) {
+      rclient.get(this.buildKey(projectId, userId), (err, serverId) => {
+        if (err) {
           return callback(err)
         }
         if (serverId == null || serverId === '') {
-          return this._populateServerIdViaRequest(
+          this._populateServerIdViaRequest(
             projectId,
             userId,
             compileGroup,
@@ -61,7 +45,7 @@ module.exports = function (backendGroup) {
             callback
           )
         } else {
-          return callback(null, serverId)
+          callback(null, serverId)
         }
       })
     },
@@ -73,16 +57,13 @@ module.exports = function (backendGroup) {
       compileBackendClass,
       callback
     ) {
-      if (callback == null) {
-        callback = function () {}
-      }
       const u = new URL(`${Settings.apis.clsi.url}/project/${projectId}/status`)
       u.search = new URLSearchParams({
         compileGroup,
         compileBackendClass,
       }).toString()
       request.post(u.href, (err, res, body) => {
-        if (err != null) {
+        if (err) {
           OError.tag(err, 'error getting initial server id for project', {
             project_id: projectId,
           })
@@ -96,25 +77,21 @@ module.exports = function (backendGroup) {
           res,
           null,
           function (err, serverId) {
-            if (err != null) {
+            if (err) {
               logger.warn(
                 { err, projectId },
                 'error setting server id via populate request'
               )
             }
-            return callback(err, serverId)
+            callback(err, serverId)
           }
         )
       })
     },
 
     _parseServerIdFromResponse(response) {
-      const cookies = Cookie.parse(
-        (response.headers['set-cookie'] != null
-          ? response.headers['set-cookie'][0]
-          : undefined) || ''
-      )
-      return cookies != null ? cookies[Settings.clsiCookie.key] : undefined
+      const cookies = Cookie.parse(response.headers['set-cookie']?.[0] || '')
+      return cookies?.[Settings.clsiCookie.key]
     },
 
     checkIsLoadSheddingEvent(clsiserverid, compileGroup, compileBackendClass) {
@@ -155,9 +132,6 @@ module.exports = function (backendGroup) {
       previous,
       callback
     ) {
-      if (callback == null) {
-        callback = function () {}
-      }
       if (!clsiCookiesEnabled) {
         return callback()
       }
@@ -195,9 +169,6 @@ module.exports = function (backendGroup) {
     },
 
     _setServerIdInRedis(rclient, projectId, userId, serverId, callback) {
-      if (callback == null) {
-        callback = function () {}
-      }
       rclient.setex(
         this.buildKey(projectId, userId),
         this._getTTLInSeconds(serverId),
@@ -207,13 +178,10 @@ module.exports = function (backendGroup) {
     },
 
     clearServerId(projectId, userId, callback) {
-      if (callback == null) {
-        callback = function () {}
-      }
       if (!clsiCookiesEnabled) {
         return callback()
       }
-      return rclient.del(this.buildKey(projectId, userId), callback)
+      rclient.del(this.buildKey(projectId, userId), callback)
     },
 
     getCookieJar(
@@ -223,13 +191,10 @@ module.exports = function (backendGroup) {
       compileBackendClass,
       callback
     ) {
-      if (callback == null) {
-        callback = function () {}
-      }
       if (!clsiCookiesEnabled) {
         return callback(null, request.jar(), undefined)
       }
-      return this._getServerId(
+      this._getServerId(
         projectId,
         userId,
         compileGroup,
@@ -246,7 +211,7 @@ module.exports = function (backendGroup) {
           )
           const jar = request.jar()
           jar.setCookie(serverCookie, Settings.apis.clsi.url)
-          return callback(null, jar, serverId)
+          callback(null, jar, serverId)
         }
       )
     },

services/web/app/src/Features/Compile/ClsiManager.js

@@ -8,12 +8,10 @@ const { URL, URLSearchParams } = require('url')
 const OError = require('@overleaf/o-error')
 
 const ClsiCookieManager = require('./ClsiCookieManager')(
-  Settings.apis.clsi != null ? Settings.apis.clsi.backendGroupName : undefined
+  Settings.apis.clsi?.backendGroupName
 )
 const NewBackendCloudClsiCookieManager = require('./ClsiCookieManager')(
-  Settings.apis.clsi_new != null
-    ? Settings.apis.clsi_new.backendGroupName
-    : undefined
+  Settings.apis.clsi_new?.backendGroupName
 )
 const ClsiStateManager = require('./ClsiStateManager')
 const _ = require('underscore')
@@ -40,117 +38,283 @@ function collectMetricsOnBlgFiles(outputFiles) {
   Metrics.count('blg_output_file', nested, 1, { path: 'nested' })
 }
 
-const ClsiManager = {
-  sendRequest(projectId, userId, options, callback) {
-    if (options == null) {
-      options = {}
+function sendRequest(projectId, userId, options, callback) {
+  if (options == null) {
+    options = {}
+  }
+  sendRequestOnce(projectId, userId, options, (err, status, ...result) => {
+    if (err != null) {
+      return callback(err)
+    }
+    if (status === 'conflict') {
+      // Try again, with a full compile
+      return sendRequestOnce(
+        projectId,
+        userId,
+        { ...options, syncType: 'full' },
+        callback
+      )
+    } else if (status === 'unavailable') {
+      return sendRequestOnce(
+        projectId,
+        userId,
+        { ...options, syncType: 'full', forceNewClsiServer: true },
+        callback
+      )
+    }
+    callback(null, status, ...result)
+  })
+}
+
+function sendRequestOnce(projectId, userId, options, callback) {
+  if (options == null) {
+    options = {}
+  }
+  _buildRequest(projectId, options, (err, req) => {
+    if (err != null) {
+      if (err.message === 'no main file specified') {
+        return callback(null, 'validation-problems', null, null, {
+          mainFile: err.message,
+        })
+      } else {
+        return callback(
+          OError.tag(err, 'Could not build request to CLSI', {
+            projectId,
+            options,
+          })
+        )
+      }
     }
-    ClsiManager.sendRequestOnce(
+    _sendBuiltRequest(
       projectId,
       userId,
+      req,
       options,
       (err, status, ...result) => {
         if (err != null) {
-          return callback(err)
-        }
-        if (status === 'conflict') {
-          // Try again, with a full compile
-          return ClsiManager.sendRequestOnce(
-            projectId,
-            userId,
-            { ...options, syncType: 'full' },
-            callback
-          )
-        } else if (status === 'unavailable') {
-          return ClsiManager.sendRequestOnce(
-            projectId,
-            userId,
-            { ...options, syncType: 'full', forceNewClsiServer: true },
-            callback
+          return callback(
+            OError.tag(err, 'CLSI compile failed', { projectId, userId })
           )
         }
         callback(null, status, ...result)
       }
     )
-  },
+  })
+}
 
-  sendRequestOnce(projectId, userId, options, callback) {
-    if (options == null) {
-      options = {}
-    }
-    ClsiManager._buildRequest(projectId, options, (err, req) => {
+// for public API requests where there is no project id
+function sendExternalRequest(submissionId, clsiRequest, options, callback) {
+  if (options == null) {
+    options = {}
+  }
+  _sendBuiltRequest(
+    submissionId,
+    null,
+    clsiRequest,
+    options,
+    (err, status, ...result) => {
       if (err != null) {
-        if (err.message === 'no main file specified') {
-          return callback(null, 'validation-problems', null, null, {
-            mainFile: err.message,
+        return callback(
+          OError.tag(err, 'CLSI compile failed', {
+            submissionId,
+            options,
           })
-        } else {
-          return callback(
-            OError.tag(err, 'Could not build request to CLSI', {
-              projectId,
-              options,
-            })
+        )
+      }
+      callback(null, status, ...result)
+    }
+  )
+}
+
+function stopCompile(projectId, userId, options, callback) {
+  if (options == null) {
+    options = {}
+  }
+  const { compileBackendClass, compileGroup } = options
+  const compilerUrl = _getCompilerUrl(
+    compileBackendClass,
+    compileGroup,
+    projectId,
+    userId,
+    'compile/stop'
+  )
+  const opts = {
+    url: compilerUrl,
+    method: 'POST',
+  }
+  _makeRequest(
+    projectId,
+    userId,
+    compileGroup,
+    compileBackendClass,
+    opts,
+    callback
+  )
+}
+
+function deleteAuxFiles(projectId, userId, options, clsiserverid, callback) {
+  if (options == null) {
+    options = {}
+  }
+  const { compileBackendClass, compileGroup } = options
+  const compilerUrl = _getCompilerUrl(
+    compileBackendClass,
+    compileGroup,
+    projectId,
+    userId
+  )
+  const opts = {
+    url: compilerUrl,
+    method: 'DELETE',
+  }
+  _makeRequestWithClsiServerId(
+    projectId,
+    userId,
+    compileGroup,
+    compileBackendClass,
+    opts,
+    clsiserverid,
+    clsiErr => {
+      // always clear the project state from the docupdater, even if there
+      // was a problem with the request to the clsi
+      DocumentUpdaterHandler.clearProjectState(projectId, docUpdaterErr => {
+        ClsiCookieManager.clearServerId(projectId, userId, redisError => {
+          if (clsiErr) {
+            return callback(
+              OError.tag(clsiErr, 'Failed to delete aux files', { projectId })
+            )
+          }
+          if (docUpdaterErr) {
+            return callback(
+              OError.tag(
+                docUpdaterErr,
+                'Failed to clear project state in doc updater',
+                { projectId }
+              )
+            )
+          }
+          if (redisError) {
+            // redis errors need wrapping as the instance may be shared
+            return callback(
+              OError(
+                'Failed to clear clsi persistence',
+                { projectId },
+                redisError
+              )
+            )
+          }
+          callback()
+        })
+      })
+    }
+  )
+}
+
+function _sendBuiltRequest(projectId, userId, req, options, callback) {
+  if (options == null) {
+    options = {}
+  }
+  if (options.forceNewClsiServer) {
+    // Clear clsi cookie, then try again
+    return ClsiCookieManager.clearServerId(projectId, userId, err => {
+      if (err) {
+        return callback(err)
+      }
+      options.forceNewClsiServer = false // backend has now been reset
+      _sendBuiltRequest(projectId, userId, req, options, callback)
+    })
+  }
+  ClsiFormatChecker.checkRecoursesForProblems(
+    req.compile != null ? req.compile.resources : undefined,
+    (err, validationProblems) => {
+      if (err != null) {
+        return callback(
+          OError.tag(
+            err,
+            'could not check resources for potential problems before sending to clsi'
           )
-        }
+        )
       }
-      ClsiManager._sendBuiltRequest(
+      if (validationProblems != null) {
+        logger.debug(
+          { projectId, validationProblems },
+          'problems with users latex before compile was attempted'
+        )
+        return callback(
+          null,
+          'validation-problems',
+          null,
+          null,
+          validationProblems
+        )
+      }
+      _postToClsi(
         projectId,
         userId,
         req,
-        options,
-        (err, status, ...result) => {
+        options.compileBackendClass,
+        options.compileGroup,
+        (err, response, clsiServerId) => {
           if (err != null) {
             return callback(
-              OError.tag(err, 'CLSI compile failed', { projectId, userId })
+              OError.tag(err, 'error sending request to clsi', {
+                projectId,
+                userId,
+              })
             )
           }
-          callback(null, status, ...result)
+          const outputFiles = _parseOutputFiles(
+            projectId,
+            response && response.compile && response.compile.outputFiles
+          )
+          collectMetricsOnBlgFiles(outputFiles)
+          const compile = (response && response.compile) || {}
+          const status = compile.status
+          const stats = compile.stats
+          const timings = compile.timings
+          const outputUrlPrefix = compile.outputUrlPrefix
+          const validationProblems = undefined
+          callback(
+            null,
+            status,
+            outputFiles,
+            clsiServerId,
+            validationProblems,
+            stats,
+            timings,
+            outputUrlPrefix
+          )
         }
       )
-    })
-  },
-
-  // for public API requests where there is no project id
-  sendExternalRequest(submissionId, clsiRequest, options, callback) {
-    if (options == null) {
-      options = {}
     }
-    ClsiManager._sendBuiltRequest(
-      submissionId,
-      null,
-      clsiRequest,
-      options,
-      (err, status, ...result) => {
-        if (err != null) {
-          return callback(
-            OError.tag(err, 'CLSI compile failed', {
-              submissionId,
-              options,
-            })
-          )
-        }
-        callback(null, status, ...result)
-      }
-    )
-  },
+  )
+}
 
-  stopCompile(projectId, userId, options, callback) {
-    if (options == null) {
-      options = {}
-    }
-    const { compileBackendClass, compileGroup } = options
-    const compilerUrl = this._getCompilerUrl(
-      compileBackendClass,
-      compileGroup,
-      projectId,
-      userId,
-      'compile/stop'
+function _makeRequestWithClsiServerId(
+  projectId,
+  userId,
+  compileGroup,
+  compileBackendClass,
+  opts,
+  clsiserverid,
+  callback
+) {
+  if (clsiserverid) {
+    // ignore cookies and newBackend, go straight to the clsi node
+    opts.qs = Object.assign(
+      { compileGroup, compileBackendClass, clsiserverid },
+      opts.qs
     )
-    const opts = {
-      url: compilerUrl,
-      method: 'POST',
-    }
-    ClsiManager._makeRequest(
+    request(opts, (err, response, body) => {
+      if (err) {
+        return callback(
+          OError.tag(err, 'error making request to CLSI', { projectId })
+        )
+      }
+      callback(null, response, body)
+    })
+  } else {
+    _makeRequest(
       projectId,
       userId,
       compileGroup,
@@ -158,883 +322,686 @@ const ClsiManager = {
       opts,
       callback
     )
-  },
+  }
+}
 
-  deleteAuxFiles(projectId, userId, options, clsiserverid, callback) {
-    if (options == null) {
-      options = {}
-    }
-    const { compileBackendClass, compileGroup } = options
-    const compilerUrl = this._getCompilerUrl(
-      compileBackendClass,
-      compileGroup,
-      projectId,
-      userId
-    )
-    const opts = {
-      url: compilerUrl,
-      method: 'DELETE',
-    }
-    ClsiManager._makeRequestWithClsiServerId(
-      projectId,
-      userId,
-      compileGroup,
-      compileBackendClass,
-      opts,
-      clsiserverid,
-      clsiErr => {
-        // always clear the project state from the docupdater, even if there
-        // was a problem with the request to the clsi
-        DocumentUpdaterHandler.clearProjectState(projectId, docUpdaterErr => {
-          ClsiCookieManager.clearServerId(projectId, userId, redisError => {
-            if (clsiErr) {
+function _makeRequest(
+  projectId,
+  userId,
+  compileGroup,
+  compileBackendClass,
+  opts,
+  callback
+) {
+  async.series(
+    {
+      currentBackend(cb) {
+        const startTime = new Date()
+        ClsiCookieManager.getCookieJar(
+          projectId,
+          userId,
+          compileGroup,
+          compileBackendClass,
+          (err, jar, clsiServerId) => {
+            if (err != null) {
               return callback(
-                OError.tag(clsiErr, 'Failed to delete aux files', { projectId })
+                OError.tag(err, 'error getting cookie jar for CLSI request', {
+                  projectId,
+                })
               )
             }
-            if (docUpdaterErr) {
-              return callback(
-                OError.tag(
-                  docUpdaterErr,
-                  'Failed to clear project state in doc updater',
-                  { projectId }
+            opts.jar = jar
+            const timer = new Metrics.Timer('compile.currentBackend')
+            request(opts, (err, response, body) => {
+              if (err != null) {
+                return callback(
+                  OError.tag(err, 'error making request to CLSI', {
+                    projectId,
+                  })
                 )
+              }
+              timer.done()
+              Metrics.inc(
+                `compile.currentBackend.response.${response.statusCode}`
               )
-            }
-            if (redisError) {
-              // redis errors need wrapping as the instance may be shared
-              return callback(
-                OError(
-                  'Failed to clear clsi persistence',
-                  { projectId },
-                  redisError
-                )
+              ClsiCookieManager.setServerId(
+                projectId,
+                userId,
+                compileGroup,
+                compileBackendClass,
+                response,
+                clsiServerId,
+                (err, newClsiServerId) => {
+                  if (err != null) {
+                    callback(
+                      OError.tag(err, 'error setting server id', {
+                        projectId,
+                      })
+                    )
+                  } else {
+                    // return as soon as the standard compile has returned
+                    callback(
+                      null,
+                      response,
+                      body,
+                      newClsiServerId || clsiServerId
+                    )
+                  }
+                  cb(err, {
+                    response,
+                    body,
+                    finishTime: new Date() - startTime,
+                  })
+                }
               )
-            }
-            callback()
-          })
-        })
-      }
-    )
-  },
-
-  _sendBuiltRequest(projectId, userId, req, options, callback) {
-    if (options == null) {
-      options = {}
-    }
-    if (options.forceNewClsiServer) {
-      // Clear clsi cookie, then try again
-      return ClsiCookieManager.clearServerId(projectId, userId, err => {
-        if (err) {
-          return callback(err)
-        }
-        options.forceNewClsiServer = false // backend has now been reset
-        return ClsiManager._sendBuiltRequest(
+            })
+          }
+        )
+      },
+      newBackend(cb) {
+        const startTime = new Date()
+        _makeNewBackendRequest(
           projectId,
           userId,
-          req,
-          options,
-          callback
+          compileGroup,
+          compileBackendClass,
+          opts,
+          (err, response, body) => {
+            if (err != null) {
+              logger.warn({ err }, 'Error making request to new CLSI backend')
+            }
+            if (response != null) {
+              Metrics.inc(`compile.newBackend.response.${response.statusCode}`)
+            }
+            cb(err, {
+              response,
+              body,
+              finishTime: new Date() - startTime,
+            })
+          }
         )
-      })
+      },
+    },
+    (err, results) => {
+      if (err != null) {
+        // This was handled higher up
+        return
+      }
+      if (results.newBackend != null && results.newBackend.response != null) {
+        const currentStatusCode = results.currentBackend.response.statusCode
+        const newStatusCode = results.newBackend.response.statusCode
+        const statusCodeSame = newStatusCode === currentStatusCode
+        const currentCompileTime = results.currentBackend.finishTime
+        const newBackendCompileTime = results.newBackend.finishTime || 0
+        const timeDifference = newBackendCompileTime - currentCompileTime
+        logger.debug(
+          {
+            statusCodeSame,
+            timeDifference,
+            currentCompileTime,
+            newBackendCompileTime,
+            projectId,
+          },
+          'both clsi requests returned'
+        )
+      }
     }
-    ClsiFormatChecker.checkRecoursesForProblems(
-      req.compile != null ? req.compile.resources : undefined,
-      (err, validationProblems) => {
+  )
+}
+
+function _makeNewBackendRequest(
+  projectId,
+  userId,
+  compileGroup,
+  compileBackendClass,
+  baseOpts,
+  callback
+) {
+  if (Settings.apis.clsi_new == null || Settings.apis.clsi_new.url == null) {
+    return callback()
+  }
+  const opts = {
+    ...baseOpts,
+    url: baseOpts.url.replace(
+      Settings.apis.clsi.url,
+      Settings.apis.clsi_new.url
+    ),
+  }
+  NewBackendCloudClsiCookieManager.getCookieJar(
+    projectId,
+    userId,
+    compileGroup,
+    compileBackendClass,
+    (err, jar, clsiServerId) => {
+      if (err != null) {
+        return callback(
+          OError.tag(err, 'error getting cookie jar for CLSI request', {
+            projectId,
+          })
+        )
+      }
+      opts.jar = jar
+      const timer = new Metrics.Timer('compile.newBackend')
+      request(opts, (err, response, body) => {
+        timer.done()
         if (err != null) {
           return callback(
-            OError.tag(
-              err,
-              'could not check resources for potential problems before sending to clsi'
-            )
-          )
-        }
-        if (validationProblems != null) {
-          logger.debug(
-            { projectId, validationProblems },
-            'problems with users latex before compile was attempted'
-          )
-          return callback(
-            null,
-            'validation-problems',
-            null,
-            null,
-            validationProblems
+            OError.tag(err, 'error making request to new CLSI', {
+              projectId,
+              opts,
+            })
           )
         }
-        ClsiManager._postToClsi(
+        NewBackendCloudClsiCookieManager.setServerId(
           projectId,
           userId,
-          req,
-          options.compileBackendClass,
-          options.compileGroup,
-          (err, response, clsiServerId) => {
+          compileGroup,
+          compileBackendClass,
+          response,
+          clsiServerId,
+          err => {
             if (err != null) {
               return callback(
-                OError.tag(err, 'error sending request to clsi', {
+                OError.tag(err, 'error setting server id on new backend', {
                   projectId,
-                  userId,
                 })
               )
             }
-            const outputFiles = ClsiManager._parseOutputFiles(
-              projectId,
-              response && response.compile && response.compile.outputFiles
-            )
-            collectMetricsOnBlgFiles(outputFiles)
-            const compile = (response && response.compile) || {}
-            const status = compile.status
-            const stats = compile.stats
-            const timings = compile.timings
-            const outputUrlPrefix = compile.outputUrlPrefix
-            const validationProblems = undefined
-            callback(
-              null,
-              status,
-              outputFiles,
-              clsiServerId,
-              validationProblems,
-              stats,
-              timings,
-              outputUrlPrefix
-            )
+            callback(null, response, body)
           }
         )
-      }
-    )
-  },
+      })
+    }
+  )
+}
 
-  _makeRequestWithClsiServerId(
+function _getCompilerUrl(
+  compileBackendClass,
+  compileGroup,
+  projectId,
+  userId,
+  action
+) {
+  const u = new URL(`/project/${projectId}`, Settings.apis.clsi.url)
+  if (userId != null) {
+    u.pathname += `/user/${userId}`
+  }
+  if (action != null) {
+    u.pathname += `/${action}`
+  }
+  u.search = new URLSearchParams({
+    compileBackendClass,
+    compileGroup,
+  }).toString()
+  return u.href
+}
+
+function _postToClsi(
+  projectId,
+  userId,
+  req,
+  compileBackendClass,
+  compileGroup,
+  callback
+) {
+  const compileUrl = _getCompilerUrl(
+    compileBackendClass,
+    compileGroup,
+    projectId,
+    userId,
+    'compile'
+  )
+  const opts = {
+    url: compileUrl,
+    json: req,
+    method: 'POST',
+  }
+  _makeRequest(
     projectId,
     userId,
     compileGroup,
     compileBackendClass,
     opts,
-    clsiserverid,
-    callback
-  ) {
-    if (clsiserverid) {
-      // ignore cookies and newBackend, go straight to the clsi node
-      opts.qs = Object.assign(
-        { compileGroup, compileBackendClass, clsiserverid },
-        opts.qs
-      )
-      request(opts, (err, response, body) => {
-        if (err) {
-          return callback(
-            OError.tag(err, 'error making request to CLSI', { projectId })
+    (err, response, body, clsiServerId) => {
+      if (err != null) {
+        return callback(
+          new OError(
+            'failed to make request to CLSI',
+            {
+              projectId,
+              userId,
+              compileOptions: req.compile.options,
+              rootResourcePath: req.compile.rootResourcePath,
+            },
+            err
           )
-        }
-        callback(null, response, body)
-      })
-    } else {
-      ClsiManager._makeRequest(
-        projectId,
-        userId,
-        compileGroup,
-        compileBackendClass,
-        opts,
-        callback
-      )
-    }
-  },
-
-  _makeRequest(
-    projectId,
-    userId,
-    compileGroup,
-    compileBackendClass,
-    opts,
-    callback
-  ) {
-    async.series(
-      {
-        currentBackend(cb) {
-          const startTime = new Date()
-          ClsiCookieManager.getCookieJar(
-            projectId,
-            userId,
-            compileGroup,
-            compileBackendClass,
-            (err, jar, clsiServerId) => {
-              if (err != null) {
-                return callback(
-                  OError.tag(err, 'error getting cookie jar for CLSI request', {
-                    projectId,
-                  })
-                )
-              }
-              opts.jar = jar
-              const timer = new Metrics.Timer('compile.currentBackend')
-              request(opts, (err, response, body) => {
-                if (err != null) {
-                  return callback(
-                    OError.tag(err, 'error making request to CLSI', {
-                      projectId,
-                    })
-                  )
-                }
-                timer.done()
-                Metrics.inc(
-                  `compile.currentBackend.response.${response.statusCode}`
-                )
-                ClsiCookieManager.setServerId(
-                  projectId,
-                  userId,
-                  compileGroup,
-                  compileBackendClass,
-                  response,
-                  clsiServerId,
-                  (err, newClsiServerId) => {
-                    if (err != null) {
-                      callback(
-                        OError.tag(err, 'error setting server id', {
-                          projectId,
-                        })
-                      )
-                    } else {
-                      // return as soon as the standard compile has returned
-                      callback(
-                        null,
-                        response,
-                        body,
-                        newClsiServerId || clsiServerId
-                      )
-                    }
-                    cb(err, {
-                      response,
-                      body,
-                      finishTime: new Date() - startTime,
-                    })
-                  }
-                )
-              })
-            }
-          )
-        },
-        newBackend(cb) {
-          const startTime = new Date()
-          ClsiManager._makeNewBackendRequest(
-            projectId,
-            userId,
-            compileGroup,
-            compileBackendClass,
-            opts,
-            (err, response, body) => {
-              if (err != null) {
-                logger.warn({ err }, 'Error making request to new CLSI backend')
-              }
-              if (response != null) {
-                Metrics.inc(
-                  `compile.newBackend.response.${response.statusCode}`
-                )
-              }
-              cb(err, {
-                response,
-                body,
-                finishTime: new Date() - startTime,
-              })
-            }
-          )
-        },
-      },
-      (err, results) => {
-        if (err != null) {
-          // This was handled higher up
-          return
-        }
-        if (results.newBackend != null && results.newBackend.response != null) {
-          const currentStatusCode = results.currentBackend.response.statusCode
-          const newStatusCode = results.newBackend.response.statusCode
-          const statusCodeSame = newStatusCode === currentStatusCode
-          const currentCompileTime = results.currentBackend.finishTime
-          const newBackendCompileTime = results.newBackend.finishTime || 0
-          const timeDifference = newBackendCompileTime - currentCompileTime
-          logger.debug(
-            {
-              statusCodeSame,
-              timeDifference,
-              currentCompileTime,
-              newBackendCompileTime,
-              projectId,
-            },
-            'both clsi requests returned'
-          )
-        }
+        )
       }
-    )
-  },
-
-  _makeNewBackendRequest(
-    projectId,
-    userId,
-    compileGroup,
-    compileBackendClass,
-    baseOpts,
-    callback
-  ) {
-    if (Settings.apis.clsi_new == null || Settings.apis.clsi_new.url == null) {
-      return callback()
-    }
-    const opts = {
-      ...baseOpts,
-      url: baseOpts.url.replace(
-        Settings.apis.clsi.url,
-        Settings.apis.clsi_new.url
-      ),
-    }
-    NewBackendCloudClsiCookieManager.getCookieJar(
-      projectId,
-      userId,
-      compileGroup,
-      compileBackendClass,
-      (err, jar, clsiServerId) => {
-        if (err != null) {
-          return callback(
-            OError.tag(err, 'error getting cookie jar for CLSI request', {
-              projectId,
-            })
-          )
-        }
-        opts.jar = jar
-        const timer = new Metrics.Timer('compile.newBackend')
-        request(opts, (err, response, body) => {
-          timer.done()
-          if (err != null) {
-            return callback(
-              OError.tag(err, 'error making request to new CLSI', {
-                projectId,
-                opts,
-              })
-            )
-          }
-          NewBackendCloudClsiCookieManager.setServerId(
+      if (response.statusCode >= 200 && response.statusCode < 300) {
+        callback(null, body, clsiServerId)
+      } else if (response.statusCode === 413) {
+        callback(null, { compile: { status: 'project-too-large' } })
+      } else if (response.statusCode === 409) {
+        callback(null, { compile: { status: 'conflict' } })
+      } else if (response.statusCode === 423) {
+        callback(null, { compile: { status: 'compile-in-progress' } })
+      } else if (response.statusCode === 503) {
+        callback(null, { compile: { status: 'unavailable' } })
+      } else {
+        callback(
+          new OError(`CLSI returned non-success code: ${response.statusCode}`, {
             projectId,
             userId,
-            compileGroup,
-            compileBackendClass,
-            response,
-            clsiServerId,
-            err => {
-              if (err != null) {
-                return callback(
-                  OError.tag(err, 'error setting server id on new backend', {
-                    projectId,
-                  })
-                )
-              }
-              callback(null, response, body)
-            }
-          )
-        })
+            compileOptions: req.compile.options,
+            rootResourcePath: req.compile.rootResourcePath,
+            clsiResponse: body,
+            statusCode: response.statusCode,
+          })
+        )
       }
-    )
-  },
+    }
+  )
+}
 
-  _getCompilerUrl(
-    compileBackendClass,
-    compileGroup,
-    projectId,
-    userId,
-    action
-  ) {
-    const u = new URL(`/project/${projectId}`, Settings.apis.clsi.url)
-    if (userId != null) {
-      u.pathname += `/user/${userId}`
+function _parseOutputFiles(projectId, rawOutputFiles = []) {
+  const outputFiles = []
+  for (const file of rawOutputFiles) {
+    const f = {
+      path: file.path, // the clsi is now sending this to web
+      url: new URL(file.url).pathname, // the location of the file on the clsi, excluding the host part
+      type: file.type,
+      build: file.build,
     }
-    if (action != null) {
-      u.pathname += `/${action}`
+    if (file.path === 'output.pdf') {
+      f.contentId = file.contentId
+      f.ranges = file.ranges || []
+      f.size = file.size
+      f.startXRefTable = file.startXRefTable
+      f.createdAt = new Date()
     }
-    u.search = new URLSearchParams({
-      compileBackendClass,
-      compileGroup,
-    }).toString()
-    return u.href
-  },
+    outputFiles.push(f)
+  }
+  return outputFiles
+}
 
-  _postToClsi(
+function _buildRequest(projectId, options, callback) {
+  if (options == null) {
+    options = {}
+  }
+  ProjectGetter.getProject(
     projectId,
-    userId,
-    req,
-    compileBackendClass,
-    compileGroup,
-    callback
-  ) {
-    const compileUrl = this._getCompilerUrl(
-      compileBackendClass,
-      compileGroup,
-      projectId,
-      userId,
-      'compile'
-    )
-    const opts = {
-      url: compileUrl,
-      json: req,
-      method: 'POST',
-    }
-    ClsiManager._makeRequest(
-      projectId,
-      userId,
-      compileGroup,
-      compileBackendClass,
-      opts,
-      (err, response, body, clsiServerId) => {
-        if (err != null) {
-          return callback(
-            new OError(
-              'failed to make request to CLSI',
-              {
-                projectId,
-                userId,
-                compileOptions: req.compile.options,
-                rootResourcePath: req.compile.rootResourcePath,
-              },
-              err
-            )
-          )
-        }
-        if (response.statusCode >= 200 && response.statusCode < 300) {
-          callback(null, body, clsiServerId)
-        } else if (response.statusCode === 413) {
-          callback(null, { compile: { status: 'project-too-large' } })
-        } else if (response.statusCode === 409) {
-          callback(null, { compile: { status: 'conflict' } })
-        } else if (response.statusCode === 423) {
-          callback(null, { compile: { status: 'compile-in-progress' } })
-        } else if (response.statusCode === 503) {
-          callback(null, { compile: { status: 'unavailable' } })
-        } else {
-          callback(
-            new OError(
-              `CLSI returned non-success code: ${response.statusCode}`,
-              {
-                projectId,
-                userId,
-                compileOptions: req.compile.options,
-                rootResourcePath: req.compile.rootResourcePath,
-                clsiResponse: body,
-                statusCode: response.statusCode,
-              }
-            )
-          )
-        }
+    { compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder: 1 },
+    (err, project) => {
+      if (err != null) {
+        return callback(OError.tag(err, 'failed to get project', { projectId }))
       }
-    )
-  },
-
-  _parseOutputFiles(projectId, rawOutputFiles = []) {
-    const outputFiles = []
-    for (const file of rawOutputFiles) {
-      const f = {
-        path: file.path, // the clsi is now sending this to web
-        url: new URL(file.url).pathname, // the location of the file on the clsi, excluding the host part
-        type: file.type,
-        build: file.build,
+      if (project == null) {
+        return callback(
+          new Errors.NotFoundError(`project does not exist: ${projectId}`)
+        )
       }
-      if (file.path === 'output.pdf') {
-        f.contentId = file.contentId
-        f.ranges = file.ranges || []
-        f.size = file.size
-        f.startXRefTable = file.startXRefTable
-        f.createdAt = new Date()
+      if (!VALID_COMPILERS.includes(project.compiler)) {
+        project.compiler = 'pdflatex'
       }
-      outputFiles.push(f)
-    }
-    return outputFiles
-  },
-
-  _buildRequest(projectId, options, callback) {
-    if (options == null) {
-      options = {}
-    }
-    ProjectGetter.getProject(
-      projectId,
-      { compiler: 1, rootDoc_id: 1, imageName: 1, rootFolder: 1 },
-      (err, project) => {
-        if (err != null) {
-          return callback(
-            OError.tag(err, 'failed to get project', { projectId })
-          )
-        }
-        if (project == null) {
-          return callback(
-            new Errors.NotFoundError(`project does not exist: ${projectId}`)
-          )
-        }
-        if (!VALID_COMPILERS.includes(project.compiler)) {
-          project.compiler = 'pdflatex'
-        }
 
-        if (options.incrementalCompilesEnabled || options.syncType != null) {
-          // new way, either incremental or full
-          const timer = new Metrics.Timer('editor.compile-getdocs-redis')
-          ClsiManager.getContentFromDocUpdaterIfMatch(
-            projectId,
-            project,
-            options,
-            (err, projectStateHash, docUpdaterDocs) => {
-              timer.done()
-              if (err != null) {
-                logger.error({ err, projectId }, 'error checking project state')
-                // note: we don't bail out when there's an error getting
-                // incremental files from the docupdater, we just fall back
-                // to a normal compile below
-              }
-              // see if we can send an incremental update to the CLSI
-              if (
-                docUpdaterDocs != null &&
-                options.syncType !== 'full' &&
-                err == null
-              ) {
-                Metrics.inc('compile-from-redis')
-                ClsiManager._buildRequestFromDocupdater(
-                  projectId,
-                  options,
-                  project,
-                  projectStateHash,
-                  docUpdaterDocs,
-                  callback
-                )
-              } else {
-                Metrics.inc('compile-from-mongo')
-                ClsiManager._buildRequestFromMongo(
-                  projectId,
-                  options,
-                  project,
-                  projectStateHash,
-                  callback
-                )
-              }
-            }
-          )
-        } else {
-          // old way, always from mongo
-          const timer = new Metrics.Timer('editor.compile-getdocs-mongo')
-          ClsiManager._getContentFromMongo(projectId, (err, docs, files) => {
+      if (options.incrementalCompilesEnabled || options.syncType != null) {
+        // new way, either incremental or full
+        const timer = new Metrics.Timer('editor.compile-getdocs-redis')
+        getContentFromDocUpdaterIfMatch(
+          projectId,
+          project,
+          options,
+          (err, projectStateHash, docUpdaterDocs) => {
             timer.done()
             if (err != null) {
-              return callback(
-                OError.tag(err, 'failed to get contents from Mongo', {
-                  projectId,
-                })
+              logger.error({ err, projectId }, 'error checking project state')
+              // note: we don't bail out when there's an error getting
+              // incremental files from the docupdater, we just fall back
+              // to a normal compile below
+            }
+            // see if we can send an incremental update to the CLSI
+            if (
+              docUpdaterDocs != null &&
+              options.syncType !== 'full' &&
+              err == null
+            ) {
+              Metrics.inc('compile-from-redis')
+              _buildRequestFromDocupdater(
+                projectId,
+                options,
+                project,
+                projectStateHash,
+                docUpdaterDocs,
+                callback
+              )
+            } else {
+              Metrics.inc('compile-from-mongo')
+              _buildRequestFromMongo(
+                projectId,
+                options,
+                project,
+                projectStateHash,
+                callback
               )
             }
-            ClsiManager._finaliseRequest(
-              projectId,
-              options,
-              project,
-              docs,
-              files,
-              callback
+          }
+        )
+      } else {
+        // old way, always from mongo
+        const timer = new Metrics.Timer('editor.compile-getdocs-mongo')
+        _getContentFromMongo(projectId, (err, docs, files) => {
+          timer.done()
+          if (err != null) {
+            return callback(
+              OError.tag(err, 'failed to get contents from Mongo', {
+                projectId,
+              })
             )
-          })
-        }
-      }
-    )
-  },
-
-  getContentFromDocUpdaterIfMatch(projectId, project, options, callback) {
-    let projectStateHash
-    try {
-      projectStateHash = ClsiStateManager.computeHash(project, options)
-    } catch (err) {
-      return callback(err)
-    }
-    DocumentUpdaterHandler.getProjectDocsIfMatch(
-      projectId,
-      projectStateHash,
-      (err, docs) => {
-        if (err != null) {
-          return callback(
-            OError.tag(err, 'Failed to get project documents', {
-              projectId,
-              projectStateHash,
-            })
-          )
-        }
-        callback(null, projectStateHash, docs)
-      }
-    )
-  },
-
-  getOutputFileStream(
-    projectId,
-    userId,
-    options,
-    clsiServerId,
-    buildId,
-    outputFilePath,
-    callback
-  ) {
-    const url = `${Settings.apis.clsi.url}/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}`
-    const { compileBackendClass, compileGroup } = options
-    const readStream = request({
-      url,
-      method: 'GET',
-      timeout: 60 * 1000,
-      qs: { compileBackendClass, compileGroup, clsiserverid: clsiServerId },
-    })
-    callback(null, readStream)
-  },
-
-  _buildRequestFromDocupdater(
-    projectId,
-    options,
-    project,
-    projectStateHash,
-    docUpdaterDocs,
-    callback
-  ) {
-    let docPath
-    try {
-      docPath = ProjectEntityHandler.getAllDocPathsFromProject(project)
-    } catch (err) {
-      return callback(
-        OError.tag(err, 'Failed to get all doc paths from project', {
-          projectId,
+          }
+          _finaliseRequest(projectId, options, project, docs, files, callback)
         })
-      )
-    }
-    const docs = {}
-    for (const doc of docUpdaterDocs || []) {
-      const path = docPath[doc._id]
-      docs[path] = doc
-    }
-    // send new docs but not files as those are already on the clsi
-    options = _.clone(options)
-    options.syncType = 'incremental'
-    options.syncState = projectStateHash
-    // create stub doc entries for any possible root docs, if not
-    // present in the docupdater. This allows finaliseRequest to
-    // identify the root doc.
-    const possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id]
-    for (const rootDocId of possibleRootDocIds) {
-      if (rootDocId != null && rootDocId in docPath) {
-        const path = docPath[rootDocId]
-        if (docs[path] == null) {
-          docs[path] = { _id: rootDocId, path }
-        }
       }
     }
-    ClsiManager._finaliseRequest(
-      projectId,
-      options,
-      project,
-      docs,
-      [],
-      callback
-    )
-  },
+  )
+}
 
-  _buildRequestFromMongo(
+function getContentFromDocUpdaterIfMatch(
+  projectId,
+  project,
+  options,
+  callback
+) {
+  let projectStateHash
+  try {
+    projectStateHash = ClsiStateManager.computeHash(project, options)
+  } catch (err) {
+    return callback(err)
+  }
+  DocumentUpdaterHandler.getProjectDocsIfMatch(
     projectId,
-    options,
-    project,
     projectStateHash,
-    callback
-  ) {
-    ClsiManager._getContentFromMongo(projectId, (err, docs, files) => {
+    (err, docs) => {
       if (err != null) {
         return callback(
-          OError.tag(err, 'failed to get project contents from Mongo', {
+          OError.tag(err, 'Failed to get project documents', {
             projectId,
+            projectStateHash,
           })
         )
       }
-      options = {
-        ...options,
-        syncType: 'full',
-        syncState: projectStateHash,
-      }
-      ClsiManager._finaliseRequest(
+      callback(null, projectStateHash, docs)
+    }
+  )
+}
+
+function getOutputFileStream(
+  projectId,
+  userId,
+  options,
+  clsiServerId,
+  buildId,
+  outputFilePath,
+  callback
+) {
+  const url = `${Settings.apis.clsi.url}/project/${projectId}/user/${userId}/build/${buildId}/output/${outputFilePath}`
+  const { compileBackendClass, compileGroup } = options
+  const readStream = request({
+    url,
+    method: 'GET',
+    timeout: 60 * 1000,
+    qs: { compileBackendClass, compileGroup, clsiserverid: clsiServerId },
+  })
+  callback(null, readStream)
+}
+
+function _buildRequestFromDocupdater(
+  projectId,
+  options,
+  project,
+  projectStateHash,
+  docUpdaterDocs,
+  callback
+) {
+  let docPath
+  try {
+    docPath = ProjectEntityHandler.getAllDocPathsFromProject(project)
+  } catch (err) {
+    return callback(
+      OError.tag(err, 'Failed to get all doc paths from project', {
         projectId,
-        options,
-        project,
-        docs,
-        files,
-        callback
+      })
+    )
+  }
+  const docs = {}
+  for (const doc of docUpdaterDocs || []) {
+    const path = docPath[doc._id]
+    docs[path] = doc
+  }
+  // send new docs but not files as those are already on the clsi
+  options = _.clone(options)
+  options.syncType = 'incremental'
+  options.syncState = projectStateHash
+  // create stub doc entries for any possible root docs, if not
+  // present in the docupdater. This allows finaliseRequest to
+  // identify the root doc.
+  const possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id]
+  for (const rootDocId of possibleRootDocIds) {
+    if (rootDocId != null && rootDocId in docPath) {
+      const path = docPath[rootDocId]
+      if (docs[path] == null) {
+        docs[path] = { _id: rootDocId, path }
+      }
+    }
+  }
+  _finaliseRequest(projectId, options, project, docs, [], callback)
+}
+
+function _buildRequestFromMongo(
+  projectId,
+  options,
+  project,
+  projectStateHash,
+  callback
+) {
+  _getContentFromMongo(projectId, (err, docs, files) => {
+    if (err != null) {
+      return callback(
+        OError.tag(err, 'failed to get project contents from Mongo', {
+          projectId,
+        })
       )
-    })
-  },
+    }
+    options = {
+      ...options,
+      syncType: 'full',
+      syncState: projectStateHash,
+    }
+    _finaliseRequest(projectId, options, project, docs, files, callback)
+  })
+}
 
-  _getContentFromMongo(projectId, callback) {
-    DocumentUpdaterHandler.flushProjectToMongo(projectId, err => {
+function _getContentFromMongo(projectId, callback) {
+  DocumentUpdaterHandler.flushProjectToMongo(projectId, err => {
+    if (err != null) {
+      return callback(
+        OError.tag(err, 'failed to flush project to Mongo', { projectId })
+      )
+    }
+    ProjectEntityHandler.getAllDocs(projectId, (err, docs) => {
       if (err != null) {
         return callback(
-          OError.tag(err, 'failed to flush project to Mongo', { projectId })
+          OError.tag(err, 'failed to get project docs', { projectId })
         )
       }
-      ProjectEntityHandler.getAllDocs(projectId, (err, docs) => {
+      ProjectEntityHandler.getAllFiles(projectId, (err, files) => {
         if (err != null) {
           return callback(
-            OError.tag(err, 'failed to get project docs', { projectId })
+            OError.tag(err, 'failed to get project files', { projectId })
           )
         }
-        ProjectEntityHandler.getAllFiles(projectId, (err, files) => {
-          if (err != null) {
-            return callback(
-              OError.tag(err, 'failed to get project files', { projectId })
-            )
-          }
-          if (files == null) {
-            files = {}
-          }
-          callback(null, docs || {}, files || {})
-        })
+        if (files == null) {
+          files = {}
+        }
+        callback(null, docs || {}, files || {})
       })
     })
-  },
-
-  _finaliseRequest(projectId, options, project, docs, files, callback) {
-    const resources = []
-    let flags
-    let rootResourcePath = null
-    let rootResourcePathOverride = null
-    let hasMainFile = false
-    let numberOfDocsInProject = 0
-
-    for (let path in docs) {
-      const doc = docs[path]
-      path = path.replace(/^\//, '') // Remove leading /
-      numberOfDocsInProject++
-      if (doc.lines != null) {
-        // add doc to resources unless it is just a stub entry
-        resources.push({
-          path,
-          content: doc.lines.join('\n'),
-        })
-      }
-      if (
-        project.rootDoc_id != null &&
-        doc._id.toString() === project.rootDoc_id.toString()
-      ) {
-        rootResourcePath = path
-      }
-      if (
-        options.rootDoc_id != null &&
-        doc._id.toString() === options.rootDoc_id.toString()
-      ) {
-        rootResourcePathOverride = path
-      }
-      if (path === 'main.tex') {
-        hasMainFile = true
-      }
-    }
+  })
+}
 
-    if (rootResourcePathOverride != null) {
-      rootResourcePath = rootResourcePathOverride
-    }
-    if (rootResourcePath == null) {
-      if (hasMainFile) {
-        rootResourcePath = 'main.tex'
-      } else if (numberOfDocsInProject === 1) {
-        // only one file, must be the main document
-        for (const path in docs) {
-          // Remove leading /
-          rootResourcePath = path.replace(/^\//, '')
-        }
-      } else {
-        return callback(new OError('no main file specified', { projectId }))
-      }
-    }
+function _finaliseRequest(projectId, options, project, docs, files, callback) {
+  const resources = []
+  let flags
+  let rootResourcePath = null
+  let rootResourcePathOverride = null
+  let hasMainFile = false
+  let numberOfDocsInProject = 0
 
-    for (let path in files) {
-      const file = files[path]
-      path = path.replace(/^\//, '') // Remove leading /
+  for (let path in docs) {
+    const doc = docs[path]
+    path = path.replace(/^\//, '') // Remove leading /
+    numberOfDocsInProject++
+    if (doc.lines != null) {
+      // add doc to resources unless it is just a stub entry
       resources.push({
         path,
-        url: `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`,
-        modified: file.created != null ? file.created.getTime() : undefined,
+        content: doc.lines.join('\n'),
       })
     }
+    if (
+      project.rootDoc_id != null &&
+      doc._id.toString() === project.rootDoc_id.toString()
+    ) {
+      rootResourcePath = path
+    }
+    if (
+      options.rootDoc_id != null &&
+      doc._id.toString() === options.rootDoc_id.toString()
+    ) {
+      rootResourcePathOverride = path
+    }
+    if (path === 'main.tex') {
+      hasMainFile = true
+    }
+  }
 
-    if (options.fileLineErrors) {
-      flags = ['-file-line-error']
+  if (rootResourcePathOverride != null) {
+    rootResourcePath = rootResourcePathOverride
+  }
+  if (rootResourcePath == null) {
+    if (hasMainFile) {
+      rootResourcePath = 'main.tex'
+    } else if (numberOfDocsInProject === 1) {
+      // only one file, must be the main document
+      for (const path in docs) {
+        // Remove leading /
+        rootResourcePath = path.replace(/^\//, '')
+      }
+    } else {
+      return callback(new OError('no main file specified', { projectId }))
     }
+  }
 
-    callback(null, {
-      compile: {
-        options: {
-          compiler: project.compiler,
-          timeout: options.timeout,
-          imageName: project.imageName,
-          draft: Boolean(options.draft),
-          stopOnFirstError: Boolean(options.stopOnFirstError),
-          check: options.check,
-          syncType: options.syncType,
-          syncState: options.syncState,
-          compileGroup: options.compileGroup,
-          enablePdfCaching:
-            (Settings.enablePdfCaching && options.enablePdfCaching) || false,
-          pdfCachingMinChunkSize: options.pdfCachingMinChunkSize,
-          flags,
-          metricsMethod: options.compileGroup,
-        },
-        rootResourcePath,
-        resources,
-      },
+  for (let path in files) {
+    const file = files[path]
+    path = path.replace(/^\//, '') // Remove leading /
+    resources.push({
+      path,
+      url: `${Settings.apis.filestore.url}/project/${project._id}/file/${file._id}`,
+      modified: file.created?.getTime(),
     })
-  },
+  }
 
-  wordCount(projectId, userId, file, options, clsiserverid, callback) {
-    const { compileBackendClass, compileGroup } = options
-    ClsiManager._buildRequest(projectId, options, (err, req) => {
-      if (err != null) {
-        return callback(
-          OError.tag(err, 'Failed to build CLSI request', {
-            projectId,
-            options,
-          })
-        )
-      }
-      const filename = file || req.compile.rootResourcePath
-      const wordCountUrl = ClsiManager._getCompilerUrl(
-        compileBackendClass,
-        compileGroup,
-        projectId,
-        userId,
-        'wordcount'
+  if (options.fileLineErrors) {
+    flags = ['-file-line-error']
+  }
+
+  callback(null, {
+    compile: {
+      options: {
+        compiler: project.compiler,
+        timeout: options.timeout,
+        imageName: project.imageName,
+        draft: Boolean(options.draft),
+        stopOnFirstError: Boolean(options.stopOnFirstError),
+        check: options.check,
+        syncType: options.syncType,
+        syncState: options.syncState,
+        compileGroup: options.compileGroup,
+        enablePdfCaching:
+          (Settings.enablePdfCaching && options.enablePdfCaching) || false,
+        pdfCachingMinChunkSize: options.pdfCachingMinChunkSize,
+        flags,
+        metricsMethod: options.compileGroup,
+      },
+      rootResourcePath,
+      resources,
+    },
+  })
+}
+
+function wordCount(projectId, userId, file, options, clsiserverid, callback) {
+  const { compileBackendClass, compileGroup } = options
+  _buildRequest(projectId, options, (err, req) => {
+    if (err != null) {
+      return callback(
+        OError.tag(err, 'Failed to build CLSI request', {
+          projectId,
+          options,
+        })
       )
-      const opts = {
-        url: wordCountUrl,
-        qs: {
-          file: filename,
-          image: req.compile.options.imageName,
-        },
-        json: true,
-        method: 'GET',
-      }
-      ClsiManager._makeRequestWithClsiServerId(
-        projectId,
-        userId,
-        compileGroup,
-        compileBackendClass,
-        opts,
-        clsiserverid,
-        (err, response, body) => {
-          if (err != null) {
-            return callback(
-              OError.tag(err, 'CLSI request failed', { projectId })
-            )
-          }
-          if (response.statusCode >= 200 && response.statusCode < 300) {
-            callback(null, body)
-          } else {
-            callback(
-              new OError(
-                `CLSI returned non-success code: ${response.statusCode}`,
-                {
-                  projectId,
-                  clsiResponse: body,
-                  statusCode: response.statusCode,
-                }
-              )
+    }
+    const filename = file || req.compile.rootResourcePath
+    const wordCountUrl = _getCompilerUrl(
+      compileBackendClass,
+      compileGroup,
+      projectId,
+      userId,
+      'wordcount'
+    )
+    const opts = {
+      url: wordCountUrl,
+      qs: {
+        file: filename,
+        image: req.compile.options.imageName,
+      },
+      json: true,
+      method: 'GET',
+    }
+    _makeRequestWithClsiServerId(
+      projectId,
+      userId,
+      compileGroup,
+      compileBackendClass,
+      opts,
+      clsiserverid,
+      (err, response, body) => {
+        if (err != null) {
+          return callback(OError.tag(err, 'CLSI request failed', { projectId }))
+        }
+        if (response.statusCode >= 200 && response.statusCode < 300) {
+          callback(null, body)
+        } else {
+          callback(
+            new OError(
+              `CLSI returned non-success code: ${response.statusCode}`,
+              {
+                projectId,
+                clsiResponse: body,
+                statusCode: response.statusCode,
+              }
             )
-          }
+          )
         }
-      )
-    })
-  },
+      }
+    )
+  })
 }
 
-module.exports = ClsiManager
+module.exports = {
+  sendRequest,
+  sendExternalRequest,
+  stopCompile,
+  deleteAuxFiles,
+  getOutputFileStream,
+  wordCount,
+}

services/web/app/src/router.js

@@ -1272,7 +1272,7 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) {
       const sendRes = _.once(function (statusCode, message) {
         res.status(statusCode)
         plainTextResponse(res, message)
-        ClsiCookieManager.clearServerId(projectId)
+        ClsiCookieManager.clearServerId(projectId, () => {})
       }) // force every compile to a new server
       // set a timeout
       let handler = setTimeout(function () {

services/web/test/unit/src/Compile/ClsiCookieManagerTests.js

@@ -1,16 +1,3 @@
-/* eslint-disable
-    n/handle-callback-err,
-    max-len,
-    no-return-assign,
-    no-unused-vars,
-*/
-// TODO: This file was created by bulk-decaffeinate.
-// Fix any style issues and re-enable lint.
-/*
- * decaffeinate suggestions:
- * DS102: Remove unnecessary code created because of implicit returns
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
 const sinon = require('sinon')
 const { assert, expect } = require('chai')
 const modulePath = '../../../../app/src/Features/Compile/ClsiCookieManager.js'
@@ -19,7 +6,6 @@ const realRequst = require('request')
 
 describe('ClsiCookieManager', function () {
   beforeEach(function () {
-    const self = this
     this.redis = {
       auth() {},
       get: sinon.stub(),
@@ -57,25 +43,28 @@ describe('ClsiCookieManager', function () {
       '@overleaf/settings': this.settings,
       request: this.request,
     }
-    return (this.ClsiCookieManager = SandboxedModule.require(modulePath, {
+    this.ClsiCookieManager = SandboxedModule.require(modulePath, {
       requires: this.requires,
-    })())
+    })()
   })
 
   describe('getServerId', function () {
     it('should call get for the key', function (done) {
       this.redis.get.callsArgWith(1, null, 'clsi-7')
-      return this.ClsiCookieManager._getServerId(
+      this.ClsiCookieManager._getServerId(
         this.project_id,
         this.user_id,
         '',
         'e2',
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           this.redis.get
             .calledWith(`clsiserver:${this.project_id}:${this.user_id}`)
             .should.equal(true)
           serverId.should.equal('clsi-7')
-          return done()
+          done()
         }
       )
     })
@@ -85,15 +74,18 @@ describe('ClsiCookieManager', function () {
         .stub()
         .yields(null)
       this.redis.get.callsArgWith(1, null)
-      return this.ClsiCookieManager._getServerId(
+      this.ClsiCookieManager._getServerId(
         this.project_id,
         this.user_id,
         '',
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           this.ClsiCookieManager._populateServerIdViaRequest
             .calledWith(this.project_id, this.user_id)
             .should.equal(true)
-          return done()
+          done()
         }
       )
     })
@@ -103,16 +95,19 @@ describe('ClsiCookieManager', function () {
         .stub()
         .yields(null)
       this.redis.get.callsArgWith(1, null, '')
-      return this.ClsiCookieManager._getServerId(
+      this.ClsiCookieManager._getServerId(
         this.project_id,
         this.user_id,
         '',
         'e2',
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           this.ClsiCookieManager._populateServerIdViaRequest
             .calledWith(this.project_id, this.user_id)
             .should.equal(true)
-          return done()
+          done()
         }
       )
     })
@@ -122,38 +117,42 @@ describe('ClsiCookieManager', function () {
     beforeEach(function () {
       this.response = 'some data'
       this.request.post.callsArgWith(1, null, this.response)
-      return (this.ClsiCookieManager.setServerId = sinon
-        .stub()
-        .yields(null, 'clsi-9'))
+      this.ClsiCookieManager.setServerId = sinon.stub().yields(null, 'clsi-9')
     })
 
     it('should make a request to the clsi', function (done) {
-      return this.ClsiCookieManager._populateServerIdViaRequest(
+      this.ClsiCookieManager._populateServerIdViaRequest(
         this.project_id,
         this.user_id,
         'standard',
         'e2',
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           const args = this.ClsiCookieManager.setServerId.args[0]
           args[0].should.equal(this.project_id)
           args[1].should.equal(this.user_id)
           args[2].should.equal('standard')
           args[3].should.equal('e2')
           args[4].should.deep.equal(this.response)
-          return done()
+          done()
         }
       )
     })
 
     it('should return the server id', function (done) {
-      return this.ClsiCookieManager._populateServerIdViaRequest(
+      this.ClsiCookieManager._populateServerIdViaRequest(
         this.project_id,
         this.user_id,
         '',
         'e2',
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           serverId.should.equal('clsi-9')
-          return done()
+          done()
         }
       )
     })
@@ -168,7 +167,7 @@ describe('ClsiCookieManager', function () {
     })
 
     it('should set the server id with a ttl', function (done) {
-      return this.ClsiCookieManager.setServerId(
+      this.ClsiCookieManager.setServerId(
         this.project_id,
         this.user_id,
         'standard',
@@ -176,6 +175,9 @@ describe('ClsiCookieManager', function () {
         this.response,
         null,
         err => {
+          if (err) {
+            return done(err)
+          }
           this.redis.setex
             .calledWith(
               `clsiserver:${this.project_id}:${this.user_id}`,
@@ -183,7 +185,7 @@ describe('ClsiCookieManager', function () {
               'clsi-8'
             )
             .should.equal(true)
-          return done()
+          done()
         }
       )
     })
@@ -200,6 +202,9 @@ describe('ClsiCookieManager', function () {
         this.response,
         null,
         err => {
+          if (err) {
+            return done(err)
+          }
           expect(this.redis.setex).to.have.been.calledWith(
             `clsiserver:${this.project_id}:${this.user_id}`,
             this.settings.clsiCookie.ttlInSecondsRegular,
@@ -211,7 +216,7 @@ describe('ClsiCookieManager', function () {
     })
 
     it('should return the server id', function (done) {
-      return this.ClsiCookieManager.setServerId(
+      this.ClsiCookieManager.setServerId(
         this.project_id,
         this.user_id,
         'standard',
@@ -219,8 +224,11 @@ describe('ClsiCookieManager', function () {
         this.response,
         null,
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           serverId.should.equal('clsi-8')
-          return done()
+          done()
         }
       )
     })
@@ -233,7 +241,7 @@ describe('ClsiCookieManager', function () {
         },
         requires: this.requires,
       })()
-      return this.ClsiCookieManager.setServerId(
+      this.ClsiCookieManager.setServerId(
         this.project_id,
         this.user_id,
         'standard',
@@ -241,8 +249,11 @@ describe('ClsiCookieManager', function () {
         this.response,
         null,
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           this.redis.setex.called.should.equal(false)
-          return done()
+          done()
         }
       )
     })
@@ -251,7 +262,7 @@ describe('ClsiCookieManager', function () {
       this.ClsiCookieManager._parseServerIdFromResponse = sinon
         .stub()
         .returns(null)
-      return this.ClsiCookieManager.setServerId(
+      this.ClsiCookieManager.setServerId(
         this.project_id,
         this.user_id,
         'standard',
@@ -259,8 +270,11 @@ describe('ClsiCookieManager', function () {
         this.response,
         null,
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           this.redis.setex.called.should.equal(false)
-          return done()
+          done()
         }
       )
     })
@@ -282,7 +296,7 @@ describe('ClsiCookieManager', function () {
       this.ClsiCookieManager._parseServerIdFromResponse = sinon
         .stub()
         .returns('clsi-8')
-      return this.ClsiCookieManager.setServerId(
+      this.ClsiCookieManager.setServerId(
         this.project_id,
         this.user_id,
         'standard',
@@ -290,6 +304,9 @@ describe('ClsiCookieManager', function () {
         this.response,
         null,
         (err, serverId) => {
+          if (err) {
+            return done(err)
+          }
           this.redis_secondary.setex
             .calledWith(
               `clsiserver:${this.project_id}:${this.user_id}`,
@@ -297,7 +314,7 @@ describe('ClsiCookieManager', function () {
               'clsi-8'
             )
             .should.equal(true)
-          return done()
+          done()
         }
       )
     })
@@ -305,25 +322,26 @@ describe('ClsiCookieManager', function () {
 
   describe('getCookieJar', function () {
     beforeEach(function () {
-      return (this.ClsiCookieManager._getServerId = sinon
-        .stub()
-        .yields(null, 'clsi-11'))
+      this.ClsiCookieManager._getServerId = sinon.stub().yields(null, 'clsi-11')
     })
 
     it('should return a jar with the cookie set populated from redis', function (done) {
-      return this.ClsiCookieManager.getCookieJar(
+      this.ClsiCookieManager.getCookieJar(
         this.project_id,
         this.user_id,
         '',
         'e2',
         (err, jar) => {
+          if (err) {
+            return done(err)
+          }
           jar._jar.store.idx['clsi.example.com']['/'][
             this.settings.clsiCookie.key
           ].key.should.equal
           jar._jar.store.idx['clsi.example.com']['/'][
             this.settings.clsiCookie.key
           ].value.should.equal('clsi-11')
-          return done()
+          done()
         }
       )
     })
@@ -336,14 +354,17 @@ describe('ClsiCookieManager', function () {
         },
         requires: this.requires,
       })()
-      return this.ClsiCookieManager.getCookieJar(
+      this.ClsiCookieManager.getCookieJar(
         this.project_id,
         this.user_id,
         '',
         'e2',
         (err, jar) => {
+          if (err) {
+            return done(err)
+          }
           assert.deepEqual(jar, realRequst.jar())
-          return done()
+          done()
         }
       )
     })

services/web/test/unit/src/Compile/ClsiManagerTests.js

@@ -4,28 +4,71 @@ const modulePath = '../../../../app/src/Features/Compile/ClsiManager.js'
 const SandboxedModule = require('sandboxed-module')
 const tk = require('timekeeper')
 
+const FILESTORE_URL = 'http://filestore.example.com'
+const CLSI_URL = 'http://clsi.example.com'
+
 describe('ClsiManager', function () {
   beforeEach(function () {
     this.jar = { cookie: 'stuff' }
+    this.user_id = 'user-id'
+    this.project = {
+      _id: 'project-id',
+      compiler: 'latex',
+      rootDoc_id: 'mock-doc-id-1',
+      imageName: 'mock-image-name',
+    }
+    this.docs = {
+      '/main.tex': {
+        name: 'main.tex',
+        _id: 'mock-doc-id-1',
+        lines: ['Hello', 'world'],
+      },
+      '/chapters/chapter1.tex': {
+        name: 'chapter1.tex',
+        _id: 'mock-doc-id-2',
+        lines: ['Chapter 1'],
+      },
+    }
+    this.files = {
+      '/images/image.png': {
+        name: 'image.png',
+        _id: 'mock-file-id-1',
+        created: new Date(),
+      },
+    }
+    this.rawOutputFiles = {}
+    this.response = { statusCode: 200 }
+    this.responseBody = {
+      compile: { status: 'success' },
+    }
+
+    this.request = sinon.stub().yields(null, this.response, this.responseBody)
     this.ClsiCookieManager = {
       clearServerId: sinon.stub().yields(),
       getCookieJar: sinon.stub().yields(null, this.jar),
       setServerId: sinon.stub().yields(null),
-      _getServerId: sinon.stub(),
     }
     this.ClsiStateManager = {
       computeHash: sinon.stub().returns('01234567890abcdef'),
     }
     this.ClsiFormatChecker = {
-      checkRecoursesForProblems: sinon.stub().callsArgWith(1),
+      checkRecoursesForProblems: sinon.stub().yields(),
     }
     this.Project = {}
-    this.ProjectEntityHandler = {}
-    this.ProjectGetter = {}
+    this.ProjectEntityHandler = {
+      getAllDocs: sinon.stub().yields(null, this.docs),
+      getAllFiles: sinon.stub().yields(null, this.files),
+      getAllDocPathsFromProject: sinon.stub(),
+    }
+    this.ProjectGetter = {
+      findById: sinon.stub().yields(null, this.project),
+      getProject: sinon.stub().yields(null, this.project),
+    }
     this.DocumentUpdaterHandler = {
-      getProjectDocsIfMatch: sinon.stub().callsArgWith(2, null, null),
+      clearProjectState: sinon.stub().yields(),
+      flushProjectToMongo: sinon.stub().yields(),
+      getProjectDocsIfMatch: sinon.stub().yields(),
     }
-    this.request = sinon.stub()
     this.Metrics = {
       Timer: class Metrics {
         constructor() {
@@ -35,16 +78,20 @@ describe('ClsiManager', function () {
       inc: sinon.stub(),
       count: sinon.stub(),
     }
+    this.SplitTestHandler = {
+      getAssignment: sinon.stub().yields(null, { variant: 'default' }),
+    }
+
     this.ClsiManager = SandboxedModule.require(modulePath, {
       requires: {
         '@overleaf/settings': (this.settings = {
           apis: {
             filestore: {
-              url: 'filestore.example.com',
+              url: FILESTORE_URL,
               secret: 'secret',
             },
             clsi: {
-              url: 'http://clsi.example.com',
+              url: CLSI_URL,
               defaultBackendClass: 'e2',
             },
             clsi_priority: {
@@ -65,15 +112,9 @@ describe('ClsiManager', function () {
         request: this.request,
         './ClsiFormatChecker': this.ClsiFormatChecker,
         '@overleaf/metrics': this.Metrics,
-        '../SplitTests/SplitTestHandler': {
-          getAssignment: (this.getAssignment = sinon.stub().yields(null, {
-            variant: 'default',
-          })),
-        },
+        '../SplitTests/SplitTestHandler': this.SplitTestHandler,
       },
     })
-    this.project_id = 'project-id'
-    this.user_id = 'user-id'
     this.callback = sinon.stub()
     tk.freeze(Date.now())
   })
@@ -84,72 +125,17 @@ describe('ClsiManager', function () {
 
   describe('sendRequest', function () {
     beforeEach(function () {
-      this.ClsiManager._buildRequest = sinon
-        .stub()
-        .callsArgWith(2, null, (this.request = 'mock-request'))
-      this.ClsiCookieManager._getServerId.callsArgWith(2, null, 'clsi3')
+      this.ClsiCookieManager.getCookieJar.yields(null, this.jar, 'clsi3')
     })
 
     describe('with a successful compile', function () {
-      beforeEach(function () {
-        this.ClsiManager._postToClsi = sinon.stub().yields(null, {
-          compile: {
-            status: (this.status = 'success'),
-            outputFiles: [
-              {
-                url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`,
-                path: 'output.pdf',
-                type: 'pdf',
-                build: 1234,
-              },
-              {
-                url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`,
-                path: 'output.log',
-                type: 'log',
-                build: 1234,
-              },
-            ],
-          },
-        })
-        this.ClsiManager.sendRequest(
-          this.project_id,
-          this.user_id,
-          { compileBackendClass: 'e2', compileGroup: 'standard' },
-          this.callback
-        )
-      })
-
-      it('should build the request', function () {
-        this.ClsiManager._buildRequest
-          .calledWith(this.project_id)
-          .should.equal(true)
-      })
-
-      it('should send the request to the CLSI', function () {
-        this.ClsiManager._postToClsi
-          .calledWith(
-            this.project_id,
-            this.user_id,
-            this.request,
-            'e2',
-            'standard'
-          )
-          .should.equal(true)
-      })
-
-      it('should call the callback with the status and output files', function () {
-        const outputFiles = [
+      beforeEach(function (done) {
+        this.outputFiles = [
           {
             url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`,
             path: 'output.pdf',
             type: 'pdf',
             build: 1234,
-            ranges: [],
-            createdAt: new Date(),
-            // gets dropped by JSON.stringify
-            contentId: undefined,
-            size: undefined,
-            startXRefTable: undefined,
           },
           {
             url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`,
@@ -158,1159 +144,855 @@ describe('ClsiManager', function () {
             build: 1234,
           },
         ]
-        this.callback
-          .calledWith(null, this.status, outputFiles)
-          .should.equal(true)
+        this.responseBody.compile.outputFiles = this.outputFiles.map(
+          outputFile => ({ ...outputFile, url: CLSI_URL + outputFile.url })
+        )
+        this.timeout = 100
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          {
+            compileBackendClass: 'e2',
+            compileGroup: 'standard',
+            timeout: this.timeout,
+          },
+          (err, status, outputFiles) => {
+            if (err) {
+              return done(err)
+            }
+            this.result = { status, outputFiles }
+            done()
+          }
+        )
       })
-    })
 
-    describe('with ranges on the pdf and stats/timings details', function () {
-      beforeEach(function () {
-        this.ClsiManager._postToClsi = sinon.stub().yields(
-          null,
-          {
+      it('should send the request to the CLSI', function () {
+        this.request.should.have.been.calledWith({
+          url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`,
+          method: 'POST',
+          json: {
             compile: {
-              status: 'success',
-              stats: { fooStat: 1 },
-              timings: { barTiming: 2 },
-              outputFiles: [
-                {
-                  url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`,
-                  path: 'output.pdf',
-                  type: 'pdf',
-                  build: 1234,
-                  contentId: '123-321',
-                  ranges: [{ start: 1, end: 42, hash: 'foo' }],
-                  startXRefTable: 42,
-                  size: 42,
-                },
-                {
-                  url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`,
-                  path: 'output.log',
-                  type: 'log',
-                  build: 1234,
-                },
-              ],
+              options: sinon.match({
+                compiler: this.project.compiler,
+                imageName: this.project.imageName,
+                timeout: this.timeout,
+                draft: false,
+                check: undefined,
+                syncType: undefined, // "full"
+                syncState: undefined,
+                compileGroup: 'standard',
+                enablePdfCaching: false,
+                pdfCachingMinChunkSize: undefined,
+                flags: undefined,
+                metricsMethod: 'standard',
+                stopOnFirstError: false,
+              }),
+              rootResourcePath: 'main.tex',
+              resources: _makeResources(this.project, this.docs, this.files),
             },
           },
-          'clsi-server-id-43'
+          jar: this.jar,
+        })
+      })
+
+      it('should get the project with the required fields', function () {
+        this.ProjectGetter.getProject.should.have.been.calledWith(
+          this.project._id,
+          {
+            compiler: 1,
+            rootDoc_id: 1,
+            imageName: 1,
+            rootFolder: 1,
+          }
         )
-        this.ClsiCookieManager._getServerId.yields(null, 'clsi-server-id-42')
-        this.ClsiManager.sendRequest(
-          this.project_id,
+      })
+
+      it('should flush the project to the database', function () {
+        this.DocumentUpdaterHandler.flushProjectToMongo.should.have.been.calledWith(
+          this.project._id
+        )
+      })
+
+      it('should get all the docs', function () {
+        this.ProjectEntityHandler.getAllDocs.should.have.been.calledWith(
+          this.project._id
+        )
+      })
+
+      it('should get all the files', function () {
+        this.ProjectEntityHandler.getAllFiles.should.have.been.calledWith(
+          this.project._id
+        )
+      })
+
+      it('should call the callback with the status and output files', function () {
+        expect(this.result.status).to.equal('success')
+        expect(this.result.outputFiles.map(f => f.path)).to.have.members(
+          this.outputFiles.map(f => f.path)
+        )
+      })
+
+      it('should process a request with a cookie jar', function () {
+        expect(this.request).to.have.been.calledWith(
+          sinon.match(opts => opts.jar === this.jar && opts.qs == null)
+        )
+      })
+
+      it('should persist the cookie from the response', function () {
+        expect(this.ClsiCookieManager.setServerId).to.have.been.calledWith(
+          this.project._id,
           this.user_id,
-          { compileBackendClass: 'e2', compileGroup: 'standard' },
-          this.callback
+          'standard',
+          'e2',
+          this.response
         )
       })
+    })
 
-      it('should emit the caching details and stats/timings', function () {
-        const outputFiles = [
+    describe('with ranges on the pdf and stats/timings details', function () {
+      beforeEach(function (done) {
+        this.ranges = [{ start: 1, end: 42, hash: 'foo' }]
+        this.startXRefTable = 123
+        this.size = 456
+        this.contentId = '123-321'
+        this.outputFiles = [
           {
-            url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.pdf`,
+            url: `/project/${this.project._id}/user/${this.user_id}/build/1234/output/output.pdf`,
             path: 'output.pdf',
             type: 'pdf',
             build: 1234,
-            contentId: '123-321',
-            ranges: [{ start: 1, end: 42, hash: 'foo' }],
-            startXRefTable: 42,
-            size: 42,
-            createdAt: new Date(),
+            contentId: this.contentId,
+            ranges: this.ranges,
+            startXRefTable: this.startXRefTable,
+            size: this.size,
           },
           {
-            url: `/project/${this.project_id}/user/${this.user_id}/build/1234/output/output.log`,
+            url: `/project/${this.project._id}/user/${this.user_id}/build/1234/output/output.log`,
             path: 'output.log',
             type: 'log',
             build: 1234,
           },
         ]
-        const validationError = undefined
-        expect(this.callback).to.have.been.calledWith(
+        this.stats = { fooStat: 1 }
+        this.timings = { barTiming: 2 }
+        this.serverId = 'clsi-server-id-42'
+        this.responseBody.compile.outputFiles = this.outputFiles.map(
+          outputFile => ({ ...outputFile, url: CLSI_URL + outputFile.url })
+        )
+        this.responseBody.compile.stats = this.stats
+        this.responseBody.compile.timings = this.timings
+        this.ClsiCookieManager.getCookieJar.yields(
           null,
-          'success',
-          outputFiles,
-          'clsi-server-id-43',
-          validationError,
-          { fooStat: 1 },
-          { barTiming: 2 }
+          this.jar,
+          this.serverId
         )
-      })
-    })
-
-    describe('with a failed compile', function () {
-      beforeEach(function () {
-        this.ClsiManager._postToClsi = sinon.stub().yields(null, {
-          compile: {
-            status: (this.status = 'failure'),
-          },
-        })
         this.ClsiManager.sendRequest(
-          this.project_id,
+          this.project._id,
           this.user_id,
-          {},
-          this.callback
+          { compileBackendClass: 'e2', compileGroup: 'standard' },
+          (
+            err,
+            status,
+            outputFiles,
+            serverId,
+            validationError,
+            stats,
+            timings
+          ) => {
+            if (err) {
+              return done(err)
+            }
+            this.result = {
+              status,
+              outputFiles,
+              serverId,
+              validationError,
+              stats,
+              timings,
+            }
+            done()
+          }
         )
       })
 
-      it('should call the callback with a failure status', function () {
-        this.callback.calledWith(null, this.status).should.equal(true)
+      it('should emit the caching details and stats/timings', function () {
+        expect(this.result.status).to.equal('success')
+        expect(this.result.serverId).to.equal(this.serverId)
+        expect(this.result.validationError).to.be.undefined
+        expect(this.result.stats).to.equal(this.stats)
+        expect(this.result.timings).to.equal(this.timings)
+        const outputPdf = this.result.outputFiles.find(
+          f => f.path === 'output.pdf'
+        )
+        expect(outputPdf.ranges).to.equal(this.ranges)
+        expect(outputPdf.startXRefTable).to.equal(this.startXRefTable)
+        expect(outputPdf.contentId).to.equal(this.contentId)
+        expect(outputPdf.size).to.equal(this.size)
       })
     })
 
-    describe('with a sync conflict', function () {
-      beforeEach(function () {
-        this.ClsiManager.sendRequestOnce = sinon.stub()
-        this.ClsiManager.sendRequestOnce
-          .withArgs(this.project_id, this.user_id, { syncType: 'full' })
-          .callsArgWith(3, null, (this.status = 'success'))
-        this.ClsiManager.sendRequestOnce
-          .withArgs(this.project_id, this.user_id, {})
-          .callsArgWith(3, null, 'conflict')
+    describe('with the incremental compile option', function () {
+      beforeEach(function (done) {
+        const doc = this.docs['/main.tex']
+        this.DocumentUpdaterHandler.getProjectDocsIfMatch.yields(null, [
+          { _id: doc._id, lines: doc.lines, v: 123 },
+        ])
+        this.ProjectEntityHandler.getAllDocPathsFromProject.returns({
+          'mock-doc-id-1': 'main.tex',
+        })
         this.ClsiManager.sendRequest(
-          this.project_id,
+          this.project._id,
           this.user_id,
-          {},
-          this.callback
+          {
+            timeout: 100,
+            incrementalCompilesEnabled: true,
+            compileBackendClass: 'e2',
+            compileGroup: 'priority',
+            enablePdfCaching: true,
+            pdfCachingMinChunkSize: 1337,
+          },
+          (err, status, outputFiles) => {
+            if (err) {
+              return done(err)
+            }
+            this.result = { status, outputFiles }
+            done()
+          }
+        )
+      })
+
+      it('should get the project with the required fields', function () {
+        this.ProjectGetter.getProject.should.have.been.calledWith(
+          this.project._id,
+          {
+            compiler: 1,
+            rootDoc_id: 1,
+            imageName: 1,
+            rootFolder: 1,
+          }
         )
       })
 
-      it('should call the sendRequestOnce method twice', function () {
-        this.ClsiManager.sendRequestOnce.calledTwice.should.equal(true)
+      it('should not explicitly flush the project to the database', function () {
+        this.DocumentUpdaterHandler.flushProjectToMongo.should.not.have.been.calledWith(
+          this.project._id
+        )
       })
 
-      it('should call the sendRequestOnce method with syncType:full', function () {
-        this.ClsiManager.sendRequestOnce
-          .calledWith(this.project_id, this.user_id, { syncType: 'full' })
-          .should.equal(true)
+      it('should get only the live docs from the docupdater with a background flush in docupdater', function () {
+        this.DocumentUpdaterHandler.getProjectDocsIfMatch.should.have.been.calledWith(
+          this.project._id
+        )
       })
 
-      it('should call the sendRequestOnce method without syncType:full', function () {
-        this.ClsiManager.sendRequestOnce
-          .calledWith(this.project_id, this.user_id, {})
-          .should.equal(true)
+      it('should not get any of the files', function () {
+        this.ProjectEntityHandler.getAllFiles.should.not.have.been.called
       })
 
-      it('should call the callback with a success status', function () {
-        this.callback.calledWith(null, this.status).should.equal(true)
+      it('should build up the CLSI request', function () {
+        this.request.should.have.been.calledWith({
+          url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=priority`,
+          method: 'POST',
+          json: {
+            compile: {
+              options: {
+                compiler: this.project.compiler,
+                timeout: 100,
+                imageName: this.project.imageName,
+                draft: false,
+                check: undefined,
+                syncType: 'incremental',
+                syncState: '01234567890abcdef',
+                compileGroup: 'priority',
+                enablePdfCaching: true,
+                pdfCachingMinChunkSize: 1337,
+                flags: undefined,
+                metricsMethod: 'priority',
+                stopOnFirstError: false,
+              },
+              rootResourcePath: 'main.tex',
+              resources: [
+                {
+                  path: 'main.tex',
+                  content: this.docs['/main.tex'].lines.join('\n'),
+                },
+              ],
+            },
+          },
+          jar: this.jar,
+        })
       })
     })
 
-    describe('with an unavailable response', function () {
-      beforeEach(function () {
-        this.ClsiManager.sendRequestOnce = sinon.stub()
-        this.ClsiManager.sendRequestOnce
-          .withArgs(this.project_id, this.user_id, {
-            syncType: 'full',
-            forceNewClsiServer: true,
-          })
-          .callsArgWith(3, null, (this.status = 'success'))
-        this.ClsiManager.sendRequestOnce
-          .withArgs(this.project_id, this.user_id, {})
-          .callsArgWith(3, null, 'unavailable')
+    describe('when the root doc is set and not in the docupdater', function () {
+      beforeEach(function (done) {
+        const doc = this.docs['/main.tex']
+        this.DocumentUpdaterHandler.getProjectDocsIfMatch.yields(null, [
+          { _id: doc._id, lines: doc.lines, v: 123 },
+        ])
+        this.ProjectEntityHandler.getAllDocPathsFromProject.returns({
+          'mock-doc-id-1': 'main.tex',
+          'mock-doc-id-2': '/chapters/chapter1.tex',
+        })
         this.ClsiManager.sendRequest(
-          this.project_id,
+          this.project._id,
           this.user_id,
-          {},
-          this.callback
+          {
+            timeout: 100,
+            incrementalCompilesEnabled: true,
+            rootDoc_id: 'mock-doc-id-2',
+          },
+          done
         )
       })
 
-      it('should call the sendRequestOnce method twice', function () {
-        this.ClsiManager.sendRequestOnce.calledTwice.should.equal(true)
-      })
-
-      it('should call the sendRequestOnce method with forceNewClsiServer:true', function () {
-        this.ClsiManager.sendRequestOnce
-          .calledWith(this.project_id, this.user_id, {
-            forceNewClsiServer: true,
-            syncType: 'full',
-          })
-          .should.equal(true)
+      it('should still change the root path', function () {
+        this.request.should.have.been.calledWith(
+          sinon.match(
+            opts =>
+              opts.json.compile.rootResourcePath === 'chapters/chapter1.tex'
+          )
+        )
       })
+    })
 
-      it('should call the sendRequestOnce method without forceNewClsiServer:true', function () {
-        this.ClsiManager.sendRequestOnce
-          .calledWith(this.project_id, this.user_id, {})
-          .should.equal(true)
+    describe('when root doc override is valid', function () {
+      beforeEach(function (done) {
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          { rootDoc_id: 'mock-doc-id-2' },
+          done
+        )
       })
 
-      it('should call the callback with a success status', function () {
-        this.callback.calledWith(null, this.status).should.equal(true)
+      it('should change root path', function () {
+        this.request.should.have.been.calledWith(
+          sinon.match(
+            opts =>
+              opts.json.compile.rootResourcePath === 'chapters/chapter1.tex'
+          )
+        )
       })
     })
 
-    describe('when the resources fail the precompile check', function () {
-      beforeEach(function () {
-        this.ClsiFormatChecker.checkRecoursesForProblems = sinon
-          .stub()
-          .callsArgWith(1, new Error('failed'))
-        this.ClsiManager._postToClsi = sinon.stub().yields(null, {
-          compile: {
-            status: (this.status = 'failure'),
-          },
-        })
+    describe('when root doc override is invalid', function () {
+      beforeEach(function (done) {
         this.ClsiManager.sendRequest(
-          this.project_id,
+          this.project._id,
           this.user_id,
-          {},
-          this.callback
+          { rootDoc_id: 'invalid-id' },
+          done
         )
       })
 
-      it('should call the callback only once', function () {
-        this.callback.calledOnce.should.equal(true)
-      })
-
-      it('should call the callback with an error', function () {
-        this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error))
+      it('should fallback to default root doc', function () {
+        this.request.should.have.been.calledWith(
+          sinon.match(opts => opts.json.compile.rootResourcePath === 'main.tex')
+        )
       })
     })
-  })
 
-  describe('sendExternalRequest', function () {
-    beforeEach(function () {
-      this.submission_id = 'submission-id'
-      this.clsi_request = 'mock-request'
-      this.ClsiCookieManager._getServerId.callsArgWith(2, null, 'clsi3')
-    })
+    describe('when the project has an invalid compiler', function () {
+      beforeEach(function (done) {
+        this.project.compiler = 'context'
+        this.ClsiManager.sendRequest(this.project._id, this.user_id, {}, done)
+      })
 
-    describe('with a successful compile', function () {
-      beforeEach(function () {
-        this.ClsiManager._postToClsi = sinon.stub().yields(null, {
-          compile: {
-            status: (this.status = 'success'),
-            outputFiles: [
-              {
-                url: `${this.settings.apis.clsi.url}/project/${this.submission_id}/build/1234/output/output.pdf`,
-                path: 'output.pdf',
-                type: 'pdf',
-                build: 1234,
-              },
-              {
-                url: `${this.settings.apis.clsi.url}/project/${this.submission_id}/build/1234/output/output.log`,
-                path: 'output.log',
-                type: 'log',
-                build: 1234,
-              },
-            ],
-          },
-        })
-        this.ClsiManager.sendExternalRequest(
-          this.submission_id,
-          this.clsi_request,
-          { compileBackendClass: 'e2', compileGroup: 'standard' },
-          this.callback
+      it('should set the compiler to pdflatex', function () {
+        expect(this.request).to.have.been.calledWith(
+          sinon.match(opts => opts.json.compile.options.compiler === 'pdflatex')
         )
       })
+    })
 
-      it('should send the request to the CLSI', function () {
-        this.ClsiManager._postToClsi
-          .calledWith(
-            this.submission_id,
-            null,
-            this.clsi_request,
-            'e2',
-            'standard'
-          )
-          .should.equal(true)
+    describe('when there is no valid root document', function () {
+      beforeEach(function (done) {
+        this.project.rootDoc_id = 'not-valid'
+        this.ClsiManager.sendRequest(this.project._id, this.user_id, {}, done)
       })
 
-      it('should call the callback with the status and output files', function () {
-        const outputFiles = [
-          {
-            url: `/project/${this.submission_id}/build/1234/output/output.pdf`,
-            path: 'output.pdf',
-            type: 'pdf',
-            build: 1234,
-            ranges: [],
-            createdAt: new Date(),
-            // gets dropped by JSON.stringify
-            contentId: undefined,
-            size: undefined,
-            startXRefTable: undefined,
-          },
-          {
-            url: `/project/${this.submission_id}/build/1234/output/output.log`,
-            path: 'output.log',
-            type: 'log',
-            build: 1234,
-          },
-        ]
-        this.callback.should.have.been.calledWith(
-          null,
-          this.status,
-          outputFiles
+      it('should set to main.tex', function () {
+        expect(this.request).to.have.been.calledWith(
+          sinon.match(opts => opts.json.compile.rootResourcePath === 'main.tex')
         )
       })
     })
 
-    describe('with a failed compile', function () {
-      beforeEach(function () {
-        this.ClsiManager._postToClsi = sinon.stub().yields(null, {
-          compile: {
-            status: (this.status = 'failure'),
-          },
-        })
-        this.ClsiManager.sendExternalRequest(
-          this.submission_id,
-          this.clsi_request,
+    describe('when there is no valid root document and no main.tex document', function () {
+      beforeEach(function (done) {
+        this.project.rootDoc_id = 'not-valid'
+        this.docs = {
+          '/other.tex': {
+            name: 'other.tex',
+            _id: 'mock-doc-id-1',
+            lines: ['Hello', 'world'],
+          },
+          '/chapters/chapter1.tex': {
+            name: 'chapter1.tex',
+            _id: 'mock-doc-id-2',
+            lines: ['Chapter 1'],
+          },
+        }
+        this.ProjectEntityHandler.getAllDocs.yields(null, this.docs)
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
           {},
-          this.callback
+          (err, status) => {
+            if (err) {
+              return done(err)
+            }
+            this.result = { status }
+            done()
+          }
         )
       })
 
-      it('should call the callback with a failure status', function () {
-        this.callback.calledWith(null, this.status).should.equal(true)
+      it('should report a validation problem', function () {
+        expect(this.result.status).to.equal('validation-problems')
       })
     })
 
-    describe('when the resources fail the precompile check', function () {
-      beforeEach(function () {
-        this.ClsiFormatChecker.checkRecoursesForProblems = sinon
-          .stub()
-          .callsArgWith(1, new Error('failed'))
-        this.ClsiManager._postToClsi = sinon.stub().yields(null, {
-          compile: {
-            status: (this.status = 'failure'),
+    describe('when there is no valid root document and a single document which is not main.tex', function () {
+      beforeEach(function (done) {
+        this.project.rootDoc_id = 'not-valid'
+        this.docs = {
+          '/other.tex': {
+            name: 'other.tex',
+            _id: 'mock-doc-id-1',
+            lines: ['Hello', 'world'],
           },
-        })
-        this.ClsiManager.sendExternalRequest(
-          this.submission_id,
-          this.clsi_request,
-          {},
-          this.callback
-        )
-      })
-
-      it('should call the callback only once', function () {
-        this.callback.calledOnce.should.equal(true)
+        }
+        this.ProjectEntityHandler.getAllDocs.yields(null, this.docs)
+        this.ClsiManager.sendRequest(this.project._id, this.user_id, {}, done)
       })
 
-      it('should call the callback with an error', function () {
-        this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error))
+      it('should set io to the only file', function () {
+        expect(this.request).to.have.been.calledWith(
+          sinon.match(
+            opts => opts.json.compile.rootResourcePath === 'other.tex'
+          )
+        )
       })
     })
-  })
 
-  describe('deleteAuxFiles', function () {
-    beforeEach(function () {
-      this.ClsiManager._makeRequestWithClsiServerId = sinon.stub().yields(null)
-      this.DocumentUpdaterHandler.clearProjectState = sinon.stub().callsArg(1)
-    })
-
-    describe('with the standard compileGroup', function () {
-      beforeEach(function () {
-        this.ClsiManager.deleteAuxFiles(
-          this.project_id,
+    describe('with the draft option', function () {
+      beforeEach(function (done) {
+        this.ClsiManager.sendRequest(
+          this.project._id,
           this.user_id,
-          { compileBackendClass: 'e2', compileGroup: 'standard' },
-          'node-1',
-          this.callback
+          { timeout: 100, draft: true },
+          done
         )
       })
 
-      it('should call the delete method in the standard CLSI', function () {
-        this.ClsiManager._makeRequestWithClsiServerId
-          .calledWith(
-            this.project_id,
-            this.user_id,
-            'standard',
-            'e2',
-            {
-              method: 'DELETE',
-              url: `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}?compileBackendClass=e2&compileGroup=standard`,
-            },
-            'node-1'
-          )
-          .should.equal(true)
-      })
-
-      it('should clear the project state from the docupdater', function () {
-        this.DocumentUpdaterHandler.clearProjectState
-          .calledWith(this.project_id)
-          .should.equal(true)
+      it('should add the draft option into the request', function () {
+        expect(this.request).to.have.been.calledWith(
+          sinon.match(opts => opts.json.compile.options.draft === true)
+        )
       })
+    })
 
-      it('should clear the clsi persistance', function () {
-        this.ClsiCookieManager.clearServerId
-          .calledWith(this.project_id, this.user_id)
-          .should.equal(true)
+    describe('with a failed compile', function () {
+      beforeEach(function (done) {
+        this.responseBody.compile.status = 'failure'
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          {},
+          (err, status) => {
+            if (err) {
+              return done(err)
+            }
+            this.result = { status }
+            done()
+          }
+        )
       })
 
-      it('should call the callback', function () {
-        this.callback.called.should.equal(true)
+      it('should call the callback with a failure status', function () {
+        expect(this.result.status).to.equal('failure')
       })
     })
-  })
-
-  describe('_buildRequest', function () {
-    beforeEach(function () {
-      this.project = {
-        _id: this.project_id,
-        compiler: (this.compiler = 'latex'),
-        rootDoc_id: 'mock-doc-id-1',
-        imageName: (this.image = 'mock-image-name'),
-      }
-
-      this.docs = {
-        '/main.tex': (this.doc_1 = {
-          name: 'main.tex',
-          _id: 'mock-doc-id-1',
-          lines: ['Hello', 'world'],
-        }),
-        '/chapters/chapter1.tex': (this.doc_2 = {
-          name: 'chapter1.tex',
-          _id: 'mock-doc-id-2',
-          lines: ['Chapter 1'],
-        }),
-      }
 
-      this.files = {
-        '/images/image.png': (this.file_1 = {
-          name: 'image.png',
-          _id: 'mock-file-id-1',
-          created: new Date(),
-        }),
-      }
-
-      this.Project.findById = sinon.stub().callsArgWith(2, null, this.project)
-      this.ProjectEntityHandler.getAllDocs = sinon
-        .stub()
-        .callsArgWith(1, null, this.docs)
-      this.ProjectEntityHandler.getAllFiles = sinon
-        .stub()
-        .callsArgWith(1, null, this.files)
-      this.ProjectGetter.getProject = sinon
-        .stub()
-        .callsArgWith(2, null, this.project)
-      this.DocumentUpdaterHandler.flushProjectToMongo = sinon
-        .stub()
-        .callsArgWith(1, null)
-    })
-
-    describe('with a valid project', function () {
+    describe('with a sync conflict', function () {
       beforeEach(function (done) {
-        this.ClsiManager._buildRequest(
-          this.project_id,
-          { timeout: 100, compileBackendClass: 'e2', compileGroup: 'standard' },
-          (err, request) => {
-            if (err != null) {
+        this.request
+          .withArgs(
+            sinon.match(opts => opts.json.compile.options.syncType !== 'full')
+          )
+          .yields(null, this.response, {
+            compile: { status: 'conflict' },
+          })
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          {},
+          (err, status) => {
+            if (err) {
               return done(err)
             }
-            this.request = request
+            this.result = { status }
             done()
           }
         )
       })
 
-      it('should get the project with the required fields', function () {
-        this.ProjectGetter.getProject
-          .calledWith(this.project_id, {
-            compiler: 1,
-            rootDoc_id: 1,
-            imageName: 1,
-            rootFolder: 1,
-          })
-          .should.equal(true)
+      it('should send two requests to CLSI', function () {
+        this.request.should.have.been.calledTwice
       })
 
-      it('should flush the project to the database', function () {
-        this.DocumentUpdaterHandler.flushProjectToMongo
-          .calledWith(this.project_id)
-          .should.equal(true)
+      it('should call the CLSI first without syncType:full', function () {
+        const compileOptions =
+          this.request.getCall(0).args[0].json.compile.options
+        expect(compileOptions.syncType).to.be.undefined
       })
 
-      it('should get all the docs', function () {
-        this.ProjectEntityHandler.getAllDocs
-          .calledWith(this.project_id)
-          .should.equal(true)
+      it('should call the CLSI a second time with syncType:full', function () {
+        const compileOptions =
+          this.request.getCall(1).args[0].json.compile.options
+        expect(compileOptions.syncType).to.equal('full')
       })
 
-      it('should get all the files', function () {
-        this.ProjectEntityHandler.getAllFiles
-          .calledWith(this.project_id)
-          .should.equal(true)
-      })
-
-      it('should build up the CLSI request', function () {
-        expect(this.request).to.deep.equal({
-          compile: {
-            options: {
-              compiler: this.compiler,
-              timeout: 100,
-              imageName: this.image,
-              draft: false,
-              check: undefined,
-              syncType: undefined, // "full"
-              syncState: undefined,
-              compileGroup: 'standard',
-              enablePdfCaching: false,
-              pdfCachingMinChunkSize: undefined,
-              flags: undefined,
-              metricsMethod: 'standard',
-              stopOnFirstError: false,
-            }, // "01234567890abcdef"
-            rootResourcePath: 'main.tex',
-            resources: [
-              {
-                path: 'main.tex',
-                content: this.doc_1.lines.join('\n'),
-              },
-              {
-                path: 'chapters/chapter1.tex',
-                content: this.doc_2.lines.join('\n'),
-              },
-              {
-                path: 'images/image.png',
-                url: `${this.settings.apis.filestore.url}/project/${this.project_id}/file/${this.file_1._id}`,
-                modified: this.file_1.created.getTime(),
-              },
-            ],
-          },
-        })
+      it('should call the callback with a success status', function () {
+        this.result.status.should.equal('success')
       })
     })
 
-    describe('with the incremental compile option', function () {
+    describe('with an unavailable response', function () {
       beforeEach(function (done) {
-        this.project_state_hash = '01234567890abcdef'
-        this.ClsiStateManager.computeHash = sinon
-          .stub()
-          .returns(this.project_state_hash)
-        this.DocumentUpdaterHandler.getProjectDocsIfMatch = sinon
-          .stub()
-          .callsArgWith(2, null, [
-            { _id: this.doc_1._id, lines: this.doc_1.lines, v: 123 },
-          ])
-        this.ProjectEntityHandler.getAllDocPathsFromProject = sinon
-          .stub()
-          .returns({ 'mock-doc-id-1': 'main.tex' })
-        this.ClsiManager._buildRequest(
-          this.project_id,
-          {
-            timeout: 100,
-            incrementalCompilesEnabled: true,
-            compileGroup: 'priority',
-            enablePdfCaching: true,
-            pdfCachingMinChunkSize: 1337,
-          },
-          (err, request) => {
-            if (err != null) {
+        this.request.onCall(0).yields(null, this.response, {
+          compile: { status: 'unavailable' },
+        })
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          {},
+          (err, status) => {
+            if (err) {
               return done(err)
             }
-            this.request = request
+            this.result = { status }
             done()
           }
         )
       })
 
-      it('should get the project with the required fields', function () {
-        this.ProjectGetter.getProject
-          .calledWith(this.project_id, {
-            compiler: 1,
-            rootDoc_id: 1,
-            imageName: 1,
-            rootFolder: 1,
-          })
-          .should.equal(true)
+      it('should send two requests to CLSI', function () {
+        this.request.should.have.been.calledTwice
       })
 
-      it('should not explicitly flush the project to the database', function () {
-        this.DocumentUpdaterHandler.flushProjectToMongo
-          .calledWith(this.project_id)
-          .should.equal(false)
+      it('should call the CLSI first without syncType:full', function () {
+        const compileOptions =
+          this.request.getCall(0).args[0].json.compile.options
+        expect(compileOptions.syncType).to.be.undefined
       })
 
-      it('should get only the live docs from the docupdater with a background flush in docupdater', function () {
-        this.DocumentUpdaterHandler.getProjectDocsIfMatch
-          .calledWith(this.project_id)
-          .should.equal(true)
+      it('should call the CLSI a second time with syncType:full', function () {
+        const compileOptions =
+          this.request.getCall(1).args[0].json.compile.options
+        expect(compileOptions.syncType).to.equal('full')
       })
 
-      it('should not get any of the files', function () {
-        this.ProjectEntityHandler.getAllFiles.called.should.equal(false)
+      it('should clear the CLSI server id cookie', function () {
+        expect(this.ClsiCookieManager.clearServerId).to.have.been.calledWith(
+          this.project._id,
+          this.user_id
+        )
       })
 
-      it('should build up the CLSI request', function () {
-        expect(this.request).to.deep.equal({
-          compile: {
-            options: {
-              compiler: this.compiler,
-              timeout: 100,
-              imageName: this.image,
-              draft: false,
-              check: undefined,
-              syncType: 'incremental',
-              syncState: '01234567890abcdef',
-              compileGroup: 'priority',
-              enablePdfCaching: true,
-              pdfCachingMinChunkSize: 1337,
-              flags: undefined,
-              metricsMethod: 'priority',
-              stopOnFirstError: false,
-            },
-            rootResourcePath: 'main.tex',
-            resources: [
-              {
-                path: 'main.tex',
-                content: this.doc_1.lines.join('\n'),
-              },
-            ],
-          },
-        })
+      it('should call the callback with a success status', function () {
+        expect(this.result.status).to.equal('success')
       })
+    })
 
-      describe('when the root doc is set and not in the docupdater', function () {
-        beforeEach(function (done) {
-          this.project_state_hash = '01234567890abcdef'
-          this.ClsiStateManager.computeHash = sinon
-            .stub()
-            .returns(this.project_state_hash)
-          this.DocumentUpdaterHandler.getProjectDocsIfMatch = sinon
-            .stub()
-            .callsArgWith(2, null, [
-              { _id: this.doc_1._id, lines: this.doc_1.lines, v: 123 },
-            ])
-          this.ProjectEntityHandler.getAllDocPathsFromProject = sinon
-            .stub()
-            .returns({
-              'mock-doc-id-1': 'main.tex',
-              'mock-doc-id-2': '/chapters/chapter1.tex',
-            })
-          this.ClsiManager._buildRequest(
-            this.project_id,
-            {
-              timeout: 100,
-              incrementalCompilesEnabled: true,
-              rootDoc_id: 'mock-doc-id-2',
-            },
-            (err, request) => {
-              if (err != null) {
-                return done(err)
-              }
-              this.request = request
-              done()
-            }
-          )
-        })
+    describe('when the resources fail the precompile check', function () {
+      beforeEach(function () {
+        this.ClsiFormatChecker.checkRecoursesForProblems.yields(
+          new Error('failed')
+        )
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          {},
+          this.callback
+        )
+      })
 
-        it('should still change the root path', function () {
-          this.request.compile.rootResourcePath.should.equal(
-            'chapters/chapter1.tex'
-          )
-        })
+      it('should call the callback only once', function () {
+        this.callback.calledOnce.should.equal(true)
+      })
+
+      it('should call the callback with an error', function () {
+        this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error))
       })
     })
 
-    describe('when root doc override is valid', function () {
+    describe('when a new backend is configured', function () {
       beforeEach(function (done) {
-        this.ClsiManager._buildRequest(
-          this.project_id,
-          { rootDoc_id: 'mock-doc-id-2' },
-          (err, request) => {
-            if (err != null) {
+        this.settings.apis.clsi_new = { url: 'https://compiles.somewhere.test' }
+        this.ClsiManager.sendRequest(
+          this.project._id,
+          this.user_id,
+          { compileBackendClass: 'e2', compileGroup: 'standard' },
+          err => {
+            if (err) {
               return done(err)
             }
-            this.request = request
-            done()
+            // wait for the background task to finish
+            setTimeout(done, 0)
           }
         )
       })
 
-      it('should change root path', function () {
-        this.request.compile.rootResourcePath.should.equal(
-          'chapters/chapter1.tex'
+      it('makes a request to the new backend', function () {
+        expect(this.request).to.have.been.calledTwice
+        expect(this.request).to.have.been.calledWith(
+          sinon.match({
+            url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`,
+          })
+        )
+        expect(this.request).to.have.been.calledWith(
+          sinon.match({
+            url: `${this.settings.apis.clsi_new.url}/project/${this.project._id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`,
+          })
         )
       })
     })
+  })
 
-    describe('when root doc override is invalid', function () {
+  describe('sendExternalRequest', function () {
+    beforeEach(function () {
+      this.submissionId = 'submission-id'
+      this.clsiRequest = 'mock-request'
+      this.ClsiCookieManager.getCookieJar.yields(null, this.jar, 'clsi3')
+    })
+
+    describe('with a successful compile', function () {
       beforeEach(function (done) {
-        this.ClsiManager._buildRequest(
-          this.project_id,
-          { rootDoc_id: 'invalid-id' },
-          (err, request) => {
-            if (err != null) {
+        this.outputFiles = [
+          {
+            url: `/project/${this.submissionId}/build/1234/output/output.pdf`,
+            path: 'output.pdf',
+            type: 'pdf',
+            build: 1234,
+          },
+          {
+            url: `/project/${this.submissionId}/build/1234/output/output.log`,
+            path: 'output.log',
+            type: 'log',
+            build: 1234,
+          },
+        ]
+        this.responseBody.compile.outputFiles = this.outputFiles.map(
+          outputFile => ({ ...outputFile, url: CLSI_URL + outputFile.url })
+        )
+        this.ClsiManager.sendExternalRequest(
+          this.submissionId,
+          this.clsiRequest,
+          { compileBackendClass: 'e2', compileGroup: 'standard' },
+          (err, status, outputFiles) => {
+            if (err) {
               return done(err)
             }
-            this.request = request
+            this.result = { status, outputFiles }
             done()
           }
         )
       })
 
-      it('should fallback to default root doc', function () {
-        this.request.compile.rootResourcePath.should.equal('main.tex')
-      })
-    })
-
-    describe('when the project has an invalid compiler', function () {
-      beforeEach(function (done) {
-        this.project.compiler = 'context'
-        this.ClsiManager._buildRequest(this.project, null, (err, request) => {
-          if (err != null) {
-            return done(err)
-          }
-          this.request = request
-          done()
+      it('should send the request to the CLSI', function () {
+        this.request.should.have.been.calledWith({
+          url: `${CLSI_URL}/project/${this.submissionId}/compile?compileBackendClass=e2&compileGroup=standard`,
+          method: 'POST',
+          json: this.clsiRequest,
+          jar: this.jar,
         })
       })
 
-      it('should set the compiler to pdflatex', function () {
-        this.request.compile.options.compiler.should.equal('pdflatex')
+      it('should call the callback with the status and output files', function () {
+        expect(this.result.status).to.equal('success')
+        expect(this.result.outputFiles.map(f => f.path)).to.have.members(
+          this.outputFiles.map(f => f.path)
+        )
       })
     })
 
-    describe('when there is no valid root document', function () {
+    describe('with a failed compile', function () {
       beforeEach(function (done) {
-        this.project.rootDoc_id = 'not-valid'
-        this.ClsiManager._buildRequest(this.project, null, (error, request) => {
-          this.error = error
-          this.request = request
-          done()
-        })
-      })
-
-      it('should set to main.tex', function () {
-        this.request.compile.rootResourcePath.should.equal('main.tex')
-      })
-    })
-
-    describe('when there is no valid root document and no main.tex document', function () {
-      beforeEach(function () {
-        this.project.rootDoc_id = 'not-valid'
-        this.docs = {
-          '/other.tex': (this.doc_1 = {
-            name: 'other.tex',
-            _id: 'mock-doc-id-1',
-            lines: ['Hello', 'world'],
-          }),
-          '/chapters/chapter1.tex': (this.doc_2 = {
-            name: 'chapter1.tex',
-            _id: 'mock-doc-id-2',
-            lines: ['Chapter 1'],
-          }),
-        }
-        this.ProjectEntityHandler.getAllDocs = sinon
-          .stub()
-          .callsArgWith(1, null, this.docs)
-        this.ClsiManager._buildRequest(this.project, null, this.callback)
+        this.responseBody.compile.status = 'failure'
+        this.ClsiManager.sendExternalRequest(
+          this.submissionId,
+          this.clsiRequest,
+          {},
+          (err, status) => {
+            if (err) {
+              return done(err)
+            }
+            this.result = { status }
+            done()
+          }
+        )
       })
 
-      it('should report an error', function () {
-        this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error))
+      it('should call the callback with a failure status', function () {
+        expect(this.result.status).to.equal('failure')
       })
     })
 
-    describe('when there is no valid root document and a single document which is not main.tex', function () {
+    describe('when the resources fail the precompile check', function () {
       beforeEach(function (done) {
-        this.project.rootDoc_id = 'not-valid'
-        this.docs = {
-          '/other.tex': (this.doc_1 = {
-            name: 'other.tex',
-            _id: 'mock-doc-id-1',
-            lines: ['Hello', 'world'],
-          }),
-        }
-        this.ProjectEntityHandler.getAllDocs = sinon
-          .stub()
-          .callsArgWith(1, null, this.docs)
-        this.ClsiManager._buildRequest(this.project, null, (error, request) => {
-          this.error = error
-          this.request = request
-          done()
-        })
-      })
-
-      it('should set io to the only file', function () {
-        this.request.compile.rootResourcePath.should.equal('other.tex')
-      })
-    })
-
-    describe('with the draft option', function () {
-      it('should add the draft option into the request', function (done) {
-        this.ClsiManager._buildRequest(
-          this.project_id,
-          { timeout: 100, draft: true },
-          (err, request) => {
-            if (err != null) {
-              return done(err)
-            }
-            request.compile.options.draft.should.equal(true)
+        this.ClsiFormatChecker.checkRecoursesForProblems.yields(
+          new Error('failed')
+        )
+        this.responseBody.compile.status = 'failure'
+        this.ClsiManager.sendExternalRequest(
+          this.submissionId,
+          this.clsiRequest,
+          {},
+          err => {
+            this.err = err
             done()
           }
         )
       })
-    })
-  })
 
-  describe('_postToClsi', function () {
-    beforeEach(function () {
-      this.req = { mock: 'req', compile: {} }
+      it('should call the callback with an error', function () {
+        expect(this.err).to.be.instanceof(Error)
+      })
     })
+  })
 
-    describe('successfully', function () {
-      beforeEach(function () {
-        this.ClsiManager._makeRequest = sinon
-          .stub()
-          .yields(null, { statusCode: 204 }, (this.body = { mock: 'foo' }))
-        this.ClsiManager._postToClsi(
-          this.project_id,
+  describe('deleteAuxFiles', function () {
+    describe('with the standard compileGroup', function () {
+      beforeEach(function (done) {
+        this.ClsiManager.deleteAuxFiles(
+          this.project._id,
           this.user_id,
-          this.req,
-          'e2',
-          'standard',
-          this.callback
+          { compileBackendClass: 'e2', compileGroup: 'standard' },
+          'node-1',
+          done
         )
       })
 
-      it('should send the request to the CLSI', function () {
-        const url = `${this.settings.apis.clsi.url}/project/${this.project_id}/user/${this.user_id}/compile?compileBackendClass=e2&compileGroup=standard`
-        this.ClsiManager._makeRequest
-          .calledWith(this.project_id, this.user_id, 'standard', 'e2', {
-            method: 'POST',
-            url,
-            json: this.req,
-          })
+      it('should call the delete method in the standard CLSI', function () {
+        this.request.should.have.been.calledWith({
+          url: `${CLSI_URL}/project/${this.project._id}/user/${this.user_id}?compileBackendClass=e2&compileGroup=standard`,
+          method: 'DELETE',
+          qs: {
+            compileGroup: 'standard',
+            compileBackendClass: 'e2',
+            clsiserverid: 'node-1',
+          },
+        })
+      })
+
+      it('should clear the project state from the docupdater', function () {
+        this.DocumentUpdaterHandler.clearProjectState
+          .calledWith(this.project._id)
           .should.equal(true)
       })
 
-      it('should call the callback with the body and no error', function () {
-        this.callback.calledWith(null, this.body).should.equal(true)
+      it('should clear the clsi persistance', function () {
+        this.ClsiCookieManager.clearServerId
+          .calledWith(this.project._id, this.user_id)
+          .should.equal(true)
       })
-    })
 
-    describe('when the CLSI returns an error', function () {
-      beforeEach(function () {
-        this.ClsiManager._makeRequest = sinon
-          .stub()
-          .yields(null, { statusCode: 500 }, (this.body = { mock: 'foo' }))
-        this.ClsiManager._postToClsi(
-          this.project_id,
-          this.user_id,
-          this.req,
-          'e2',
-          'standard',
-          this.callback
+      it('should not add a cookie jar', function () {
+        expect(this.request).to.have.been.calledWith(
+          sinon.match(opts => opts.jar == null)
         )
       })
 
-      it('should call the callback with an error', function () {
-        this.callback.should.have.been.calledWith(sinon.match.instanceOf(Error))
+      it('should not persist a cookie on response', function () {
+        expect(this.ClsiCookieManager.setServerId).not.to.have.been.called
       })
     })
   })
 
   describe('wordCount', function () {
-    beforeEach(function () {
-      this.ClsiManager._makeRequestWithClsiServerId = sinon
-        .stub()
-        .yields(null, { statusCode: 200 }, (this.body = { mock: 'foo' }))
-      this.ClsiManager._buildRequest = sinon.stub().yields(
-        null,
-        (this.req = {
-          compile: { rootResourcePath: 'rootfile.text', options: {} },
-        })
-      )
-    })
-
     describe('with root file', function () {
-      beforeEach(function () {
+      beforeEach(function (done) {
         this.ClsiManager.wordCount(
-          this.project_id,
+          this.project._id,
           this.user_id,
           false,
           { compileBackendClass: 'e2', compileGroup: 'standard' },
           'node-1',
-          this.callback
+          done
         )
       })
 
       it('should call wordCount with root file', function () {
-        this.ClsiManager._makeRequestWithClsiServerId
-          .calledWith(
-            this.project_id,
-            this.user_id,
-            'standard',
-            'e2',
-            {
-              method: 'GET',
-              url: `http://clsi.example.com/project/${this.project_id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`,
-              qs: {
-                file: 'rootfile.text',
-                image: undefined,
-              },
-              json: true,
-            },
-            'node-1'
-          )
-          .should.equal(true)
+        expect(this.request).to.have.been.calledWith({
+          url: `http://clsi.example.com/project/${this.project._id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`,
+          method: 'GET',
+          qs: {
+            compileGroup: 'standard',
+            compileBackendClass: 'e2',
+            clsiserverid: 'node-1',
+            file: 'main.tex',
+            image: 'mock-image-name',
+          },
+          json: true,
+        })
       })
 
-      it('should call the callback', function () {
-        this.callback.called.should.equal(true)
+      it('should not persist a cookie on response', function () {
+        expect(this.ClsiCookieManager.setServerId).not.to.have.been.called
       })
     })
 
     describe('with param file', function () {
-      beforeEach(function () {
+      beforeEach(function (done) {
         this.ClsiManager.wordCount(
-          this.project_id,
+          this.project._id,
           this.user_id,
-          'main.tex',
+          'other.tex',
           { compileBackendClass: 'e2', compileGroup: 'standard' },
           'node-2',
-          this.callback
+          done
         )
       })
 
       it('should call wordCount with param file', function () {
-        this.ClsiManager._makeRequestWithClsiServerId
-          .calledWith(
-            this.project_id,
-            this.user_id,
-            'standard',
-            'e2',
-            {
-              method: 'GET',
-              url: `http://clsi.example.com/project/${this.project_id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`,
-              qs: { file: 'main.tex', image: undefined },
-              json: true,
-            },
-            'node-2'
-          )
-          .should.equal(true)
-      })
-    })
-
-    describe('with image', function () {
-      beforeEach(function () {
-        this.req.compile.options.imageName = this.image =
-          'example.com/mock/image'
-        this.ClsiManager.wordCount(
-          this.project_id,
-          this.user_id,
-          'main.tex',
-          { compileBackendClass: 'e2', compileGroup: 'standard' },
-          'node-3',
-          this.callback
-        )
-      })
-
-      it('should call wordCount with file and image', function () {
-        this.ClsiManager._makeRequestWithClsiServerId
-          .calledWith(
-            this.project_id,
-            this.user_id,
-            'standard',
-            'e2',
-            {
-              method: 'GET',
-              url: `http://clsi.example.com/project/${this.project_id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`,
-              qs: { file: 'main.tex', image: this.image },
-              json: true,
-            },
-            'node-3'
-          )
-          .should.equal(true)
-      })
-    })
-  })
-
-  describe('_makeRequest', function () {
-    beforeEach(function () {
-      this.response = { there: 'something' }
-      this.request.callsArgWith(1, null, this.response)
-      this.opts = {
-        method: 'SOMETHIGN',
-        url: 'http://a place on the web',
-      }
-    })
-
-    it('should process a request with a cookie jar', function (done) {
-      this.ClsiManager._makeRequest(
-        this.project_id,
-        this.user_id,
-        'standard',
-        'e2',
-        this.opts,
-        () => {
-          const args = this.request.args[0]
-          args[0].method.should.equal(this.opts.method)
-          args[0].url.should.equal(this.opts.url)
-          args[0].jar.should.equal(this.jar)
-          done()
-        }
-      )
-    })
-
-    it('should set the cookie again on response as it might have changed', function (done) {
-      this.ClsiManager._makeRequest(
-        this.project_id,
-        this.user_id,
-        'standard',
-        'e2',
-        this.opts,
-        () => {
-          this.ClsiCookieManager.setServerId
-            .calledWith(
-              this.project_id,
-              this.user_id,
-              'standard',
-              'e2',
-              this.response
-            )
-            .should.equal(true)
-          done()
-        }
-      )
-    })
-  })
-
-  describe('_makeRequestWithClsiServerId', function () {
-    beforeEach(function () {
-      this.response = { statusCode: 200 }
-      this.request.yields(null, this.response)
-      this.opts = {
-        method: 'GET',
-        url: 'http://clsi',
-      }
-    })
-
-    describe('with a regular request', function () {
-      it('should process a request with a cookie jar', function (done) {
-        this.ClsiManager._makeRequestWithClsiServerId(
-          this.project_id,
-          this.user_id,
-          'standard',
-          'e2',
-          this.opts,
-          undefined,
-          err => {
-            if (err) return done(err)
-            const args = this.request.args[0]
-            args[0].method.should.equal(this.opts.method)
-            args[0].url.should.equal(this.opts.url)
-            args[0].jar.should.equal(this.jar)
-            expect(args[0].qs).to.not.exist
-            done()
-          }
-        )
-      })
-
-      it('should persist the cookie from the response', function (done) {
-        this.ClsiManager._makeRequestWithClsiServerId(
-          this.project_id,
-          this.user_id,
-          'standard',
-          'e2',
-          this.opts,
-          undefined,
-          err => {
-            if (err) return done(err)
-            this.ClsiCookieManager.setServerId
-              .calledWith(
-                this.project_id,
-                this.user_id,
-                'standard',
-                'e2',
-                this.response
-              )
-              .should.equal(true)
-            done()
-          }
-        )
-      })
-    })
-
-    describe('with a persistent request', function () {
-      it('should not add a cookie jar', function (done) {
-        this.ClsiManager._makeRequestWithClsiServerId(
-          this.project_id,
-          this.user_id,
-          'standard',
-          'e2',
-          this.opts,
-          'node-1',
-          err => {
-            if (err) return done(err)
-            const requestOpts = this.request.args[0][0]
-            expect(requestOpts.method).to.equal(this.opts.method)
-            expect(requestOpts.url).to.equal(this.opts.url)
-            expect(requestOpts.jar).to.not.exist
-            expect(requestOpts.qs).to.deep.equal({
-              clsiserverid: 'node-1',
-              compileGroup: 'standard',
-              compileBackendClass: 'e2',
-            })
-            done()
-          }
-        )
+        expect(this.request).to.have.been.calledWith({
+          url: `http://clsi.example.com/project/${this.project._id}/user/${this.user_id}/wordcount?compileBackendClass=e2&compileGroup=standard`,
+          method: 'GET',
+          qs: {
+            compileGroup: 'standard',
+            compileBackendClass: 'e2',
+            clsiserverid: 'node-2',
+            file: 'other.tex',
+            image: 'mock-image-name',
+          },
+          json: true,
+        })
       })
 
-      it('should not persist a cookie on response', function (done) {
-        this.ClsiManager._makeRequestWithClsiServerId(
-          this.project_id,
-          this.user_id,
-          'standard',
-          'e2',
-          this.opts,
-          'node-1',
-          err => {
-            if (err) return done(err)
-            expect(this.ClsiCookieManager.setServerId.called).to.equal(false)
-            done()
-          }
-        )
+      it('should not persist a cookie on response', function () {
+        expect(this.ClsiCookieManager.setServerId).not.to.have.been.called
       })
     })
   })
-
-  describe('_makeGoogleCloudRequest', function () {
-    beforeEach(function () {
-      this.settings.apis.clsi_new = { url: 'https://compiles.somewhere.test' }
-      this.response = { there: 'something' }
-      this.request.callsArgWith(1, null, this.response)
-      this.opts = {
-        url: this.ClsiManager._getCompilerUrl(
-          'e2',
-          'standard',
-          this.project_id
-        ),
-      }
-    })
-
-    it('should change the domain on the url', function (done) {
-      this.ClsiManager._makeNewBackendRequest(
-        this.project_id,
-        this.user_id,
-        'standard',
-        'e2',
-        this.opts,
-        () => {
-          const args = this.request.args[0]
-          args[0].url.should.equal(
-            `https://compiles.somewhere.test/project/${this.project_id}?compileBackendClass=e2&compileGroup=standard`
-          )
-          done()
-        }
-      )
-    })
-
-    it('should not make a request if there is not clsi_new url', function (done) {
-      this.settings.apis.clsi_new = undefined
-      this.ClsiManager._makeNewBackendRequest(
-        this.project_id,
-        this.user_id,
-        'standard',
-        'e2',
-        this.opts,
-        err => {
-          expect(err).to.equal(undefined)
-          this.request.callCount.should.equal(0)
-          done()
-        }
-      )
-    })
-  })
 })
+
+function _makeResources(project, docs, files) {
+  const resources = []
+  for (const [path, doc] of Object.entries(docs)) {
+    resources.push({
+      path: path.replace(/^\//, ''),
+      content: doc.lines.join('\n'),
+    })
+  }
+  for (const [path, file] of Object.entries(files)) {
+    resources.push({
+      path: path.replace(/^\//, ''),
+      url: `${FILESTORE_URL}/project/${project._id}/file/${file._id}`,
+      modified: file.created.getTime(),
+    })
+  }
+  return resources
+}