Overleaf/services/clsi/app/js/CompileManager.js

/* eslint-disable
    camelcase,
    handle-callback-err,
    no-return-assign,
    no-undef,
    no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
 * decaffeinate suggestions:
 * DS101: Remove unnecessary use of Array.from
 * DS102: Remove unnecessary code created because of implicit returns
 * DS103: Rewrite code to no longer use __guard__
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
let CompileManager
const ResourceWriter = require('./ResourceWriter')
const LatexRunner = require('./LatexRunner')
const OutputFileFinder = require('./OutputFileFinder')
const OutputCacheManager = require('./OutputCacheManager')
const Settings = require('@overleaf/settings')
const Path = require('path')
const logger = require('logger-sharelatex')
const Metrics = require('./Metrics')
const child_process = require('child_process')
const DraftModeManager = require('./DraftModeManager')
const TikzManager = require('./TikzManager')
const LockManager = require('./LockManager')
const fs = require('fs')
const fse = require('fs-extra')
const os = require('os')
const async = require('async')
const Errors = require('./Errors')
const CommandRunner = require('./CommandRunner')
const { emitPdfStats } = require('./ContentCacheMetrics')

const getCompileName = function (project_id, user_id) {
  if (user_id != null) {
    return `${project_id}-${user_id}`
  } else {
    return project_id
  }
}

const getCompileDir = (project_id, user_id) =>
  Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))

const getOutputDir = (project_id, user_id) =>
  Path.join(Settings.path.outputDir, getCompileName(project_id, user_id))

module.exports = CompileManager = {
  doCompileWithLock(request, callback) {
    if (callback == null) {
      callback = function (error, outputFiles) {}
    }
    const compileDir = getCompileDir(request.project_id, request.user_id)
    const lockFile = Path.join(compileDir, '.project-lock')
    // use a .project-lock file in the compile directory to prevent
    // simultaneous compiles
    return fse.ensureDir(compileDir, function (error) {
      if (error != null) {
        return callback(error)
      }
      return LockManager.runWithLock(
        lockFile,
        releaseLock => CompileManager.doCompile(request, releaseLock),
        callback
      )
    })
  },

  doCompile(request, callback) {
    if (callback == null) {
      callback = function (error, outputFiles) {}
    }
    const compileDir = getCompileDir(request.project_id, request.user_id)
    const outputDir = getOutputDir(request.project_id, request.user_id)

    const timerE2E = new Metrics.Timer('compile-e2e')
    let timer = new Metrics.Timer('write-to-disk')
    logger.log(
      { project_id: request.project_id, user_id: request.user_id },
      'syncing resources to disk'
    )
    return ResourceWriter.syncResourcesToDisk(
      request,
      compileDir,
      function (error, resourceList) {
        // NOTE: resourceList is insecure, it should only be used to exclude files from the output list
        if (error != null && error instanceof Errors.FilesOutOfSyncError) {
          logger.warn(
            { project_id: request.project_id, user_id: request.user_id },
            'files out of sync, please retry'
          )
          return callback(error)
        } else if (error != null) {
          logger.err(
            {
              err: error,
              project_id: request.project_id,
              user_id: request.user_id,
            },
            'error writing resources to disk'
          )
          return callback(error)
        }
        logger.log(
          {
            project_id: request.project_id,
            user_id: request.user_id,
            time_taken: Date.now() - timer.start,
          },
          'written files to disk'
        )
        const syncStage = timer.done()

        const injectDraftModeIfRequired = function (callback) {
          if (request.draft) {
            return DraftModeManager.injectDraftMode(
              Path.join(compileDir, request.rootResourcePath),
              callback
            )
          } else {
            return callback()
          }
        }

        const createTikzFileIfRequired = callback =>
          TikzManager.checkMainFile(
            compileDir,
            request.rootResourcePath,
            resourceList,
            function (error, needsMainFile) {
              if (error != null) {
                return callback(error)
              }
              if (needsMainFile) {
                return TikzManager.injectOutputFile(
                  compileDir,
                  request.rootResourcePath,
                  callback
                )
              } else {
                return callback()
              }
            }
          )
        // set up environment variables for chktex
        const env = {}
        if (Settings.texliveOpenoutAny && Settings.texliveOpenoutAny !== '') {
          // override default texlive openout_any environment variable
          env.openout_any = Settings.texliveOpenoutAny
        }
        // only run chktex on LaTeX files (not knitr .Rtex files or any others)
        const isLaTeXFile =
          request.rootResourcePath != null
            ? request.rootResourcePath.match(/\.tex$/i)
            : undefined
        if (request.check != null && isLaTeXFile) {
          env.CHKTEX_OPTIONS = '-nall -e9 -e10 -w15 -w16'
          env.CHKTEX_ULIMIT_OPTIONS = '-t 5 -v 64000'
          if (request.check === 'error') {
            env.CHKTEX_EXIT_ON_ERROR = 1
          }
          if (request.check === 'validate') {
            env.CHKTEX_VALIDATE = 1
          }
        }

        // apply a series of file modifications/creations for draft mode and tikz
        return async.series(
          [injectDraftModeIfRequired, createTikzFileIfRequired],
          function (error) {
            if (error != null) {
              return callback(error)
            }
            timer = new Metrics.Timer('run-compile')
            // find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
            let tag =
              __guard__(
                __guard__(
                  request.imageName != null
                    ? request.imageName.match(/:(.*)/)
                    : undefined,
                  x1 => x1[1]
                ),
                x => x.replace(/\./g, '-')
              ) || 'default'
            if (!request.project_id.match(/^[0-9a-f]{24}$/)) {
              tag = 'other'
            } // exclude smoke test
            Metrics.inc('compiles')
            Metrics.inc(`compiles-with-image.${tag}`)
            const compileName = getCompileName(
              request.project_id,
              request.user_id
            )
            return LatexRunner.runLatex(
              compileName,
              {
                directory: compileDir,
                mainFile: request.rootResourcePath,
                compiler: request.compiler,
                timeout: request.timeout,
                image: request.imageName,
                flags: request.flags,
                environment: env,
                compileGroup: request.compileGroup,
              },
              function (error, output, stats, timings) {
                // request was for validation only
                let metric_key, metric_value
                if (request.check === 'validate') {
                  const result = (error != null ? error.code : undefined)
                    ? 'fail'
                    : 'pass'
                  error = new Error('validation')
                  error.validate = result
                }
                // request was for compile, and failed on validation
                if (
                  request.check === 'error' &&
                  (error != null ? error.message : undefined) === 'exited'
                ) {
                  error = new Error('compilation')
                  error.validate = 'fail'
                }
                // compile was killed by user, was a validation, or a compile which failed validation
                if (
                  (error != null ? error.terminated : undefined) ||
                  (error != null ? error.validate : undefined) ||
                  (error != null ? error.timedout : undefined)
                ) {
                  OutputFileFinder.findOutputFiles(
                    resourceList,
                    compileDir,
                    function (err, outputFiles) {
                      if (err != null) {
                        return callback(err)
                      }
                      error.outputFiles = outputFiles // return output files so user can check logs
                      return callback(error)
                    }
                  )
                  return
                }
                // compile completed normally
                if (error != null) {
                  return callback(error)
                }
                Metrics.inc('compiles-succeeded')
                stats = stats || {}
                const object = stats || {}
                for (metric_key in object) {
                  metric_value = object[metric_key]
                  Metrics.count(metric_key, metric_value)
                }
                timings = timings || {}
                const object1 = timings || {}
                for (metric_key in object1) {
                  metric_value = object1[metric_key]
                  Metrics.timing(metric_key, metric_value)
                }
                const loadavg =
                  typeof os.loadavg === 'function' ? os.loadavg() : undefined
                if (loadavg != null) {
                  Metrics.gauge('load-avg', loadavg[0])
                }
                const ts = timer.done()
                logger.log(
                  {
                    project_id: request.project_id,
                    user_id: request.user_id,
                    time_taken: ts,
                    stats,
                    timings,
                    loadavg,
                  },
                  'done compile'
                )
                if ((stats != null ? stats['latex-runs'] : undefined) > 0) {
                  Metrics.timing(
                    'run-compile-per-pass',
                    ts / stats['latex-runs']
                  )
                }
                if (
                  (stats != null ? stats['latex-runs'] : undefined) > 0 &&
                  (timings != null ? timings['cpu-time'] : undefined) > 0
                ) {
                  Metrics.timing(
                    'run-compile-cpu-time-per-pass',
                    timings['cpu-time'] / stats['latex-runs']
                  )
                }
                // Emit compile time.
                timings.compile = ts

                timer = new Metrics.Timer('process-output-files')

                return OutputFileFinder.findOutputFiles(
                  resourceList,
                  compileDir,
                  function (error, outputFiles) {
                    if (error != null) {
                      return callback(error)
                    }
                    return OutputCacheManager.saveOutputFiles(
                      { request, stats, timings },
                      outputFiles,
                      compileDir,
                      outputDir,
                      (err, newOutputFiles) => {
                        if (err) {
                          const { project_id: projectId, user_id: userId } =
                            request
                          logger.err(
                            { projectId, userId, err },
                            'failed to save output files'
                          )
                        }

                        const outputStage = timer.done()
                        timings.sync = syncStage
                        timings.output = outputStage

                        // Emit e2e compile time.
                        timings.compileE2E = timerE2E.done()

                        if (stats['pdf-size']) {
                          emitPdfStats(stats, timings)
                        }

                        callback(null, newOutputFiles, stats, timings)
                      }
                    )
                  }
                )
              }
            )
          }
        )
      }
    )
  },

  stopCompile(project_id, user_id, callback) {
    if (callback == null) {
      callback = function (error) {}
    }
    const compileName = getCompileName(project_id, user_id)
    return LatexRunner.killLatex(compileName, callback)
  },

  clearProject(project_id, user_id, _callback) {
    if (_callback == null) {
      _callback = function (error) {}
    }
    const callback = function (error) {
      _callback(error)
      return (_callback = function () {})
    }

    const compileDir = getCompileDir(project_id, user_id)
    const outputDir = getOutputDir(project_id, user_id)

    return CompileManager._checkDirectory(compileDir, function (err, exists) {
      if (err != null) {
        return callback(err)
      }
      if (!exists) {
        return callback()
      } // skip removal if no directory present

      const proc = child_process.spawn('rm', [
        '-r',
        '-f',
        '--',
        compileDir,
        outputDir,
      ])

      proc.on('error', callback)

      let stderr = ''
      proc.stderr.setEncoding('utf8').on('data', chunk => (stderr += chunk))

      return proc.on('close', function (code) {
        if (code === 0) {
          return callback(null)
        } else {
          return callback(
            new Error(`rm -r ${compileDir} ${outputDir} failed: ${stderr}`)
          )
        }
      })
    })
  },

  _findAllDirs(callback) {
    if (callback == null) {
      callback = function (error, allDirs) {}
    }
    const root = Settings.path.compilesDir
    return fs.readdir(root, function (err, files) {
      if (err != null) {
        return callback(err)
      }
      const allDirs = Array.from(files).map(file => Path.join(root, file))
      return callback(null, allDirs)
    })
  },

  clearExpiredProjects(max_cache_age_ms, callback) {
    if (callback == null) {
      callback = function (error) {}
    }
    const now = Date.now()
    // action for each directory
    const expireIfNeeded = (checkDir, cb) =>
      fs.stat(checkDir, function (err, stats) {
        if (err != null) {
          return cb()
        } // ignore errors checking directory
        const age = now - stats.mtime
        const hasExpired = age > max_cache_age_ms
        if (hasExpired) {
          return fse.remove(checkDir, cb)
        } else {
          return cb()
        }
      })
    // iterate over all project directories
    return CompileManager._findAllDirs(function (error, allDirs) {
      if (error != null) {
        return callback()
      }
      return async.eachSeries(allDirs, expireIfNeeded, callback)
    })
  },

  _checkDirectory(compileDir, callback) {
    if (callback == null) {
      callback = function (error, exists) {}
    }
    return fs.lstat(compileDir, function (err, stats) {
      if ((err != null ? err.code : undefined) === 'ENOENT') {
        return callback(null, false) //  directory does not exist
      } else if (err != null) {
        logger.err(
          { dir: compileDir, err },
          'error on stat of project directory for removal'
        )
        return callback(err)
      } else if (!(stats != null ? stats.isDirectory() : undefined)) {
        logger.err(
          { dir: compileDir, stats },
          'bad project directory for removal'
        )
        return callback(new Error('project directory is not directory'))
      } else {
        return callback(null, true)
      }
    })
  }, // directory exists

  syncFromCode(
    project_id,
    user_id,
    file_name,
    line,
    column,
    imageName,
    callback
  ) {
    // If LaTeX was run in a virtual environment, the file path that synctex expects
    // might not match the file path on the host. The .synctex.gz file however, will be accessed
    // wherever it is on the host.
    if (callback == null) {
      callback = function (error, pdfPositions) {}
    }
    const compileName = getCompileName(project_id, user_id)
    const base_dir = Settings.path.synctexBaseDir(compileName)
    const file_path = base_dir + '/' + file_name
    const compileDir = getCompileDir(project_id, user_id)
    const synctex_path = `${base_dir}/output.pdf`
    const command = ['code', synctex_path, file_path, line, column]
    CompileManager._runSynctex(
      project_id,
      user_id,
      command,
      imageName,
      function (error, stdout) {
        if (error != null) {
          return callback(error)
        }
        logger.log(
          { project_id, user_id, file_name, line, column, command, stdout },
          'synctex code output'
        )
        return callback(
          null,
          CompileManager._parseSynctexFromCodeOutput(stdout)
        )
      }
    )
  },

  syncFromPdf(project_id, user_id, page, h, v, imageName, callback) {
    if (callback == null) {
      callback = function (error, filePositions) {}
    }
    const compileName = getCompileName(project_id, user_id)
    const compileDir = getCompileDir(project_id, user_id)
    const base_dir = Settings.path.synctexBaseDir(compileName)
    const synctex_path = `${base_dir}/output.pdf`
    const command = ['pdf', synctex_path, page, h, v]
    CompileManager._runSynctex(
      project_id,
      user_id,
      command,
      imageName,
      function (error, stdout) {
        if (error != null) {
          return callback(error)
        }
        logger.log(
          { project_id, user_id, page, h, v, stdout },
          'synctex pdf output'
        )
        return callback(
          null,
          CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
        )
      }
    )
  },

  _checkFileExists(dir, filename, callback) {
    if (callback == null) {
      callback = function (error) {}
    }
    const file = Path.join(dir, filename)
    return fs.stat(dir, function (error, stats) {
      if ((error != null ? error.code : undefined) === 'ENOENT') {
        return callback(new Errors.NotFoundError('no output directory'))
      }
      if (error != null) {
        return callback(error)
      }
      return fs.stat(file, function (error, stats) {
        if ((error != null ? error.code : undefined) === 'ENOENT') {
          return callback(new Errors.NotFoundError('no output file'))
        }
        if (error != null) {
          return callback(error)
        }
        if (!(stats != null ? stats.isFile() : undefined)) {
          return callback(new Error('not a file'))
        }
        return callback()
      })
    })
  },

  _runSynctex(project_id, user_id, command, imageName, callback) {
    if (callback == null) {
      callback = function (error, stdout) {}
    }
    const seconds = 1000

    command.unshift('/opt/synctex')

    const directory = getCompileDir(project_id, user_id)
    const timeout = 60 * 1000 // increased to allow for large projects
    const compileName = getCompileName(project_id, user_id)
    const compileGroup = 'synctex'
    CompileManager._checkFileExists(directory, 'output.synctex.gz', error => {
      if (error) {
        return callback(error)
      }
      return CommandRunner.run(
        compileName,
        command,
        directory,
        imageName ||
          (Settings.clsi && Settings.clsi.docker
            ? Settings.clsi.docker.image
            : undefined),
        timeout,
        {},
        compileGroup,
        function (error, output) {
          if (error != null) {
            logger.err(
              { err: error, command, project_id, user_id },
              'error running synctex'
            )
            return callback(error)
          }
          return callback(null, output.stdout)
        }
      )
    })
  },

  _parseSynctexFromCodeOutput(output) {
    const results = []
    for (const line of Array.from(output.split('\n'))) {
      const [node, page, h, v, width, height] = Array.from(line.split('\t'))
      if (node === 'NODE') {
        results.push({
          page: parseInt(page, 10),
          h: parseFloat(h),
          v: parseFloat(v),
          height: parseFloat(height),
          width: parseFloat(width),
        })
      }
    }
    return results
  },

  _parseSynctexFromPdfOutput(output, base_dir) {
    const results = []
    for (let line of Array.from(output.split('\n'))) {
      let column, file_path, node
      ;[node, file_path, line, column] = Array.from(line.split('\t'))
      if (node === 'NODE') {
        const file = file_path.slice(base_dir.length + 1)
        results.push({
          file,
          line: parseInt(line, 10),
          column: parseInt(column, 10),
        })
      }
    }
    return results
  },

  wordcount(project_id, user_id, file_name, image, callback) {
    if (callback == null) {
      callback = function (error, pdfPositions) {}
    }
    logger.log({ project_id, user_id, file_name, image }, 'running wordcount')
    const file_path = `$COMPILE_DIR/${file_name}`
    const command = [
      'texcount',
      '-nocol',
      '-inc',
      file_path,
      `-out=${file_path}.wc`,
    ]
    const compileDir = getCompileDir(project_id, user_id)
    const timeout = 60 * 1000
    const compileName = getCompileName(project_id, user_id)
    const compileGroup = 'wordcount'
    return fse.ensureDir(compileDir, function (error) {
      if (error != null) {
        logger.err(
          { error, project_id, user_id, file_name },
          'error ensuring dir for sync from code'
        )
        return callback(error)
      }
      return CommandRunner.run(
        compileName,
        command,
        compileDir,
        image,
        timeout,
        {},
        compileGroup,
        function (error) {
          if (error != null) {
            return callback(error)
          }
          return fs.readFile(
            compileDir + '/' + file_name + '.wc',
            'utf-8',
            function (err, stdout) {
              if (err != null) {
                // call it node_err so sentry doesn't use random path error as unique id so it can't be ignored
                logger.err(
                  { node_err: err, command, compileDir, project_id, user_id },
                  'error reading word count output'
                )
                return callback(err)
              }
              const results = CompileManager._parseWordcountFromOutput(stdout)
              logger.log(
                { project_id, user_id, wordcount: results },
                'word count results'
              )
              return callback(null, results)
            }
          )
        }
      )
    })
  },

  _parseWordcountFromOutput(output) {
    const results = {
      encode: '',
      textWords: 0,
      headWords: 0,
      outside: 0,
      headers: 0,
      elements: 0,
      mathInline: 0,
      mathDisplay: 0,
      errors: 0,
      messages: '',
    }
    for (const line of Array.from(output.split('\n'))) {
      const [data, info] = Array.from(line.split(':'))
      if (data.indexOf('Encoding') > -1) {
        results.encode = info.trim()
      }
      if (data.indexOf('in text') > -1) {
        results.textWords = parseInt(info, 10)
      }
      if (data.indexOf('in head') > -1) {
        results.headWords = parseInt(info, 10)
      }
      if (data.indexOf('outside') > -1) {
        results.outside = parseInt(info, 10)
      }
      if (data.indexOf('of head') > -1) {
        results.headers = parseInt(info, 10)
      }
      if (data.indexOf('Number of floats/tables/figures') > -1) {
        results.elements = parseInt(info, 10)
      }
      if (data.indexOf('Number of math inlines') > -1) {
        results.mathInline = parseInt(info, 10)
      }
      if (data.indexOf('Number of math displayed') > -1) {
        results.mathDisplay = parseInt(info, 10)
      }
      if (data === '(errors') {
        // errors reported as (errors:123)
        results.errors = parseInt(info, 10)
      }
      if (line.indexOf('!!! ') > -1) {
        // errors logged as !!! message !!!
        results.messages += line + '\n'
      }
    }
    return results
  },
}

function __guard__(value, transform) {
  return typeof value !== 'undefined' && value !== null
    ? transform(value)
    : undefined
}