overleaf/services/clsi/app/js/ResourceWriter.js

/* eslint-disable
    no-return-assign,
    no-unused-vars,
    no-useless-escape,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
 * decaffeinate suggestions:
 * DS101: Remove unnecessary use of Array.from
 * DS102: Remove unnecessary code created because of implicit returns
 * DS207: Consider shorter variations of null checks
 * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
 */
import { promisify } from 'node:util'
import UrlCache from './UrlCache.js'
import Path from 'node:path'
import fs from 'node:fs'
import async from 'async'
import OutputFileFinder from './OutputFileFinder.js'
import ResourceStateManager from './ResourceStateManager.js'
import Metrics from '@overleaf/metrics'
import logger from '@overleaf/logger'
import settings from '@overleaf/settings'
import ClsiMetrics from './Metrics.js'
import { Minimatch } from 'minimatch'

const { shouldSkipMetrics } = ClsiMetrics

// Additional file patterns to keep between compiles
const preciousFileMatcher = new Minimatch(settings.preciousFilePattern, {
  dot: true,
})

let ResourceWriter

const parallelFileDownloads = settings.parallelFileDownloads || 1

export default ResourceWriter = {
  syncResourcesToDisk(request, basePath, callback) {
    if (callback == null) {
      callback = function () {}
    }
    if (request.syncType === 'incremental') {
      logger.debug(
        { projectId: request.project_id, userId: request.user_id },
        'incremental sync'
      )
      return ResourceStateManager.checkProjectStateMatches(
        request.syncState,
        basePath,
        function (error, resourceList) {
          if (error != null) {
            return callback(error)
          }
          return ResourceWriter._removeExtraneousFiles(
            request,
            resourceList,
            basePath,
            function (error, outputFiles, allFiles) {
              if (error != null) {
                return callback(error)
              }
              return ResourceStateManager.checkResourceFiles(
                resourceList,
                allFiles,
                basePath,
                function (error) {
                  if (error != null) {
                    return callback(error)
                  }
                  return ResourceWriter.saveIncrementalResourcesToDisk(
                    request.project_id,
                    request.resources,
                    basePath,
                    function (error) {
                      if (error != null) {
                        return callback(error)
                      }
                      return callback(null, resourceList)
                    }
                  )
                }
              )
            }
          )
        }
      )
    }
    logger.debug(
      { projectId: request.project_id, userId: request.user_id },
      'full sync'
    )
    UrlCache.createProjectDir(request.project_id, error => {
      if (error != null) {
        return callback(error)
      }
      ResourceWriter.saveAllResourcesToDisk(
        request,
        basePath,
        function (error) {
          if (error != null) {
            return callback(error)
          }
          return ResourceStateManager.saveProjectState(
            request.syncState,
            request.resources,
            basePath,
            function (error) {
              if (error != null) {
                return callback(error)
              }
              return callback(null, request.resources)
            }
          )
        }
      )
    })
  },

  saveIncrementalResourcesToDisk(projectId, resources, basePath, callback) {
    if (callback == null) {
      callback = function () {}
    }
    return ResourceWriter._createDirectory(basePath, error => {
      if (error != null) {
        return callback(error)
      }
      const jobs = Array.from(resources).map(resource =>
        (resource => {
          return callback =>
            ResourceWriter._writeResourceToDisk(
              projectId,
              resource,
              basePath,
              callback
            )
        })(resource)
      )
      return async.parallelLimit(jobs, parallelFileDownloads, callback)
    })
  },

  saveAllResourcesToDisk(request, basePath, callback) {
    if (callback == null) {
      callback = function () {}
    }
    return ResourceWriter._createDirectory(basePath, error => {
      if (error != null) {
        return callback(error)
      }
      const { project_id: projectId, resources } = request
      ResourceWriter._removeExtraneousFiles(
        request,
        resources,
        basePath,
        error => {
          if (error != null) {
            return callback(error)
          }
          const jobs = Array.from(resources).map(resource =>
            (resource => {
              return callback =>
                ResourceWriter._writeResourceToDisk(
                  projectId,
                  resource,
                  basePath,
                  callback
                )
            })(resource)
          )
          return async.parallelLimit(jobs, parallelFileDownloads, callback)
        }
      )
    })
  },

  _createDirectory(basePath, callback) {
    if (callback == null) {
      callback = function () {}
    }
    return fs.mkdir(basePath, function (err) {
      if (err != null) {
        if (err.code === 'EEXIST') {
          return callback()
        } else {
          logger.debug({ err, dir: basePath }, 'error creating directory')
          return callback(err)
        }
      } else {
        return callback()
      }
    })
  },

  _removeExtraneousFiles(request, resources, basePath, _callback) {
    if (_callback == null) {
      _callback = function () {}
    }
    const timer = new Metrics.Timer(
      'unlink-output-files',
      1,
      request.metricsOpts
    )
    const callback = function (error, ...result) {
      if (!shouldSkipMetrics(request)) timer.done()
      return _callback(error, ...Array.from(result))
    }

    return OutputFileFinder.findOutputFiles(
      resources,
      basePath,
      (error, outputFiles, allFiles) => {
        if (error != null) {
          return callback(error)
        }

        const jobs = []
        for (const { path } of outputFiles || []) {
          const shouldDelete = ResourceWriter.isExtraneousFile(path)
          if (shouldDelete) {
            jobs.push(callback =>
              ResourceWriter._deleteFileIfNotDirectory(
                Path.join(basePath, path),
                callback
              )
            )
          }
        }

        return async.series(jobs, function (error) {
          if (error != null) {
            return callback(error)
          }
          return callback(null, outputFiles, allFiles)
        })
      }
    )
  },

  isExtraneousFile(path) {
    let shouldDelete = true
    if (
      path.match(/^output\./) ||
      path.match(/\.aux$/) ||
      path.match(/^cache\//)
    ) {
      // knitr cache
      shouldDelete = false
    }
    if (path.match(/^output-.*/)) {
      // Tikz cached figures (default case)
      shouldDelete = false
    }
    if (path.match(/\.(pdf|dpth|md5)$/)) {
      // Tikz cached figures (by extension)
      shouldDelete = false
    }
    if (
      path.match(/\.(pygtex|pygstyle)$/) ||
      path.match(/(^|\/)_minted(-[^\/]+)?\//)
    ) {
      // minted files/directory
      shouldDelete = false
    }
    if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) {
      // markdown files/directory
      shouldDelete = false
    }
    if (path.match(/-eps-converted-to\.pdf$/)) {
      // Epstopdf generated files
      shouldDelete = false
    }
    // Keep additional precious files and directories
    if (shouldDelete && preciousFileMatcher.match(path)) {
      shouldDelete = false
    }
    if (
      path === 'output.tar.gz' ||
      path === 'output.synctex.gz' ||
      path === 'output.pdfxref' ||
      path === 'output.pdf' ||
      path === 'output.dvi' ||
      path === 'output.log' ||
      path === 'output.xdv' ||
      path === 'output.stdout' ||
      path === 'output.stderr'
    ) {
      shouldDelete = true
    }
    if (path === 'output.tex') {
      // created by TikzManager if present in output files
      shouldDelete = true
    }
    return shouldDelete
  },

  _deleteFileIfNotDirectory(path, callback) {
    if (callback == null) {
      callback = function () {}
    }
    return fs.stat(path, function (error, stat) {
      if (error != null && error.code === 'ENOENT') {
        return callback()
      } else if (error != null) {
        logger.err(
          { err: error, path },
          'error stating file in deleteFileIfNotDirectory'
        )
        return callback(error)
      } else if (stat.isFile()) {
        return fs.unlink(path, function (error) {
          if (error != null) {
            logger.err(
              { err: error, path },
              'error removing file in deleteFileIfNotDirectory'
            )
            return callback(error)
          } else {
            return callback()
          }
        })
      } else {
        return callback()
      }
    })
  },

  _writeResourceToDisk(projectId, resource, basePath, callback) {
    if (callback == null) {
      callback = function () {}
    }
    return ResourceWriter.checkPath(
      basePath,
      resource.path,
      function (error, path) {
        if (error != null) {
          return callback(error)
        }
        return fs.mkdir(
          Path.dirname(path),
          { recursive: true },
          function (error) {
            if (error != null) {
              return callback(error)
            }
            // TODO: Don't overwrite file if it hasn't been modified
            if (resource.url != null) {
              return UrlCache.downloadUrlToFile(
                projectId,
                resource.url,
                resource.fallbackURL,
                path,
                resource.modified,
                function (err) {
                  if (err != null) {
                    logger.err(
                      {
                        err,
                        projectId,
                        path,
                        resourceUrl: resource.url,
                        modified: resource.modified,
                      },
                      'error downloading file for resources'
                    )
                    Metrics.inc('download-failed')
                  }
                  return callback()
                }
              ) // try and continue compiling even if http resource can not be downloaded at this time
            } else {
              fs.writeFile(path, resource.content, callback)
            }
          }
        )
      }
    )
  },

  checkPath(basePath, resourcePath, callback) {
    const path = Path.normalize(Path.join(basePath, resourcePath))
    if (path.slice(0, basePath.length + 1) !== basePath + '/') {
      return callback(new Error('resource path is outside root directory'))
    } else {
      return callback(null, path)
    }
  },
}

ResourceWriter.promises = {
  syncResourcesToDisk: promisify(ResourceWriter.syncResourcesToDisk),
  saveIncrementalResourcesToDisk: promisify(
    ResourceWriter.saveIncrementalResourcesToDisk
  ),
  saveAllResourcesToDisk: promisify(ResourceWriter.saveAllResourcesToDisk),
  checkPath: promisify(ResourceWriter.checkPath),
}