OutputCacheManager.coffee 5.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. async = require "async"
  2. fs = require "fs"
  3. fse = require "fs-extra"
  4. Path = require "path"
  5. logger = require "logger-sharelatex"
  6. _ = require "underscore"
  7. Settings = require "settings-sharelatex"
  8. OutputFileOptimiser = require "./OutputFileOptimiser"
  9. module.exports = OutputCacheManager =
  10. CACHE_SUBDIR: '.cache/clsi'
  11. ARCHIVE_SUBDIR: '.archive/clsi'
  12. BUILD_REGEX: /^[0-9a-f]+$/ # build id is Date.now() converted to hex
  13. CACHE_LIMIT: 2 # maximum number of cache directories
  14. CACHE_AGE: 60*60*1000 # up to one hour old
  15. path: (buildId, file) ->
  16. # used by static server, given build id return '.cache/clsi/buildId'
  17. if buildId.match OutputCacheManager.BUILD_REGEX
  18. return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
  19. else
  20. # for invalid build id, return top level
  21. return file
  22. saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) ->
  23. # make a compileDir/CACHE_SUBDIR/build_id directory and
  24. # copy all the output files into it
  25. cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
  26. # Put the files into a new cache subdirectory
  27. buildId = Date.now().toString(16)
  28. cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
  29. # let file expiry run in the background
  30. OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId}
  31. # Archive logs in background
  32. if Settings.clsi?.archive_logs or Settings.clsi?.strace
  33. OutputCacheManager.archiveLogs outputFiles, compileDir, (err) ->
  34. if err?
  35. logger.warn err:err, "erroring archiving log files"
  36. # make the new cache directory
  37. fse.ensureDir cacheDir, (err) ->
  38. if err?
  39. logger.error err: err, directory: cacheDir, "error creating cache directory"
  40. callback(err, outputFiles)
  41. else
  42. # copy all the output files into the new cache directory
  43. results = []
  44. async.mapSeries outputFiles, (file, cb) ->
  45. newFile = _.clone(file)
  46. [src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
  47. OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
  48. return cb(err) if err?
  49. if !isSafe
  50. return cb()
  51. OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) ->
  52. return cb(err) if err?
  53. if !shouldCopy
  54. return cb()
  55. OutputCacheManager._copyFile src, dst, (err) ->
  56. return cb(err) if err?
  57. newFile.build = buildId # attach a build id if we cached the file
  58. results.push newFile
  59. cb()
  60. , (err) ->
  61. if err?
  62. # pass back the original files if we encountered *any* error
  63. callback(err, outputFiles)
  64. else
  65. # pass back the list of new files in the cache
  66. callback(err, results)
  67. archiveLogs: (outputFiles, compileDir, callback = (error) ->) ->
  68. buildId = Date.now().toString(16)
  69. archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId)
  70. logger.log {dir: archiveDir}, "archiving log files for project"
  71. fse.ensureDir archiveDir, (err) ->
  72. return callback(err) if err?
  73. async.mapSeries outputFiles, (file, cb) ->
  74. [src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]
  75. OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
  76. return cb(err) if err?
  77. return cb() if !isSafe
  78. OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) ->
  79. return cb(err) if err?
  80. return cb() if !shouldArchive
  81. OutputCacheManager._copyFile src, dst, cb
  82. , callback
  83. expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
  84. # look in compileDir for build dirs and delete if > N or age of mod time > T
  85. fs.readdir cacheRoot, (err, results) ->
  86. if err?
  87. return callback(null) if err.code == 'ENOENT' # cache directory is empty
  88. logger.error err: err, project_id: cacheRoot, "error clearing cache"
  89. return callback(err)
  90. dirs = results.sort().reverse()
  91. currentTime = Date.now()
  92. isExpired = (dir, index) ->
  93. return false if options?.keep == dir
  94. # remove any directories over the hard limit
  95. return true if index > OutputCacheManager.CACHE_LIMIT
  96. # we can get the build time from the directory name
  97. dirTime = parseInt(dir, 16)
  98. age = currentTime - dirTime
  99. return age > OutputCacheManager.CACHE_AGE
  100. toRemove = _.filter(dirs, isExpired)
  101. removeDir = (dir, cb) ->
  102. fse.remove Path.join(cacheRoot, dir), (err, result) ->
  103. logger.log cache: cacheRoot, dir: dir, "removed expired cache dir"
  104. if err?
  105. logger.error err: err, dir: dir, "cache remove error"
  106. cb(err, result)
  107. async.eachSeries toRemove, (dir, cb) ->
  108. removeDir dir, cb
  109. , callback
  110. _checkFileIsSafe: (src, callback = (error, isSafe) ->) ->
  111. # check if we have a valid file to copy into the cache
  112. fs.stat src, (err, stats) ->
  113. if err?
  114. # some problem reading the file
  115. logger.error err: err, file: src, "stat error for file in cache"
  116. callback(err, false)
  117. else if not stats.isFile()
  118. # other filetype - reject it
  119. logger.warn src: src, stat: stats, "nonfile output - refusing to copy to cache"
  120. callback(null, false)
  121. else
  122. # it's a plain file, ok to copy
  123. callback(null, true)
  124. _copyFile: (src, dst, callback) ->
  125. # copy output file into the cache
  126. fse.copy src, dst, (err) ->
  127. if err?
  128. logger.error err: err, src: src, dst: dst, "copy error for file in cache"
  129. callback(err)
  130. else
  131. # call the optimiser for the file too
  132. OutputFileOptimiser.optimiseFile src, dst, callback
  133. _checkIfShouldCopy: (src, callback = (err, shouldCopy) ->) ->
  134. return callback(null, !Path.basename(src).match(/^strace/))
  135. _checkIfShouldArchive: (src, callback = (err, shouldCopy) ->) ->
  136. if Path.basename(src).match(/^strace/)
  137. return callback(null, true)
  138. if Settings.clsi?.archive_logs and Path.basename(src) in ["output.log", "output.blg"]
  139. return callback(null, true)
  140. return callback(null, false)