CompileManager.coffee 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284
  1. ResourceWriter = require "./ResourceWriter"
  2. LatexRunner = require "./LatexRunner"
  3. OutputFileFinder = require "./OutputFileFinder"
  4. OutputCacheManager = require "./OutputCacheManager"
  5. Settings = require("settings-sharelatex")
  6. Path = require "path"
  7. logger = require "logger-sharelatex"
  8. Metrics = require "./Metrics"
  9. child_process = require "child_process"
  10. DraftModeManager = require "./DraftModeManager"
  11. fs = require("fs")
  12. fse = require "fs-extra"
  13. os = require("os")
  14. async = require "async"
  15. commandRunner = Settings.clsi?.commandRunner or "./CommandRunner"
  16. logger.info commandRunner:commandRunner, "selecting command runner for clsi"
  17. CommandRunner = require(commandRunner)
  18. getCompileName = (project_id, user_id) ->
  19. if user_id? then "#{project_id}-#{user_id}" else project_id
  20. getCompileDir = (project_id, user_id) ->
  21. Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
  22. module.exports = CompileManager =
  23. doCompile: (request, callback = (error, outputFiles) ->) ->
  24. compileDir = getCompileDir(request.project_id, request.user_id)
  25. timer = new Metrics.Timer("write-to-disk")
  26. logger.log project_id: request.project_id, user_id: request.user_id, "syncing resources to disk"
  27. ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) ->
  28. if error?
  29. logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk"
  30. return callback(error)
  31. logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk"
  32. timer.done()
  33. injectDraftModeIfRequired = (callback) ->
  34. if request.draft
  35. DraftModeManager.injectDraftMode Path.join(compileDir, request.rootResourcePath), callback
  36. else
  37. callback()
  38. # set up environment variables for chktex
  39. env = {}
  40. if request.check?
  41. env['CHKTEX_OPTIONS'] = '-nall -e9 -e10 -w15 -w16'
  42. if request.check is 'error'
  43. env['CHKTEX_EXIT_ON_ERROR'] = 1
  44. if request.check is 'validate'
  45. env['CHKTEX_VALIDATE'] = 1
  46. injectDraftModeIfRequired (error) ->
  47. return callback(error) if error?
  48. timer = new Metrics.Timer("run-compile")
  49. # find the image tag to log it as a metric, e.g. 2015.1 (convert . to - for graphite)
  50. tag = request.imageName?.match(/:(.*)/)?[1]?.replace(/\./g,'-') or "default"
  51. tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test
  52. Metrics.inc("compiles")
  53. Metrics.inc("compiles-with-image.#{tag}")
  54. compileName = getCompileName(request.project_id, request.user_id)
  55. LatexRunner.runLatex compileName, {
  56. directory: compileDir
  57. mainFile: request.rootResourcePath
  58. compiler: request.compiler
  59. timeout: request.timeout
  60. image: request.imageName
  61. environment: env
  62. }, (error, output, stats, timings) ->
  63. # request was for validation only
  64. if request.check is "validate"
  65. result = if error?.code then "fail" else "pass"
  66. error = new Error("validation")
  67. error.validate = result
  68. # request was for compile, and failed on validation
  69. if request.check is "error" and error?.message is 'exited'
  70. error = new Error("compilation")
  71. error.validate = "fail"
  72. # compile was killed by user, was a validation, or a compile which failed validation
  73. if error?.terminated or error?.validate
  74. OutputFileFinder.findOutputFiles request.resources, compileDir, (err, outputFiles) ->
  75. return callback(err) if err?
  76. callback(error, outputFiles) # return output files so user can check logs
  77. return
  78. # compile completed normally
  79. return callback(error) if error?
  80. Metrics.inc("compiles-succeeded")
  81. for metric_key, metric_value of stats or {}
  82. Metrics.count(metric_key, metric_value)
  83. for metric_key, metric_value of timings or {}
  84. Metrics.timing(metric_key, metric_value)
  85. loadavg = os.loadavg?()
  86. Metrics.gauge("load-avg", loadavg[0]) if loadavg?
  87. ts = timer.done()
  88. logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile"
  89. if stats?["latex-runs"] > 0
  90. Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"])
  91. if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0
  92. Metrics.timing("run-compile-cpu-time-per-pass", timings["cpu-time"] / stats["latex-runs"])
  93. OutputFileFinder.findOutputFiles request.resources, compileDir, (error, outputFiles) ->
  94. return callback(error) if error?
  95. OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) ->
  96. callback null, newOutputFiles
  97. stopCompile: (project_id, user_id, callback = (error) ->) ->
  98. compileName = getCompileName(project_id, user_id)
  99. LatexRunner.killLatex compileName, callback
  100. clearProject: (project_id, user_id, _callback = (error) ->) ->
  101. callback = (error) ->
  102. _callback(error)
  103. _callback = () ->
  104. compileDir = getCompileDir(project_id, user_id)
  105. CompileManager._checkDirectory compileDir, (err, exists) ->
  106. return callback(err) if err?
  107. return callback() if not exists # skip removal if no directory present
  108. proc = child_process.spawn "rm", ["-r", compileDir]
  109. proc.on "error", callback
  110. stderr = ""
  111. proc.stderr.on "data", (chunk) -> stderr += chunk.toString()
  112. proc.on "close", (code) ->
  113. if code == 0
  114. return callback(null)
  115. else
  116. return callback(new Error("rm -r #{compileDir} failed: #{stderr}"))
  117. _findAllDirs: (callback = (error, allDirs) ->) ->
  118. root = Settings.path.compilesDir
  119. fs.readdir root, (err, files) ->
  120. return callback(err) if err?
  121. allDirs = (Path.join(root, file) for file in files)
  122. callback(null, allDirs)
  123. clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) ->
  124. now = Date.now()
  125. # action for each directory
  126. expireIfNeeded = (checkDir, cb) ->
  127. fs.stat checkDir, (err, stats) ->
  128. return cb() if err? # ignore errors checking directory
  129. age = now - stats.mtime
  130. hasExpired = (age > max_cache_age_ms)
  131. if hasExpired then fse.remove(checkDir, cb) else cb()
  132. # iterate over all project directories
  133. CompileManager._findAllDirs (error, allDirs) ->
  134. return callback() if error?
  135. async.eachSeries allDirs, expireIfNeeded, callback
  136. _checkDirectory: (compileDir, callback = (error, exists) ->) ->
  137. fs.lstat compileDir, (err, stats) ->
  138. if err?.code is 'ENOENT'
  139. return callback(null, false) # directory does not exist
  140. else if err?
  141. logger.err {dir: compileDir, err:err}, "error on stat of project directory for removal"
  142. return callback(err)
  143. else if not stats?.isDirectory()
  144. logger.err {dir: compileDir, stats:stats}, "bad project directory for removal"
  145. return callback new Error("project directory is not directory")
  146. else
  147. callback(null, true) # directory exists
  148. syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) ->
  149. # If LaTeX was run in a virtual environment, the file path that synctex expects
  150. # might not match the file path on the host. The .synctex.gz file however, will be accessed
  151. # wherever it is on the host.
  152. compileName = getCompileName(project_id, user_id)
  153. base_dir = Settings.path.synctexBaseDir(compileName)
  154. file_path = base_dir + "/" + file_name
  155. compileDir = getCompileDir(project_id, user_id)
  156. synctex_path = Path.join(compileDir, "output.pdf")
  157. CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
  158. return callback(error) if error?
  159. logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
  160. callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
  161. syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
  162. compileName = getCompileName(project_id, user_id)
  163. base_dir = Settings.path.synctexBaseDir(compileName)
  164. compileDir = getCompileDir(project_id, user_id)
  165. synctex_path = Path.join(compileDir, "output.pdf")
  166. CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
  167. return callback(error) if error?
  168. logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
  169. callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
  170. _runSynctex: (args, callback = (error, stdout) ->) ->
  171. bin_path = Path.resolve(__dirname + "/../../bin/synctex")
  172. seconds = 1000
  173. if Settings.clsi?.synctexCommandWrapper?
  174. [bin_path, args] = Settings.clsi?.synctexCommandWrapper bin_path, args
  175. child_process.execFile bin_path, args, timeout: 10 * seconds, (error, stdout, stderr) ->
  176. if error?
  177. logger.err err:error, args:args, "error running synctex"
  178. return callback(error)
  179. callback(null, stdout)
  180. _parseSynctexFromCodeOutput: (output) ->
  181. results = []
  182. for line in output.split("\n")
  183. [node, page, h, v, width, height] = line.split("\t")
  184. if node == "NODE"
  185. results.push {
  186. page: parseInt(page, 10)
  187. h: parseFloat(h)
  188. v: parseFloat(v)
  189. height: parseFloat(height)
  190. width: parseFloat(width)
  191. }
  192. return results
  193. _parseSynctexFromPdfOutput: (output, base_dir) ->
  194. results = []
  195. for line in output.split("\n")
  196. [node, file_path, line, column] = line.split("\t")
  197. if node == "NODE"
  198. file = file_path.slice(base_dir.length + 1)
  199. results.push {
  200. file: file
  201. line: parseInt(line, 10)
  202. column: parseInt(column, 10)
  203. }
  204. return results
  205. wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) ->
  206. logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount"
  207. file_path = "$COMPILE_DIR/" + file_name
  208. command = [ "texcount", '-nocol', '-inc', file_path, "-out=" + file_path + ".wc"]
  209. directory = getCompileDir(project_id, user_id)
  210. timeout = 10 * 1000
  211. compileName = getCompileName(project_id, user_id)
  212. CommandRunner.run compileName, command, directory, image, timeout, {}, (error) ->
  213. return callback(error) if error?
  214. fs.readFile directory + "/" + file_name + ".wc", "utf-8", (err, stdout) ->
  215. if err?
  216. logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output"
  217. return callback(err)
  218. results = CompileManager._parseWordcountFromOutput(stdout)
  219. logger.log project_id:project_id, user_id:user_id, wordcount: results, "word count results"
  220. callback null, results
  221. _parseWordcountFromOutput: (output) ->
  222. results = {
  223. encode: ""
  224. textWords: 0
  225. headWords: 0
  226. outside: 0
  227. headers: 0
  228. elements: 0
  229. mathInline: 0
  230. mathDisplay: 0
  231. errors: 0
  232. messages: ""
  233. }
  234. for line in output.split("\n")
  235. [data, info] = line.split(":")
  236. if data.indexOf("Encoding") > -1
  237. results['encode'] = info.trim()
  238. if data.indexOf("in text") > -1
  239. results['textWords'] = parseInt(info, 10)
  240. if data.indexOf("in head") > -1
  241. results['headWords'] = parseInt(info, 10)
  242. if data.indexOf("outside") > -1
  243. results['outside'] = parseInt(info, 10)
  244. if data.indexOf("of head") > -1
  245. results['headers'] = parseInt(info, 10)
  246. if data.indexOf("Number of floats/tables/figures") > -1
  247. results['elements'] = parseInt(info, 10)
  248. if data.indexOf("Number of math inlines") > -1
  249. results['mathInline'] = parseInt(info, 10)
  250. if data.indexOf("Number of math displayed") > -1
  251. results['mathDisplay'] = parseInt(info, 10)
  252. if data is "(errors" # errors reported as (errors:123)
  253. results['errors'] = parseInt(info, 10)
  254. if line.indexOf("!!! ") > -1 # errors logged as !!! message !!!
  255. results['messages'] += line + "\n"
  256. return results