ResourceWriter.coffee 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130
  1. UrlCache = require "./UrlCache"
  2. Path = require "path"
  3. fs = require "fs"
  4. async = require "async"
  5. mkdirp = require "mkdirp"
  6. OutputFileFinder = require "./OutputFileFinder"
  7. ResourceStateManager = require "./ResourceStateManager"
  8. Metrics = require "./Metrics"
  9. logger = require "logger-sharelatex"
  10. settings = require("settings-sharelatex")
  11. parallelFileDownloads = settings.parallelFileDownloads or 1
  12. module.exports = ResourceWriter =
  13. syncResourcesToDisk: (request, basePath, callback = (error, resourceList) ->) ->
  14. if request.syncType is "incremental"
  15. logger.log project_id: request.project_id, user_id: request.user_id, "incremental sync"
  16. ResourceStateManager.checkProjectStateMatches request.syncState, basePath, (error, resourceList) ->
  17. return callback(error) if error?
  18. ResourceWriter._removeExtraneousFiles resourceList, basePath, (error, outputFiles, allFiles) ->
  19. return callback(error) if error?
  20. ResourceStateManager.checkResourceFiles resourceList, allFiles, basePath, (error) ->
  21. return callback(error) if error?
  22. ResourceWriter.saveIncrementalResourcesToDisk request.project_id, request.resources, basePath, (error) ->
  23. return callback(error) if error?
  24. callback(null, resourceList)
  25. else
  26. logger.log project_id: request.project_id, user_id: request.user_id, "full sync"
  27. @saveAllResourcesToDisk request.project_id, request.resources, basePath, (error) ->
  28. return callback(error) if error?
  29. ResourceStateManager.saveProjectState request.syncState, request.resources, basePath, (error) ->
  30. return callback(error) if error?
  31. callback(null, request.resources)
  32. saveIncrementalResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
  33. @_createDirectory basePath, (error) =>
  34. return callback(error) if error?
  35. jobs = for resource in resources
  36. do (resource) =>
  37. (callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
  38. async.parallelLimit jobs, parallelFileDownloads, callback
  39. saveAllResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
  40. @_createDirectory basePath, (error) =>
  41. return callback(error) if error?
  42. @_removeExtraneousFiles resources, basePath, (error) =>
  43. return callback(error) if error?
  44. jobs = for resource in resources
  45. do (resource) =>
  46. (callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
  47. async.parallelLimit jobs, parallelFileDownloads, callback
  48. _createDirectory: (basePath, callback = (error) ->) ->
  49. fs.mkdir basePath, (err) ->
  50. if err?
  51. if err.code is 'EEXIST'
  52. return callback()
  53. else
  54. logger.log {err: err, dir:basePath}, "error creating directory"
  55. return callback(err)
  56. else
  57. return callback()
  58. _removeExtraneousFiles: (resources, basePath, _callback = (error, outputFiles, allFiles) ->) ->
  59. timer = new Metrics.Timer("unlink-output-files")
  60. callback = (error, result...) ->
  61. timer.done()
  62. _callback(error, result...)
  63. OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles, allFiles) ->
  64. return callback(error) if error?
  65. jobs = []
  66. for file in outputFiles or []
  67. do (file) ->
  68. path = file.path
  69. should_delete = true
  70. if path.match(/^output\./) or path.match(/\.aux$/) or path.match(/^cache\//) # knitr cache
  71. should_delete = false
  72. if path.match(/^output-.*/) # Tikz cached figures
  73. should_delete = false
  74. if path == "output.pdf" or path == "output.dvi" or path == "output.log" or path == "output.xdv"
  75. should_delete = true
  76. if path == "output.tex" # created by TikzManager if present in output files
  77. should_delete = true
  78. if should_delete
  79. jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback
  80. async.series jobs, (error) ->
  81. return callback(error) if error?
  82. callback(null, outputFiles, allFiles)
  83. _deleteFileIfNotDirectory: (path, callback = (error) ->) ->
  84. fs.stat path, (error, stat) ->
  85. if error? and error.code is 'ENOENT'
  86. return callback()
  87. else if error?
  88. logger.err {err: error, path: path}, "error stating file in deleteFileIfNotDirectory"
  89. return callback(error)
  90. else if stat.isFile()
  91. fs.unlink path, (error) ->
  92. if error?
  93. logger.err {err: error, path: path}, "error removing file in deleteFileIfNotDirectory"
  94. callback(error)
  95. else
  96. callback()
  97. else
  98. callback()
  99. _writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
  100. ResourceWriter.checkPath basePath, resource.path, (error, path) ->
  101. return callback(error) if error?
  102. mkdirp Path.dirname(path), (error) ->
  103. return callback(error) if error?
  104. # TODO: Don't overwrite file if it hasn't been modified
  105. if resource.url?
  106. UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)->
  107. if err?
  108. logger.err err:err, project_id:project_id, path:path, resource_url:resource.url, modified:resource.modified, "error downloading file for resources"
  109. callback() #try and continue compiling even if http resource can not be downloaded at this time
  110. else
  111. fs.writeFile path, resource.content, callback
  112. checkPath: (basePath, resourcePath, callback) ->
  113. path = Path.normalize(Path.join(basePath, resourcePath))
  114. if (path.slice(0, basePath.length + 1) != basePath + "/")
  115. return callback new Error("resource path is outside root directory")
  116. else
  117. return callback(null, path)