ArchiveManager.coffee 4.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142
  1. logger = require "logger-sharelatex"
  2. metrics = require "metrics-sharelatex"
  3. fs = require "fs"
  4. Path = require "path"
  5. fse = require "fs-extra"
  6. yauzl = require "yauzl"
  7. Settings = require "settings-sharelatex"
  8. _ = require("underscore")
  9. ONE_MEG = 1024 * 1024
  10. module.exports = ArchiveManager =
  11. _isZipTooLarge: (source, callback = (err, isTooLarge)->)->
  12. callback = _.once callback
  13. totalSizeInBytes = null
  14. yauzl.open source, {lazyEntries: true}, (err, zipfile) ->
  15. return callback(err) if err?
  16. if Settings.maxEntitiesPerProject? and zipfile.entryCount > Settings.maxEntitiesPerProject
  17. return callback(null, true) # too many files in zip file
  18. zipfile.on "error", callback
  19. # read all the entries
  20. zipfile.readEntry()
  21. zipfile.on "entry", (entry) ->
  22. totalSizeInBytes += entry.uncompressedSize
  23. zipfile.readEntry() # get the next entry
  24. # no more entries to read
  25. zipfile.on "end", () ->
  26. if !totalSizeInBytes? or isNaN(totalSizeInBytes)
  27. logger.err source:source, totalSizeInBytes:totalSizeInBytes, "error getting bytes of zip"
  28. return callback(new Error("error getting bytes of zip"))
  29. isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
  30. callback(null, isTooLarge)
  31. _checkFilePath: (entry, destination, callback = (err, destFile) ->) ->
  32. # transform backslashes to forwardslashes to accommodate badly-behaved zip archives
  33. transformedFilename = entry.fileName.replace(/\\/g, '/')
  34. # check if the entry is a directory
  35. endsWithSlash = /\/$/
  36. if endsWithSlash.test(transformedFilename)
  37. return callback() # don't give a destfile for directory
  38. # check that the file does not use a relative path
  39. for dir in transformedFilename.split('/')
  40. if dir == '..'
  41. return callback(new Error("relative path"))
  42. # check that the destination file path is normalized
  43. dest = "#{destination}/#{transformedFilename}"
  44. if dest != Path.normalize(dest)
  45. return callback(new Error("unnormalized path"))
  46. else
  47. return callback(null, dest)
  48. _writeFileEntry: (zipfile, entry, destFile, callback = (err)->) ->
  49. callback = _.once callback
  50. zipfile.openReadStream entry, (err, readStream) ->
  51. return callback(err) if err?
  52. readStream.on "error", callback
  53. readStream.on "end", callback
  54. errorHandler = (err) -> # clean up before calling callback
  55. readStream.unpipe()
  56. readStream.destroy()
  57. callback(err)
  58. fse.ensureDir Path.dirname(destFile), (err) ->
  59. return errorHandler(err) if err?
  60. writeStream = fs.createWriteStream destFile
  61. writeStream.on 'error', errorHandler
  62. readStream.pipe(writeStream)
  63. _extractZipFiles: (source, destination, callback = (err) ->) ->
  64. callback = _.once callback
  65. yauzl.open source, {lazyEntries: true}, (err, zipfile) ->
  66. return callback(err) if err?
  67. zipfile.on "error", callback
  68. # read all the entries
  69. zipfile.readEntry()
  70. zipfile.on "entry", (entry) ->
  71. logger.log {source:source, fileName: entry.fileName}, "processing zip file entry"
  72. ArchiveManager._checkFilePath entry, destination, (err, destFile) ->
  73. if err?
  74. logger.warn err:err, source:source, destination:destination, "skipping bad file path"
  75. zipfile.readEntry() # bad path, just skip to the next file
  76. return
  77. if destFile? # only write files
  78. ArchiveManager._writeFileEntry zipfile, entry, destFile, (err) ->
  79. if err?
  80. logger.error err:err, source:source, destFile:destFile, "error unzipping file entry"
  81. zipfile.close() # bail out, stop reading file entries
  82. return callback(err)
  83. else
  84. zipfile.readEntry() # continue to the next file
  85. else # if it's a directory, continue
  86. zipfile.readEntry()
  87. # no more entries to read
  88. zipfile.on "end", callback
  89. extractZipArchive: (source, destination, _callback = (err) ->) ->
  90. callback = (args...) ->
  91. _callback(args...)
  92. _callback = () ->
  93. ArchiveManager._isZipTooLarge source, (err, isTooLarge)->
  94. if err?
  95. logger.err err:err, "error checking size of zip file"
  96. return callback(err)
  97. if isTooLarge
  98. return callback(new Error("zip_too_large"))
  99. timer = new metrics.Timer("unzipDirectory")
  100. logger.log source: source, destination: destination, "unzipping file"
  101. ArchiveManager._extractZipFiles source, destination, (err) ->
  102. timer.done()
  103. if err?
  104. logger.error {err, source, destination}, "unzip failed"
  105. callback(err)
  106. else
  107. callback()
  108. findTopLevelDirectory: (directory, callback = (error, topLevelDir) ->) ->
  109. fs.readdir directory, (error, files) ->
  110. return callback(error) if error?
  111. if files.length == 1
  112. childPath = Path.join(directory, files[0])
  113. fs.stat childPath, (error, stat) ->
  114. return callback(error) if error?
  115. if stat.isDirectory()
  116. return callback(null, childPath)
  117. else
  118. return callback(null, directory)
  119. else
  120. return callback(null, directory)