DocArchiveManager.coffee 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. MongoManager = require "./MongoManager"
  2. Errors = require "./Errors"
  3. logger = require "logger-sharelatex"
  4. _ = require "underscore"
  5. async = require "async"
  6. settings = require("settings-sharelatex")
  7. request = require("request")
  8. crypto = require("crypto")
  9. thirtySeconds = 30 * 1000
  10. module.exports = DocArchive =
  11. archiveAllDocs: (project_id, callback = (err, docs) ->) ->
  12. MongoManager.getProjectsDocs project_id, {include_deleted: true}, {lines: true, rev: true, inS3: true}, (err, docs) ->
  13. if err?
  14. return callback(err)
  15. else if !docs?
  16. return callback new Errors.NotFoundError("No docs for project #{project_id}")
  17. docs = _.filter docs, (doc)-> doc.inS3 != true
  18. jobs = _.map docs, (doc) ->
  19. (cb)->
  20. DocArchive.archiveDoc project_id, doc, cb
  21. async.parallelLimit jobs, 5, callback
  22. archiveDoc: (project_id, doc, callback)->
  23. logger.log project_id: project_id, doc_id: doc._id, "sending doc to s3"
  24. try
  25. options = DocArchive.buildS3Options(doc.lines, project_id+"/"+doc._id)
  26. catch e
  27. return callback e
  28. request.put options, (err, res)->
  29. if err? || res.statusCode != 200
  30. logger.err err:err, res:res, project_id:project_id, doc_id: doc._id, statusCode: res?.statusCode, "something went wrong archiving doc in aws"
  31. return callback new Error("Error in S3 request")
  32. md5lines = crypto.createHash("md5").update(JSON.stringify(doc.lines), "utf8").digest("hex")
  33. md5response = res.headers.etag.toString().replace(/\"/g, '')
  34. if md5lines != md5response
  35. logger.err responseMD5:md5response, linesMD5:md5lines, project_id:project_id, doc_id: doc?._id, "err in response md5 from s3"
  36. return callback new Error("Error in S3 md5 response")
  37. MongoManager.markDocAsArchived doc._id, doc.rev, (err) ->
  38. return callback(err) if err?
  39. callback()
  40. unArchiveAllDocs: (project_id, callback = (err) ->) ->
  41. MongoManager.getArchivedProjectDocs project_id, (err, docs) ->
  42. if err?
  43. logger.err err:err, project_id:project_id, "error unarchiving all docs"
  44. return callback(err)
  45. else if !docs?
  46. return callback new Errors.NotFoundError("No docs for project #{project_id}")
  47. jobs = _.map docs, (doc) ->
  48. (cb)->
  49. if !doc.inS3?
  50. return cb()
  51. else
  52. DocArchive.unarchiveDoc project_id, doc._id, cb
  53. async.parallelLimit jobs, 5, callback
  54. unarchiveDoc: (project_id, doc_id, callback)->
  55. logger.log project_id: project_id, doc_id: doc_id, "getting doc from s3"
  56. try
  57. options = DocArchive.buildS3Options(true, project_id+"/"+doc_id)
  58. catch e
  59. return callback e
  60. request.get options, (err, res, lines)->
  61. if err? || res.statusCode != 200
  62. logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong unarchiving doc from aws"
  63. return callback new Errors.NotFoundError("Error in S3 request")
  64. MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), {lines}, (err) ->
  65. return callback(err) if err?
  66. logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3"
  67. request.del options, (err, res, body)->
  68. if err? || res.statusCode != 204
  69. logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws"
  70. return callback new Errors.NotFoundError("Error in S3 request")
  71. callback()
  72. buildS3Options: (content, key)->
  73. if !settings.docstore.s3?
  74. throw new Error("S3 settings are not configured")
  75. return {
  76. aws:
  77. key: settings.docstore.s3.key
  78. secret: settings.docstore.s3.secret
  79. bucket: settings.docstore.s3.bucket
  80. timeout: thirtySeconds
  81. json: content
  82. uri:"https://#{settings.docstore.s3.bucket}.s3.amazonaws.com/#{key}"
  83. }