S3PersistorManager.js 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376
  1. /* eslint-disable
  2. handle-callback-err,
  3. new-cap,
  4. no-return-assign,
  5. no-unused-vars,
  6. node/no-deprecated-api,
  7. standard/no-callback-literal,
  8. */
  9. // TODO: This file was created by bulk-decaffeinate.
  10. // Fix any style issues and re-enable lint.
  11. /*
  12. * decaffeinate suggestions:
  13. * DS101: Remove unnecessary use of Array.from
  14. * DS102: Remove unnecessary code created because of implicit returns
  15. * DS207: Consider shorter variations of null checks
  16. * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  17. */
  18. // This module is the one which is used in production. It needs to be migrated
  19. // to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
  20. // details. The knox library is unmaintained and has bugs.
  21. const http = require('http')
  22. http.globalAgent.maxSockets = 300
  23. const https = require('https')
  24. https.globalAgent.maxSockets = 300
  25. const settings = require('settings-sharelatex')
  26. const request = require('request')
  27. const logger = require('logger-sharelatex')
  28. const metrics = require('metrics-sharelatex')
  29. const fs = require('fs')
  30. const knox = require('knox')
  31. const path = require('path')
  32. const LocalFileWriter = require('./LocalFileWriter')
  33. const Errors = require('./Errors')
  34. const _ = require('underscore')
  35. const awsS3 = require('aws-sdk/clients/s3')
  36. const URL = require('url')
  37. const thirtySeconds = 30 * 1000
  38. const buildDefaultOptions = function(bucketName, method, key) {
  39. let endpoint
  40. if (settings.filestore.s3.endpoint) {
  41. endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`
  42. } else {
  43. endpoint = `https://${bucketName}.s3.amazonaws.com`
  44. }
  45. return {
  46. aws: {
  47. key: settings.filestore.s3.key,
  48. secret: settings.filestore.s3.secret,
  49. bucket: bucketName
  50. },
  51. method,
  52. timeout: thirtySeconds,
  53. uri: `${endpoint}/${key}`
  54. }
  55. }
  56. const getS3Options = function(credentials) {
  57. const options = {
  58. credentials: {
  59. accessKeyId: credentials.auth_key,
  60. secretAccessKey: credentials.auth_secret
  61. }
  62. }
  63. if (settings.filestore.s3.endpoint) {
  64. const endpoint = URL.parse(settings.filestore.s3.endpoint)
  65. options.endpoint = settings.filestore.s3.endpoint
  66. options.sslEnabled = endpoint.protocol === 'https'
  67. }
  68. return options
  69. }
  70. const defaultS3Client = new awsS3(
  71. getS3Options({
  72. auth_key: settings.filestore.s3.key,
  73. auth_secret: settings.filestore.s3.secret
  74. })
  75. )
  76. const getS3Client = function(credentials) {
  77. if (credentials != null) {
  78. return new awsS3(getS3Options(credentials))
  79. } else {
  80. return defaultS3Client
  81. }
  82. }
  83. const getKnoxClient = bucketName => {
  84. const options = {
  85. key: settings.filestore.s3.key,
  86. secret: settings.filestore.s3.secret,
  87. bucket: bucketName
  88. }
  89. if (settings.filestore.s3.endpoint) {
  90. const endpoint = URL.parse(settings.filestore.s3.endpoint)
  91. options.endpoint = endpoint.hostname
  92. options.port = endpoint.port
  93. }
  94. return knox.createClient(options)
  95. }
  96. module.exports = {
  97. sendFile(bucketName, key, fsPath, callback) {
  98. const s3Client = getKnoxClient(bucketName)
  99. let uploaded = 0
  100. const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) {
  101. metrics.count('s3.egress', uploaded)
  102. if (err != null) {
  103. logger.err(
  104. { err, bucketName, key, fsPath },
  105. 'something went wrong uploading file to s3'
  106. )
  107. return callback(err)
  108. }
  109. if (res == null) {
  110. logger.err(
  111. { err, res, bucketName, key, fsPath },
  112. 'no response from s3 put file'
  113. )
  114. return callback('no response from put file')
  115. }
  116. if (res.statusCode !== 200) {
  117. logger.err(
  118. { bucketName, key, fsPath },
  119. 'non 200 response from s3 putting file'
  120. )
  121. return callback('non 200 response from s3 on put file')
  122. }
  123. logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3')
  124. return callback(err)
  125. })
  126. putEventEmiter.on('error', function(err) {
  127. logger.err(
  128. { err, bucketName, key, fsPath },
  129. 'error emmited on put of file'
  130. )
  131. return callback(err)
  132. })
  133. return putEventEmiter.on(
  134. 'progress',
  135. progress => (uploaded = progress.written)
  136. )
  137. },
  138. sendStream(bucketName, key, readStream, callback) {
  139. logger.log({ bucketName, key }, 'sending file to s3')
  140. readStream.on('error', err =>
  141. logger.err({ bucketName, key }, 'error on stream to send to s3')
  142. )
  143. return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => {
  144. if (err != null) {
  145. logger.err(
  146. { bucketName, key, fsPath, err },
  147. 'something went wrong writing stream to disk'
  148. )
  149. return callback(err)
  150. }
  151. return this.sendFile(bucketName, key, fsPath, (
  152. err // delete the temporary file created above and return the original error
  153. ) => LocalFileWriter.deleteFile(fsPath, () => callback(err)))
  154. })
  155. },
  156. // opts may be {start: Number, end: Number}
  157. getFileStream(bucketName, key, opts, callback) {
  158. if (callback == null) {
  159. callback = function(err, res) {}
  160. }
  161. opts = opts || {}
  162. callback = _.once(callback)
  163. logger.log({ bucketName, key }, 'getting file from s3')
  164. const s3 = getS3Client(opts.credentials)
  165. const s3Params = {
  166. Bucket: bucketName,
  167. Key: key
  168. }
  169. if (opts.start != null && opts.end != null) {
  170. s3Params.Range = `bytes=${opts.start}-${opts.end}`
  171. }
  172. const s3Request = s3.getObject(s3Params)
  173. s3Request.on(
  174. 'httpHeaders',
  175. (statusCode, headers, response, statusMessage) => {
  176. if ([403, 404].includes(statusCode)) {
  177. // S3 returns a 403 instead of a 404 when the user doesn't have
  178. // permission to list the bucket contents.
  179. logger.log({ bucketName, key }, 'file not found in s3')
  180. return callback(
  181. new Errors.NotFoundError(
  182. `File not found in S3: ${bucketName}:${key}`
  183. ),
  184. null
  185. )
  186. }
  187. if (![200, 206].includes(statusCode)) {
  188. logger.log(
  189. { bucketName, key },
  190. `error getting file from s3: ${statusCode}`
  191. )
  192. return callback(
  193. new Error(
  194. `Got non-200 response from S3: ${statusCode} ${statusMessage}`
  195. ),
  196. null
  197. )
  198. }
  199. const stream = response.httpResponse.createUnbufferedStream()
  200. stream.on('data', data => metrics.count('s3.ingress', data.byteLength))
  201. return callback(null, stream)
  202. }
  203. )
  204. s3Request.on('error', err => {
  205. logger.err({ err, bucketName, key }, 'error getting file stream from s3')
  206. return callback(err)
  207. })
  208. return s3Request.send()
  209. },
  210. getFileSize(bucketName, key, callback) {
  211. logger.log({ bucketName, key }, 'getting file size from S3')
  212. const s3 = getS3Client()
  213. return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
  214. if (err != null) {
  215. if ([403, 404].includes(err.statusCode)) {
  216. // S3 returns a 403 instead of a 404 when the user doesn't have
  217. // permission to list the bucket contents.
  218. logger.log(
  219. {
  220. bucketName,
  221. key
  222. },
  223. 'file not found in s3'
  224. )
  225. callback(
  226. new Errors.NotFoundError(
  227. `File not found in S3: ${bucketName}:${key}`
  228. )
  229. )
  230. } else {
  231. logger.err(
  232. {
  233. bucketName,
  234. key,
  235. err
  236. },
  237. 'error performing S3 HeadObject'
  238. )
  239. callback(err)
  240. }
  241. return
  242. }
  243. return callback(null, data.ContentLength)
  244. })
  245. },
  246. copyFile(bucketName, sourceKey, destKey, callback) {
  247. logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
  248. const source = bucketName + '/' + sourceKey
  249. // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
  250. const s3 = getS3Client()
  251. return s3.copyObject(
  252. { Bucket: bucketName, Key: destKey, CopySource: source },
  253. function(err) {
  254. if (err != null) {
  255. if (err.code === 'NoSuchKey') {
  256. logger.err(
  257. { bucketName, sourceKey },
  258. 'original file not found in s3 when copying'
  259. )
  260. return callback(
  261. new Errors.NotFoundError(
  262. 'original file not found in S3 when copying'
  263. )
  264. )
  265. } else {
  266. logger.err(
  267. { err, bucketName, sourceKey, destKey },
  268. 'something went wrong copying file in aws'
  269. )
  270. return callback(err)
  271. }
  272. } else {
  273. return callback()
  274. }
  275. }
  276. )
  277. },
  278. deleteFile(bucketName, key, callback) {
  279. logger.log({ bucketName, key }, 'delete file in s3')
  280. const options = buildDefaultOptions(bucketName, 'delete', key)
  281. return request(options, function(err, res) {
  282. if (err != null) {
  283. logger.err(
  284. { err, res, bucketName, key },
  285. 'something went wrong deleting file in aws'
  286. )
  287. }
  288. return callback(err)
  289. })
  290. },
  291. deleteDirectory(bucketName, key, _callback) {
  292. // deleteMultiple can call the callback multiple times so protect against this.
  293. const callback = function(...args) {
  294. _callback(...Array.from(args || []))
  295. return (_callback = function() {})
  296. }
  297. logger.log({ key, bucketName }, 'deleting directory')
  298. const s3Client = getKnoxClient(bucketName)
  299. return s3Client.list({ prefix: key }, function(err, data) {
  300. if (err != null) {
  301. logger.err(
  302. { err, bucketName, key },
  303. 'something went wrong listing prefix in aws'
  304. )
  305. return callback(err)
  306. }
  307. const keys = _.map(data.Contents, entry => entry.Key)
  308. return s3Client.deleteMultiple(keys, callback)
  309. })
  310. },
  311. checkIfFileExists(bucketName, key, callback) {
  312. logger.log({ bucketName, key }, 'checking if file exists in s3')
  313. const options = buildDefaultOptions(bucketName, 'head', key)
  314. return request(options, function(err, res) {
  315. if (err != null) {
  316. logger.err(
  317. { err, res, bucketName, key },
  318. 'something went wrong checking file in aws'
  319. )
  320. return callback(err)
  321. }
  322. if (res == null) {
  323. logger.err(
  324. { err, res, bucketName, key },
  325. 'no response object returned when checking if file exists'
  326. )
  327. err = new Error(`no response from s3 ${bucketName} ${key}`)
  328. return callback(err)
  329. }
  330. const exists = res.statusCode === 200
  331. logger.log({ bucketName, key, exists }, 'checked if file exsists in s3')
  332. return callback(err, exists)
  333. })
  334. },
  335. directorySize(bucketName, key, callback) {
  336. logger.log({ bucketName, key }, 'get project size in s3')
  337. const s3Client = getKnoxClient(bucketName)
  338. return s3Client.list({ prefix: key }, function(err, data) {
  339. if (err != null) {
  340. logger.err(
  341. { err, bucketName, key },
  342. 'something went wrong listing prefix in aws'
  343. )
  344. return callback(err)
  345. }
  346. let totalSize = 0
  347. _.each(data.Contents, entry => (totalSize += entry.Size))
  348. logger.log({ totalSize }, 'total size')
  349. return callback(null, totalSize)
  350. })
  351. }
  352. }