| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376 |
- /* eslint-disable
- handle-callback-err,
- new-cap,
- no-return-assign,
- no-unused-vars,
- node/no-deprecated-api,
- standard/no-callback-literal,
- */
- // TODO: This file was created by bulk-decaffeinate.
- // Fix any style issues and re-enable lint.
- /*
- * decaffeinate suggestions:
- * DS101: Remove unnecessary use of Array.from
- * DS102: Remove unnecessary code created because of implicit returns
- * DS207: Consider shorter variations of null checks
- * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
- */
- // This module is the one which is used in production. It needs to be migrated
- // to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
- // details. The knox library is unmaintained and has bugs.
- const http = require('http')
- http.globalAgent.maxSockets = 300
- const https = require('https')
- https.globalAgent.maxSockets = 300
- const settings = require('settings-sharelatex')
- const request = require('request')
- const logger = require('logger-sharelatex')
- const metrics = require('metrics-sharelatex')
- const fs = require('fs')
- const knox = require('knox')
- const path = require('path')
- const LocalFileWriter = require('./LocalFileWriter')
- const Errors = require('./Errors')
- const _ = require('underscore')
- const awsS3 = require('aws-sdk/clients/s3')
- const URL = require('url')
- const thirtySeconds = 30 * 1000
- const buildDefaultOptions = function(bucketName, method, key) {
- let endpoint
- if (settings.filestore.s3.endpoint) {
- endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`
- } else {
- endpoint = `https://${bucketName}.s3.amazonaws.com`
- }
- return {
- aws: {
- key: settings.filestore.s3.key,
- secret: settings.filestore.s3.secret,
- bucket: bucketName
- },
- method,
- timeout: thirtySeconds,
- uri: `${endpoint}/${key}`
- }
- }
- const getS3Options = function(credentials) {
- const options = {
- credentials: {
- accessKeyId: credentials.auth_key,
- secretAccessKey: credentials.auth_secret
- }
- }
- if (settings.filestore.s3.endpoint) {
- const endpoint = URL.parse(settings.filestore.s3.endpoint)
- options.endpoint = settings.filestore.s3.endpoint
- options.sslEnabled = endpoint.protocol === 'https'
- }
- return options
- }
- const defaultS3Client = new awsS3(
- getS3Options({
- auth_key: settings.filestore.s3.key,
- auth_secret: settings.filestore.s3.secret
- })
- )
- const getS3Client = function(credentials) {
- if (credentials != null) {
- return new awsS3(getS3Options(credentials))
- } else {
- return defaultS3Client
- }
- }
- const getKnoxClient = bucketName => {
- const options = {
- key: settings.filestore.s3.key,
- secret: settings.filestore.s3.secret,
- bucket: bucketName
- }
- if (settings.filestore.s3.endpoint) {
- const endpoint = URL.parse(settings.filestore.s3.endpoint)
- options.endpoint = endpoint.hostname
- options.port = endpoint.port
- }
- return knox.createClient(options)
- }
- module.exports = {
- sendFile(bucketName, key, fsPath, callback) {
- const s3Client = getKnoxClient(bucketName)
- let uploaded = 0
- const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) {
- metrics.count('s3.egress', uploaded)
- if (err != null) {
- logger.err(
- { err, bucketName, key, fsPath },
- 'something went wrong uploading file to s3'
- )
- return callback(err)
- }
- if (res == null) {
- logger.err(
- { err, res, bucketName, key, fsPath },
- 'no response from s3 put file'
- )
- return callback('no response from put file')
- }
- if (res.statusCode !== 200) {
- logger.err(
- { bucketName, key, fsPath },
- 'non 200 response from s3 putting file'
- )
- return callback('non 200 response from s3 on put file')
- }
- logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3')
- return callback(err)
- })
- putEventEmiter.on('error', function(err) {
- logger.err(
- { err, bucketName, key, fsPath },
- 'error emmited on put of file'
- )
- return callback(err)
- })
- return putEventEmiter.on(
- 'progress',
- progress => (uploaded = progress.written)
- )
- },
- sendStream(bucketName, key, readStream, callback) {
- logger.log({ bucketName, key }, 'sending file to s3')
- readStream.on('error', err =>
- logger.err({ bucketName, key }, 'error on stream to send to s3')
- )
- return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => {
- if (err != null) {
- logger.err(
- { bucketName, key, fsPath, err },
- 'something went wrong writing stream to disk'
- )
- return callback(err)
- }
- return this.sendFile(bucketName, key, fsPath, (
- err // delete the temporary file created above and return the original error
- ) => LocalFileWriter.deleteFile(fsPath, () => callback(err)))
- })
- },
- // opts may be {start: Number, end: Number}
- getFileStream(bucketName, key, opts, callback) {
- if (callback == null) {
- callback = function(err, res) {}
- }
- opts = opts || {}
- callback = _.once(callback)
- logger.log({ bucketName, key }, 'getting file from s3')
- const s3 = getS3Client(opts.credentials)
- const s3Params = {
- Bucket: bucketName,
- Key: key
- }
- if (opts.start != null && opts.end != null) {
- s3Params.Range = `bytes=${opts.start}-${opts.end}`
- }
- const s3Request = s3.getObject(s3Params)
- s3Request.on(
- 'httpHeaders',
- (statusCode, headers, response, statusMessage) => {
- if ([403, 404].includes(statusCode)) {
- // S3 returns a 403 instead of a 404 when the user doesn't have
- // permission to list the bucket contents.
- logger.log({ bucketName, key }, 'file not found in s3')
- return callback(
- new Errors.NotFoundError(
- `File not found in S3: ${bucketName}:${key}`
- ),
- null
- )
- }
- if (![200, 206].includes(statusCode)) {
- logger.log(
- { bucketName, key },
- `error getting file from s3: ${statusCode}`
- )
- return callback(
- new Error(
- `Got non-200 response from S3: ${statusCode} ${statusMessage}`
- ),
- null
- )
- }
- const stream = response.httpResponse.createUnbufferedStream()
- stream.on('data', data => metrics.count('s3.ingress', data.byteLength))
- return callback(null, stream)
- }
- )
- s3Request.on('error', err => {
- logger.err({ err, bucketName, key }, 'error getting file stream from s3')
- return callback(err)
- })
- return s3Request.send()
- },
- getFileSize(bucketName, key, callback) {
- logger.log({ bucketName, key }, 'getting file size from S3')
- const s3 = getS3Client()
- return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
- if (err != null) {
- if ([403, 404].includes(err.statusCode)) {
- // S3 returns a 403 instead of a 404 when the user doesn't have
- // permission to list the bucket contents.
- logger.log(
- {
- bucketName,
- key
- },
- 'file not found in s3'
- )
- callback(
- new Errors.NotFoundError(
- `File not found in S3: ${bucketName}:${key}`
- )
- )
- } else {
- logger.err(
- {
- bucketName,
- key,
- err
- },
- 'error performing S3 HeadObject'
- )
- callback(err)
- }
- return
- }
- return callback(null, data.ContentLength)
- })
- },
- copyFile(bucketName, sourceKey, destKey, callback) {
- logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
- const source = bucketName + '/' + sourceKey
- // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
- const s3 = getS3Client()
- return s3.copyObject(
- { Bucket: bucketName, Key: destKey, CopySource: source },
- function(err) {
- if (err != null) {
- if (err.code === 'NoSuchKey') {
- logger.err(
- { bucketName, sourceKey },
- 'original file not found in s3 when copying'
- )
- return callback(
- new Errors.NotFoundError(
- 'original file not found in S3 when copying'
- )
- )
- } else {
- logger.err(
- { err, bucketName, sourceKey, destKey },
- 'something went wrong copying file in aws'
- )
- return callback(err)
- }
- } else {
- return callback()
- }
- }
- )
- },
- deleteFile(bucketName, key, callback) {
- logger.log({ bucketName, key }, 'delete file in s3')
- const options = buildDefaultOptions(bucketName, 'delete', key)
- return request(options, function(err, res) {
- if (err != null) {
- logger.err(
- { err, res, bucketName, key },
- 'something went wrong deleting file in aws'
- )
- }
- return callback(err)
- })
- },
- deleteDirectory(bucketName, key, _callback) {
- // deleteMultiple can call the callback multiple times so protect against this.
- const callback = function(...args) {
- _callback(...Array.from(args || []))
- return (_callback = function() {})
- }
- logger.log({ key, bucketName }, 'deleting directory')
- const s3Client = getKnoxClient(bucketName)
- return s3Client.list({ prefix: key }, function(err, data) {
- if (err != null) {
- logger.err(
- { err, bucketName, key },
- 'something went wrong listing prefix in aws'
- )
- return callback(err)
- }
- const keys = _.map(data.Contents, entry => entry.Key)
- return s3Client.deleteMultiple(keys, callback)
- })
- },
- checkIfFileExists(bucketName, key, callback) {
- logger.log({ bucketName, key }, 'checking if file exists in s3')
- const options = buildDefaultOptions(bucketName, 'head', key)
- return request(options, function(err, res) {
- if (err != null) {
- logger.err(
- { err, res, bucketName, key },
- 'something went wrong checking file in aws'
- )
- return callback(err)
- }
- if (res == null) {
- logger.err(
- { err, res, bucketName, key },
- 'no response object returned when checking if file exists'
- )
- err = new Error(`no response from s3 ${bucketName} ${key}`)
- return callback(err)
- }
- const exists = res.statusCode === 200
- logger.log({ bucketName, key, exists }, 'checked if file exsists in s3')
- return callback(err, exists)
- })
- },
- directorySize(bucketName, key, callback) {
- logger.log({ bucketName, key }, 'get project size in s3')
- const s3Client = getKnoxClient(bucketName)
- return s3Client.list({ prefix: key }, function(err, data) {
- if (err != null) {
- logger.err(
- { err, bucketName, key },
- 'something went wrong listing prefix in aws'
- )
- return callback(err)
- }
- let totalSize = 0
- _.each(data.Contents, entry => (totalSize += entry.Size))
- logger.log({ totalSize }, 'total size')
- return callback(null, totalSize)
- })
- }
- }
|