|
|
@@ -19,282 +19,358 @@
|
|
|
// to use aws-sdk throughout, see the comments in AWSSDKPersistorManager for
|
|
|
// details. The knox library is unmaintained and has bugs.
|
|
|
|
|
|
-const http = require('http');
|
|
|
-http.globalAgent.maxSockets = 300;
|
|
|
-const https = require('https');
|
|
|
-https.globalAgent.maxSockets = 300;
|
|
|
-const settings = require("settings-sharelatex");
|
|
|
-const request = require("request");
|
|
|
-const logger = require("logger-sharelatex");
|
|
|
-const metrics = require("metrics-sharelatex");
|
|
|
-const fs = require("fs");
|
|
|
-const knox = require("knox");
|
|
|
-const path = require("path");
|
|
|
-const LocalFileWriter = require("./LocalFileWriter");
|
|
|
-const Errors = require("./Errors");
|
|
|
-const _ = require("underscore");
|
|
|
-const awsS3 = require("aws-sdk/clients/s3");
|
|
|
-const URL = require('url');
|
|
|
+const http = require('http')
|
|
|
+http.globalAgent.maxSockets = 300
|
|
|
+const https = require('https')
|
|
|
+https.globalAgent.maxSockets = 300
|
|
|
+const settings = require('settings-sharelatex')
|
|
|
+const request = require('request')
|
|
|
+const logger = require('logger-sharelatex')
|
|
|
+const metrics = require('metrics-sharelatex')
|
|
|
+const fs = require('fs')
|
|
|
+const knox = require('knox')
|
|
|
+const path = require('path')
|
|
|
+const LocalFileWriter = require('./LocalFileWriter')
|
|
|
+const Errors = require('./Errors')
|
|
|
+const _ = require('underscore')
|
|
|
+const awsS3 = require('aws-sdk/clients/s3')
|
|
|
+const URL = require('url')
|
|
|
|
|
|
-const thirtySeconds = 30 * 1000;
|
|
|
+const thirtySeconds = 30 * 1000
|
|
|
|
|
|
-const buildDefaultOptions = function(bucketName, method, key){
|
|
|
- let endpoint;
|
|
|
- if (settings.filestore.s3.endpoint) {
|
|
|
- endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`;
|
|
|
- } else {
|
|
|
- endpoint = `https://${bucketName}.s3.amazonaws.com`;
|
|
|
- }
|
|
|
- return {
|
|
|
- aws: {
|
|
|
- key: settings.filestore.s3.key,
|
|
|
- secret: settings.filestore.s3.secret,
|
|
|
- bucket: bucketName
|
|
|
- },
|
|
|
- method,
|
|
|
- timeout: thirtySeconds,
|
|
|
- uri:`${endpoint}/${key}`
|
|
|
- };
|
|
|
-};
|
|
|
+const buildDefaultOptions = function(bucketName, method, key) {
|
|
|
+ let endpoint
|
|
|
+ if (settings.filestore.s3.endpoint) {
|
|
|
+ endpoint = `${settings.filestore.s3.endpoint}/${bucketName}`
|
|
|
+ } else {
|
|
|
+ endpoint = `https://${bucketName}.s3.amazonaws.com`
|
|
|
+ }
|
|
|
+ return {
|
|
|
+ aws: {
|
|
|
+ key: settings.filestore.s3.key,
|
|
|
+ secret: settings.filestore.s3.secret,
|
|
|
+ bucket: bucketName
|
|
|
+ },
|
|
|
+ method,
|
|
|
+ timeout: thirtySeconds,
|
|
|
+ uri: `${endpoint}/${key}`
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
const getS3Options = function(credentials) {
|
|
|
- const options = {
|
|
|
- credentials: {
|
|
|
- accessKeyId: credentials.auth_key,
|
|
|
- secretAccessKey: credentials.auth_secret
|
|
|
- }
|
|
|
- };
|
|
|
+ const options = {
|
|
|
+ credentials: {
|
|
|
+ accessKeyId: credentials.auth_key,
|
|
|
+ secretAccessKey: credentials.auth_secret
|
|
|
+ }
|
|
|
+ }
|
|
|
|
|
|
- if (settings.filestore.s3.endpoint) {
|
|
|
- const endpoint = URL.parse(settings.filestore.s3.endpoint);
|
|
|
- options.endpoint = settings.filestore.s3.endpoint;
|
|
|
- options.sslEnabled = endpoint.protocol === 'https';
|
|
|
- }
|
|
|
+ if (settings.filestore.s3.endpoint) {
|
|
|
+ const endpoint = URL.parse(settings.filestore.s3.endpoint)
|
|
|
+ options.endpoint = settings.filestore.s3.endpoint
|
|
|
+ options.sslEnabled = endpoint.protocol === 'https'
|
|
|
+ }
|
|
|
|
|
|
- return options;
|
|
|
-};
|
|
|
+ return options
|
|
|
+}
|
|
|
|
|
|
-const defaultS3Client = new awsS3(getS3Options({
|
|
|
- auth_key: settings.filestore.s3.key,
|
|
|
- auth_secret: settings.filestore.s3.secret
|
|
|
-}));
|
|
|
+const defaultS3Client = new awsS3(
|
|
|
+ getS3Options({
|
|
|
+ auth_key: settings.filestore.s3.key,
|
|
|
+ auth_secret: settings.filestore.s3.secret
|
|
|
+ })
|
|
|
+)
|
|
|
|
|
|
const getS3Client = function(credentials) {
|
|
|
- if (credentials != null) {
|
|
|
- return new awsS3(getS3Options(credentials));
|
|
|
- } else {
|
|
|
- return defaultS3Client;
|
|
|
- }
|
|
|
-};
|
|
|
+ if (credentials != null) {
|
|
|
+ return new awsS3(getS3Options(credentials))
|
|
|
+ } else {
|
|
|
+ return defaultS3Client
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
const getKnoxClient = bucketName => {
|
|
|
- const options = {
|
|
|
- key: settings.filestore.s3.key,
|
|
|
- secret: settings.filestore.s3.secret,
|
|
|
- bucket: bucketName
|
|
|
- };
|
|
|
- if (settings.filestore.s3.endpoint) {
|
|
|
- const endpoint = URL.parse(settings.filestore.s3.endpoint);
|
|
|
- options.endpoint = endpoint.hostname;
|
|
|
- options.port = endpoint.port;
|
|
|
- }
|
|
|
- return knox.createClient(options);
|
|
|
-};
|
|
|
+ const options = {
|
|
|
+ key: settings.filestore.s3.key,
|
|
|
+ secret: settings.filestore.s3.secret,
|
|
|
+ bucket: bucketName
|
|
|
+ }
|
|
|
+ if (settings.filestore.s3.endpoint) {
|
|
|
+ const endpoint = URL.parse(settings.filestore.s3.endpoint)
|
|
|
+ options.endpoint = endpoint.hostname
|
|
|
+ options.port = endpoint.port
|
|
|
+ }
|
|
|
+ return knox.createClient(options)
|
|
|
+}
|
|
|
|
|
|
module.exports = {
|
|
|
+ sendFile(bucketName, key, fsPath, callback) {
|
|
|
+ const s3Client = getKnoxClient(bucketName)
|
|
|
+ let uploaded = 0
|
|
|
+ const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res) {
|
|
|
+ metrics.count('s3.egress', uploaded)
|
|
|
+ if (err != null) {
|
|
|
+ logger.err(
|
|
|
+ { err, bucketName, key, fsPath },
|
|
|
+ 'something went wrong uploading file to s3'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ if (res == null) {
|
|
|
+ logger.err(
|
|
|
+ { err, res, bucketName, key, fsPath },
|
|
|
+ 'no response from s3 put file'
|
|
|
+ )
|
|
|
+ return callback('no response from put file')
|
|
|
+ }
|
|
|
+ if (res.statusCode !== 200) {
|
|
|
+ logger.err(
|
|
|
+ { bucketName, key, fsPath },
|
|
|
+ 'non 200 response from s3 putting file'
|
|
|
+ )
|
|
|
+ return callback('non 200 response from s3 on put file')
|
|
|
+ }
|
|
|
+ logger.log({ res, bucketName, key, fsPath }, 'file uploaded to s3')
|
|
|
+ return callback(err)
|
|
|
+ })
|
|
|
+ putEventEmiter.on('error', function(err) {
|
|
|
+ logger.err(
|
|
|
+ { err, bucketName, key, fsPath },
|
|
|
+ 'error emmited on put of file'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ })
|
|
|
+ return putEventEmiter.on(
|
|
|
+ 'progress',
|
|
|
+ progress => (uploaded = progress.written)
|
|
|
+ )
|
|
|
+ },
|
|
|
|
|
|
- sendFile(bucketName, key, fsPath, callback){
|
|
|
- const s3Client = getKnoxClient(bucketName);
|
|
|
- let uploaded = 0;
|
|
|
- const putEventEmiter = s3Client.putFile(fsPath, key, function(err, res){
|
|
|
- metrics.count('s3.egress', uploaded);
|
|
|
- if (err != null) {
|
|
|
- logger.err({err, bucketName, key, fsPath},"something went wrong uploading file to s3");
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- if ((res == null)) {
|
|
|
- logger.err({err, res, bucketName, key, fsPath}, "no response from s3 put file");
|
|
|
- return callback("no response from put file");
|
|
|
- }
|
|
|
- if (res.statusCode !== 200) {
|
|
|
- logger.err({bucketName, key, fsPath}, "non 200 response from s3 putting file");
|
|
|
- return callback("non 200 response from s3 on put file");
|
|
|
- }
|
|
|
- logger.log({res, bucketName, key, fsPath},"file uploaded to s3");
|
|
|
- return callback(err);
|
|
|
- });
|
|
|
- putEventEmiter.on("error", function(err){
|
|
|
- logger.err({err, bucketName, key, fsPath}, "error emmited on put of file");
|
|
|
- return callback(err);
|
|
|
- });
|
|
|
- return putEventEmiter.on("progress", progress => uploaded = progress.written);
|
|
|
- },
|
|
|
+ sendStream(bucketName, key, readStream, callback) {
|
|
|
+ logger.log({ bucketName, key }, 'sending file to s3')
|
|
|
+ readStream.on('error', err =>
|
|
|
+ logger.err({ bucketName, key }, 'error on stream to send to s3')
|
|
|
+ )
|
|
|
+ return LocalFileWriter.writeStream(readStream, null, (err, fsPath) => {
|
|
|
+ if (err != null) {
|
|
|
+ logger.err(
|
|
|
+ { bucketName, key, fsPath, err },
|
|
|
+ 'something went wrong writing stream to disk'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ return this.sendFile(bucketName, key, fsPath, (
|
|
|
+ err // delete the temporary file created above and return the original error
|
|
|
+ ) => LocalFileWriter.deleteFile(fsPath, () => callback(err)))
|
|
|
+ })
|
|
|
+ },
|
|
|
|
|
|
- sendStream(bucketName, key, readStream, callback){
|
|
|
- logger.log({bucketName, key}, "sending file to s3");
|
|
|
- readStream.on("error", err => logger.err({bucketName, key}, "error on stream to send to s3"));
|
|
|
- return LocalFileWriter.writeStream(readStream, null, (err, fsPath)=> {
|
|
|
- if (err != null) {
|
|
|
- logger.err({bucketName, key, fsPath, err}, "something went wrong writing stream to disk");
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- return this.sendFile(bucketName, key, fsPath, err => // delete the temporary file created above and return the original error
|
|
|
- LocalFileWriter.deleteFile(fsPath, () => callback(err)));
|
|
|
- });
|
|
|
- },
|
|
|
+ // opts may be {start: Number, end: Number}
|
|
|
+ getFileStream(bucketName, key, opts, callback) {
|
|
|
+ if (callback == null) {
|
|
|
+ callback = function(err, res) {}
|
|
|
+ }
|
|
|
+ opts = opts || {}
|
|
|
+ callback = _.once(callback)
|
|
|
+ logger.log({ bucketName, key }, 'getting file from s3')
|
|
|
|
|
|
- // opts may be {start: Number, end: Number}
|
|
|
- getFileStream(bucketName, key, opts, callback){
|
|
|
- if (callback == null) { callback = function(err, res){}; }
|
|
|
- opts = opts || {};
|
|
|
- callback = _.once(callback);
|
|
|
- logger.log({bucketName, key}, "getting file from s3");
|
|
|
+ const s3 = getS3Client(opts.credentials)
|
|
|
+ const s3Params = {
|
|
|
+ Bucket: bucketName,
|
|
|
+ Key: key
|
|
|
+ }
|
|
|
+ if (opts.start != null && opts.end != null) {
|
|
|
+ s3Params.Range = `bytes=${opts.start}-${opts.end}`
|
|
|
+ }
|
|
|
+ const s3Request = s3.getObject(s3Params)
|
|
|
|
|
|
- const s3 = getS3Client(opts.credentials);
|
|
|
- const s3Params = {
|
|
|
- Bucket: bucketName,
|
|
|
- Key: key
|
|
|
- };
|
|
|
- if ((opts.start != null) && (opts.end != null)) {
|
|
|
- s3Params.Range = `bytes=${opts.start}-${opts.end}`;
|
|
|
- }
|
|
|
- const s3Request = s3.getObject(s3Params);
|
|
|
+ s3Request.on(
|
|
|
+ 'httpHeaders',
|
|
|
+ (statusCode, headers, response, statusMessage) => {
|
|
|
+ if ([403, 404].includes(statusCode)) {
|
|
|
+ // S3 returns a 403 instead of a 404 when the user doesn't have
|
|
|
+ // permission to list the bucket contents.
|
|
|
+ logger.log({ bucketName, key }, 'file not found in s3')
|
|
|
+ return callback(
|
|
|
+ new Errors.NotFoundError(
|
|
|
+ `File not found in S3: ${bucketName}:${key}`
|
|
|
+ ),
|
|
|
+ null
|
|
|
+ )
|
|
|
+ }
|
|
|
+ if (![200, 206].includes(statusCode)) {
|
|
|
+ logger.log(
|
|
|
+ { bucketName, key },
|
|
|
+ `error getting file from s3: ${statusCode}`
|
|
|
+ )
|
|
|
+ return callback(
|
|
|
+ new Error(
|
|
|
+ `Got non-200 response from S3: ${statusCode} ${statusMessage}`
|
|
|
+ ),
|
|
|
+ null
|
|
|
+ )
|
|
|
+ }
|
|
|
+ const stream = response.httpResponse.createUnbufferedStream()
|
|
|
+ stream.on('data', data => metrics.count('s3.ingress', data.byteLength))
|
|
|
|
|
|
- s3Request.on('httpHeaders', (statusCode, headers, response, statusMessage) => {
|
|
|
- if ([403, 404].includes(statusCode)) {
|
|
|
- // S3 returns a 403 instead of a 404 when the user doesn't have
|
|
|
- // permission to list the bucket contents.
|
|
|
- logger.log({ bucketName, key }, "file not found in s3");
|
|
|
- return callback(new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`), null);
|
|
|
- }
|
|
|
- if (![200, 206].includes(statusCode)) {
|
|
|
- logger.log({bucketName, key }, `error getting file from s3: ${statusCode}`);
|
|
|
- return callback(new Error(`Got non-200 response from S3: ${statusCode} ${statusMessage}`), null);
|
|
|
- }
|
|
|
- const stream = response.httpResponse.createUnbufferedStream();
|
|
|
- stream.on('data', data => metrics.count('s3.ingress', data.byteLength));
|
|
|
+ return callback(null, stream)
|
|
|
+ }
|
|
|
+ )
|
|
|
|
|
|
- return callback(null, stream);
|
|
|
- });
|
|
|
+ s3Request.on('error', err => {
|
|
|
+ logger.err({ err, bucketName, key }, 'error getting file stream from s3')
|
|
|
+ return callback(err)
|
|
|
+ })
|
|
|
|
|
|
- s3Request.on('error', err => {
|
|
|
- logger.err({ err, bucketName, key }, "error getting file stream from s3");
|
|
|
- return callback(err);
|
|
|
- });
|
|
|
+ return s3Request.send()
|
|
|
+ },
|
|
|
|
|
|
- return s3Request.send();
|
|
|
- },
|
|
|
+ getFileSize(bucketName, key, callback) {
|
|
|
+ logger.log({ bucketName, key }, 'getting file size from S3')
|
|
|
+ const s3 = getS3Client()
|
|
|
+ return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
|
|
|
+ if (err != null) {
|
|
|
+ if ([403, 404].includes(err.statusCode)) {
|
|
|
+ // S3 returns a 403 instead of a 404 when the user doesn't have
|
|
|
+ // permission to list the bucket contents.
|
|
|
+ logger.log(
|
|
|
+ {
|
|
|
+ bucketName,
|
|
|
+ key
|
|
|
+ },
|
|
|
+ 'file not found in s3'
|
|
|
+ )
|
|
|
+ callback(
|
|
|
+ new Errors.NotFoundError(
|
|
|
+ `File not found in S3: ${bucketName}:${key}`
|
|
|
+ )
|
|
|
+ )
|
|
|
+ } else {
|
|
|
+ logger.err(
|
|
|
+ {
|
|
|
+ bucketName,
|
|
|
+ key,
|
|
|
+ err
|
|
|
+ },
|
|
|
+ 'error performing S3 HeadObject'
|
|
|
+ )
|
|
|
+ callback(err)
|
|
|
+ }
|
|
|
+ return
|
|
|
+ }
|
|
|
+ return callback(null, data.ContentLength)
|
|
|
+ })
|
|
|
+ },
|
|
|
|
|
|
- getFileSize(bucketName, key, callback) {
|
|
|
- logger.log({ bucketName, key }, "getting file size from S3");
|
|
|
- const s3 = getS3Client();
|
|
|
- return s3.headObject({ Bucket: bucketName, Key: key }, function(err, data) {
|
|
|
- if (err != null) {
|
|
|
- if ([403, 404].includes(err.statusCode)) {
|
|
|
- // S3 returns a 403 instead of a 404 when the user doesn't have
|
|
|
- // permission to list the bucket contents.
|
|
|
- logger.log({
|
|
|
- bucketName,
|
|
|
- key
|
|
|
- }, "file not found in s3");
|
|
|
- callback(
|
|
|
- new Errors.NotFoundError(`File not found in S3: ${bucketName}:${key}`)
|
|
|
- );
|
|
|
- } else {
|
|
|
- logger.err({
|
|
|
- bucketName,
|
|
|
- key,
|
|
|
- err
|
|
|
- }, "error performing S3 HeadObject");
|
|
|
- callback(err);
|
|
|
- }
|
|
|
- return;
|
|
|
- }
|
|
|
- return callback(null, data.ContentLength);
|
|
|
- });
|
|
|
- },
|
|
|
+ copyFile(bucketName, sourceKey, destKey, callback) {
|
|
|
+ logger.log({ bucketName, sourceKey, destKey }, 'copying file in s3')
|
|
|
+ const source = bucketName + '/' + sourceKey
|
|
|
+ // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
|
|
|
+ const s3 = getS3Client()
|
|
|
+ return s3.copyObject(
|
|
|
+ { Bucket: bucketName, Key: destKey, CopySource: source },
|
|
|
+ function(err) {
|
|
|
+ if (err != null) {
|
|
|
+ if (err.code === 'NoSuchKey') {
|
|
|
+ logger.err(
|
|
|
+ { bucketName, sourceKey },
|
|
|
+ 'original file not found in s3 when copying'
|
|
|
+ )
|
|
|
+ return callback(
|
|
|
+ new Errors.NotFoundError(
|
|
|
+ 'original file not found in S3 when copying'
|
|
|
+ )
|
|
|
+ )
|
|
|
+ } else {
|
|
|
+ logger.err(
|
|
|
+ { err, bucketName, sourceKey, destKey },
|
|
|
+ 'something went wrong copying file in aws'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ return callback()
|
|
|
+ }
|
|
|
+ }
|
|
|
+ )
|
|
|
+ },
|
|
|
|
|
|
- copyFile(bucketName, sourceKey, destKey, callback){
|
|
|
- logger.log({bucketName, sourceKey, destKey}, "copying file in s3");
|
|
|
- const source = bucketName + '/' + sourceKey;
|
|
|
- // use the AWS SDK instead of knox due to problems with error handling (https://github.com/Automattic/knox/issues/114)
|
|
|
- const s3 = getS3Client();
|
|
|
- return s3.copyObject({Bucket: bucketName, Key: destKey, CopySource: source}, function(err) {
|
|
|
- if (err != null) {
|
|
|
- if (err.code === 'NoSuchKey') {
|
|
|
- logger.err({bucketName, sourceKey}, "original file not found in s3 when copying");
|
|
|
- return callback(new Errors.NotFoundError("original file not found in S3 when copying"));
|
|
|
- } else {
|
|
|
- logger.err({err, bucketName, sourceKey, destKey}, "something went wrong copying file in aws");
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- } else {
|
|
|
- return callback();
|
|
|
- }
|
|
|
- });
|
|
|
- },
|
|
|
+ deleteFile(bucketName, key, callback) {
|
|
|
+ logger.log({ bucketName, key }, 'delete file in s3')
|
|
|
+ const options = buildDefaultOptions(bucketName, 'delete', key)
|
|
|
+ return request(options, function(err, res) {
|
|
|
+ if (err != null) {
|
|
|
+ logger.err(
|
|
|
+ { err, res, bucketName, key },
|
|
|
+ 'something went wrong deleting file in aws'
|
|
|
+ )
|
|
|
+ }
|
|
|
+ return callback(err)
|
|
|
+ })
|
|
|
+ },
|
|
|
|
|
|
- deleteFile(bucketName, key, callback){
|
|
|
- logger.log({bucketName, key}, "delete file in s3");
|
|
|
- const options = buildDefaultOptions(bucketName, "delete", key);
|
|
|
- return request(options, function(err, res){
|
|
|
- if (err != null) {
|
|
|
- logger.err({err, res, bucketName, key}, "something went wrong deleting file in aws");
|
|
|
- }
|
|
|
- return callback(err);
|
|
|
- });
|
|
|
- },
|
|
|
+ deleteDirectory(bucketName, key, _callback) {
|
|
|
+ // deleteMultiple can call the callback multiple times so protect against this.
|
|
|
+ const callback = function(...args) {
|
|
|
+ _callback(...Array.from(args || []))
|
|
|
+ return (_callback = function() {})
|
|
|
+ }
|
|
|
|
|
|
- deleteDirectory(bucketName, key, _callback){
|
|
|
- // deleteMultiple can call the callback multiple times so protect against this.
|
|
|
- const callback = function(...args) {
|
|
|
- _callback(...Array.from(args || []));
|
|
|
- return _callback = function() {};
|
|
|
- };
|
|
|
+ logger.log({ key, bucketName }, 'deleting directory')
|
|
|
+ const s3Client = getKnoxClient(bucketName)
|
|
|
+ return s3Client.list({ prefix: key }, function(err, data) {
|
|
|
+ if (err != null) {
|
|
|
+ logger.err(
|
|
|
+ { err, bucketName, key },
|
|
|
+ 'something went wrong listing prefix in aws'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ const keys = _.map(data.Contents, entry => entry.Key)
|
|
|
+ return s3Client.deleteMultiple(keys, callback)
|
|
|
+ })
|
|
|
+ },
|
|
|
|
|
|
- logger.log({key, bucketName}, "deleting directory");
|
|
|
- const s3Client = getKnoxClient(bucketName);
|
|
|
- return s3Client.list({prefix:key}, function(err, data){
|
|
|
- if (err != null) {
|
|
|
- logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- const keys = _.map(data.Contents, entry => entry.Key);
|
|
|
- return s3Client.deleteMultiple(keys, callback);
|
|
|
- });
|
|
|
- },
|
|
|
+ checkIfFileExists(bucketName, key, callback) {
|
|
|
+ logger.log({ bucketName, key }, 'checking if file exists in s3')
|
|
|
+ const options = buildDefaultOptions(bucketName, 'head', key)
|
|
|
+ return request(options, function(err, res) {
|
|
|
+ if (err != null) {
|
|
|
+ logger.err(
|
|
|
+ { err, res, bucketName, key },
|
|
|
+ 'something went wrong checking file in aws'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ if (res == null) {
|
|
|
+ logger.err(
|
|
|
+ { err, res, bucketName, key },
|
|
|
+ 'no response object returned when checking if file exists'
|
|
|
+ )
|
|
|
+ err = new Error(`no response from s3 ${bucketName} ${key}`)
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ const exists = res.statusCode === 200
|
|
|
+ logger.log({ bucketName, key, exists }, 'checked if file exsists in s3')
|
|
|
+ return callback(err, exists)
|
|
|
+ })
|
|
|
+ },
|
|
|
|
|
|
- checkIfFileExists(bucketName, key, callback){
|
|
|
- logger.log({bucketName, key}, "checking if file exists in s3");
|
|
|
- const options = buildDefaultOptions(bucketName, "head", key);
|
|
|
- return request(options, function(err, res){
|
|
|
- if (err != null) {
|
|
|
- logger.err({err, res, bucketName, key}, "something went wrong checking file in aws");
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- if ((res == null)) {
|
|
|
- logger.err({err, res, bucketName, key}, "no response object returned when checking if file exists");
|
|
|
- err = new Error(`no response from s3 ${bucketName} ${key}`);
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- const exists = res.statusCode === 200;
|
|
|
- logger.log({bucketName, key, exists}, "checked if file exsists in s3");
|
|
|
- return callback(err, exists);
|
|
|
- });
|
|
|
- },
|
|
|
-
|
|
|
- directorySize(bucketName, key, callback){
|
|
|
- logger.log({bucketName, key}, "get project size in s3");
|
|
|
- const s3Client = getKnoxClient(bucketName);
|
|
|
- return s3Client.list({prefix:key}, function(err, data){
|
|
|
- if (err != null) {
|
|
|
- logger.err({err, bucketName, key}, "something went wrong listing prefix in aws");
|
|
|
- return callback(err);
|
|
|
- }
|
|
|
- let totalSize = 0;
|
|
|
- _.each(data.Contents, entry => totalSize += entry.Size);
|
|
|
- logger.log({totalSize}, "total size");
|
|
|
- return callback(null, totalSize);
|
|
|
- });
|
|
|
- }
|
|
|
-};
|
|
|
+ directorySize(bucketName, key, callback) {
|
|
|
+ logger.log({ bucketName, key }, 'get project size in s3')
|
|
|
+ const s3Client = getKnoxClient(bucketName)
|
|
|
+ return s3Client.list({ prefix: key }, function(err, data) {
|
|
|
+ if (err != null) {
|
|
|
+ logger.err(
|
|
|
+ { err, bucketName, key },
|
|
|
+ 'something went wrong listing prefix in aws'
|
|
|
+ )
|
|
|
+ return callback(err)
|
|
|
+ }
|
|
|
+ let totalSize = 0
|
|
|
+ _.each(data.Contents, entry => (totalSize += entry.Size))
|
|
|
+ logger.log({ totalSize }, 'total size')
|
|
|
+ return callback(null, totalSize)
|
|
|
+ })
|
|
|
+ }
|
|
|
+}
|