Merge pull request #19 from EDP-Sciences/master

Add a PersistorManager library using the aws-sdk library

services/filestore/.gitignore

@@ -40,6 +40,7 @@ test/IntergrationTests/js/*
 data/*/*
 
 app.js
+cluster.js
 app/js/*
 test/IntergrationTests/js/*
 test/UnitTests/js/*

services/filestore/app/coffee/AWSSDKPersistorManager.coffee

@@ -0,0 +1,87 @@
+logger = require "logger-sharelatex"
+aws = require "aws-sdk"
+_ = require "underscore"
+fs = require "fs"
+Errors = require "./Errors"
+
+s3 = new aws.S3()
+
+module.exports =
+	sendFile: (bucketName, key, fsPath, callback)->
+		logger.log bucketName:bucketName, key:key, "send file data to s3"
+		stream = fs.createReadStream fsPath
+		s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) ->
+			if err?
+				logger.err err: err, Bucket: bucketName, Key: key, "error sending file data to s3"
+			callback err
+
+	sendStream: (bucketName, key, stream, callback)->
+		logger.log bucketName:bucketName, key:key, "send file stream to s3"
+		s3.upload Bucket: bucketName, Key: key, Body: stream, (err, data) ->
+			if err?
+				logger.err err: err, Bucket: bucketName, Key: key, "error sending file stream to s3"
+			callback err
+
+	getFileStream: (bucketName, key, opts, callback = (err, res)->)->
+		logger.log bucketName:bucketName, key:key, "get file stream from s3"
+		callback = _.once callback
+		params =
+			Bucket:bucketName
+			Key: key
+		if opts.start? and opts.end?
+			params['Range'] = "bytes=#{opts.start}-#{opts.end}"
+		request = s3.getObject params
+		stream = request.createReadStream()
+		stream.on 'readable', () ->
+			callback null, stream
+		stream.on 'error', (err) ->
+			logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3"
+			if err.code == 'NoSuchKey'
+				return callback new Errors.NotFoundError "File not found in S3: #{bucketName}:#{key}"
+			callback err
+
+	copyFile: (bucketName, sourceKey, destKey, callback)->
+		logger.log bucketName:bucketName, sourceKey:sourceKey, destKey: destKey, "copying file in s3"
+		source = bucketName + '/' + sourceKey
+		s3.copyObject {Bucket: bucketName, Key: destKey, CopySource: source}, (err) ->
+			if err?
+				logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in s3"
+			callback err
+
+	deleteFile: (bucketName, key, callback)->
+		logger.log bucketName:bucketName, key:key, "delete file in s3"
+		s3.deleteObject {Bucket: bucketName, Key: key}, (err) ->
+			if err?
+				logger.err err:err, bucketName:bucketName, key:key, "something went wrong deleting file in s3"
+			callback err
+
+	deleteDirectory: (bucketName, key, callback)->
+		logger.log bucketName:bucketName, key:key, "delete directory in s3"
+		s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) ->
+			if err?
+				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3"
+				return callback err
+			if data.Contents.length == 0
+				logger.log bucketName:bucketName, key:key, "the directory is empty"
+				return callback()
+			keys = _.map data.Contents, (entry)->
+				Key: entry.Key
+			s3.deleteObjects
+				Bucket: bucketName
+				Delete:
+					Objects: keys
+					Quiet: true
+			, (err) ->
+					if err?
+						logger.err err:err, bucketName:bucketName, key:keys, "something went wrong deleting directory in s3"
+					callback err
+
+	checkIfFileExists:(bucketName, key, callback)->
+		logger.log bucketName:bucketName, key:key, "check file existence in s3"
+		s3.headObject {Bucket: bucketName, Key: key}, (err, data) ->
+			if err?
+				return (callback null, false) if err.code == 'NotFound'
+				logger.err err:err, bucketName:bucketName, key:key, "something went wrong checking head in s3"
+				return callback err
+			callback null, data.ETag?
+

services/filestore/app/coffee/FSPersistorManager.coffee

@@ -3,6 +3,7 @@ fs = require("fs")
 LocalFileWriter = require("./LocalFileWriter")
 Errors = require('./Errors')
 rimraf = require("rimraf")
+_ = require "underscore"
 
 filterName = (key) ->
   return key.replace /\//g, "_"
@@ -29,9 +30,7 @@ module.exports =
 
   # opts may be {start: Number, end: Number}
   getFileStream: (location, name, opts, _callback = (err, res)->) ->
-    callback = (args...) ->
-      _callback(args...)
-      _callback = () ->
+    callback = _.once _callback
     filteredName = filterName name
     logger.log location:location, name:filteredName, "getting file"
     sourceStream = fs.createReadStream "#{location}/#{filteredName}", opts

services/filestore/app/coffee/FileHandler.coffee

@@ -10,16 +10,16 @@ ImageOptimiser = require("./ImageOptimiser")
 module.exports =
 
 	insertFile: (bucket, key, stream, callback)->
-		convetedKey = KeyBuilder.getConvertedFolderKey(key)
-		PersistorManager.deleteDirectory bucket, convetedKey, (error) ->
+		convertedKey = KeyBuilder.getConvertedFolderKey key
+		PersistorManager.deleteDirectory bucket, convertedKey, (error) ->
 			return callback(error) if error?
 			PersistorManager.sendStream bucket, key, stream, callback
 
 	deleteFile: (bucket, key, callback)->
-		convetedKey = KeyBuilder.getConvertedFolderKey(key)
+		convertedKey = KeyBuilder.getConvertedFolderKey key
 		async.parallel [
 			(done)-> PersistorManager.deleteFile bucket, key, done
-			(done)-> PersistorManager.deleteDirectory bucket, convetedKey, done
+			(done)-> PersistorManager.deleteDirectory bucket, convertedKey, done
 		], callback
 
 	getFile: (bucket, key, opts = {}, callback)->
@@ -36,49 +36,48 @@ module.exports =
 			callback err, fileStream
 
 	_getConvertedFile: (bucket, key, opts, callback)->
-		convetedKey = KeyBuilder.addCachingToKey(key, opts)
-		PersistorManager.checkIfFileExists bucket, convetedKey, (err, exists)=>
+		convertedKey = KeyBuilder.addCachingToKey key, opts
+		PersistorManager.checkIfFileExists bucket, convertedKey, (err, exists)=>
 			if err?
-				return callback(err)
+				return callback err
 			if exists
-				PersistorManager.getFileStream bucket, convetedKey, opts, callback
+				PersistorManager.getFileStream bucket, convertedKey, opts, callback
 			else
-				@_getConvertedFileAndCache bucket, key, convetedKey, opts, callback
+				@_getConvertedFileAndCache bucket, key, convertedKey, opts, callback
 
-	_getConvertedFileAndCache: (bucket, key, convetedKey, opts, callback)->
-		self = @
+	_getConvertedFileAndCache: (bucket, key, convertedKey, opts, callback)->
 		convertedFsPath = ""
 		async.series [
-			(cb)->
-				self._convertFile bucket, key, opts, (err, fileSystemPath)->
+			(cb) =>
+				@_convertFile bucket, key, opts, (err, fileSystemPath) ->
 					convertedFsPath = fileSystemPath
 					cb err
 			(cb)->
 				ImageOptimiser.compressPng convertedFsPath, cb
 			(cb)->
-				PersistorManager.sendFile bucket, convetedKey, convertedFsPath, cb
+				PersistorManager.sendFile bucket, convertedKey, convertedFsPath, cb
 		], (err)->
 			if err?
 				return callback(err)
-			PersistorManager.getFileStream bucket, convetedKey, opts, callback
+			PersistorManager.getFileStream bucket, convertedKey, opts, callback
 
-	_convertFile: (bucket, origonalKey, opts, callback)->
-		@_writeS3FileToDisk bucket, origonalKey, opts, (err, origonalFsPath)->
+	_convertFile: (bucket, originalKey, opts, callback)->
+		@_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)->
 			if err?
 				return callback(err)
 			done = (err, destPath)->
 				if err?
-					logger.err err:err, bucket:bucket, origonalKey:origonalKey, opts:opts, "error converting file"
+					logger.err err:err, bucket:bucket, originalKey:originalKey, opts:opts, "error converting file"
 					return callback(err)
-				LocalFileWriter.deleteFile origonalFsPath, ->
+				LocalFileWriter.deleteFile originalFsPath, ->
 				callback(err, destPath)
 
 			if opts.format?
-				FileConverter.convert origonalFsPath, opts.format, done
+				FileConverter.convert originalFsPath, opts.format, done
 			else if opts.style == "thumbnail"
-				FileConverter.thumbnail origonalFsPath, done
+				FileConverter.thumbnail originalFsPath, done
 			else if opts.style == "preview"
-				FileConverter.preview origonalFsPath, done
+				FileConverter.preview originalFsPath, done
 			else
 				return callback(new Error("should have specified opts to convert file with #{JSON.stringify(opts)}"))
 

services/filestore/app/coffee/PersistorManager.coffee

@@ -7,6 +7,8 @@ settings.filestore.backend ||= "s3"
 
 logger.log backend:settings.filestore.backend, "Loading backend"
 module.exports = switch settings.filestore.backend
+	when "aws-sdk"
+		require "./AWSSDKPersistorManager"
 	when "s3"
 		require("./S3PersistorManager")
 	when "fs"

services/filestore/package.json

@@ -8,17 +8,12 @@
   },
   "dependencies": {
     "async": "~0.2.10",
-    "bunyan": "^1.3.5",
+    "aws-sdk": "^2.1.39",
     "coffee-script": "~1.7.1",
     "express": "~3.4.8",
-    "grunt-bunyan": "^0.5.0",
-    "grunt-execute": "^0.2.2",
-    "grunt-mocha-test": "~0.8.2",
     "heapdump": "^0.3.2",
     "knox": "~0.9.1",
     "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0",
-    "longjohn": "~0.2.2",
-    "lynx": "0.0.11",
     "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.3.0",
     "node-transloadit": "0.0.4",
     "node-uuid": "~1.4.1",
@@ -36,7 +31,11 @@
     "sinon": "",
     "chai": "",
     "sandboxed-module": "",
+    "bunyan": "^1.3.5",
     "grunt": "0.4.1",
+    "grunt-bunyan": "^0.5.0",
+    "grunt-execute": "^0.2.2",
+    "grunt-mocha-test": "~0.8.2",
     "grunt-contrib-requirejs": "0.4.1",
     "grunt-contrib-coffee": "0.7.0",
     "grunt-contrib-watch": "0.5.3",

services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee

@@ -0,0 +1,249 @@
+sinon = require 'sinon'
+chai = require 'chai'
+
+should = chai.should()
+expect = chai.expect
+
+modulePath = "../../../app/js/AWSSDKPersistorManager.js"
+SandboxedModule = require 'sandboxed-module'
+
+describe "AWSSDKPersistorManager", ->
+	beforeEach ->
+		@settings =
+			filestore:
+				backend: "aws-sdk"
+		@s3 =
+			upload: sinon.stub()
+			getObject: sinon.stub()
+			copyObject: sinon.stub()
+			deleteObject: sinon.stub()
+			listObjects: sinon.stub()
+			deleteObjects: sinon.stub()
+			headObject: sinon.stub()
+		@awssdk =
+			S3: sinon.stub().returns @s3
+
+		@requires =
+			"aws-sdk": @awssdk
+			"settings-sharelatex": @settings
+			"logger-sharelatex":
+				log:->
+				err:->
+			"fs": @fs =
+				createReadStream: sinon.stub()
+			"./Errors": @Errors =
+				NotFoundError: sinon.stub()
+		@key = "my/key"
+		@bucketName = "my-bucket"
+		@error = "my error"
+		@AWSSDKPersistorManager = SandboxedModule.require modulePath, requires: @requires
+
+	describe "sendFile", ->
+		beforeEach ->
+			@stream = {}
+			@fsPath = "/usr/local/some/file"
+			@fs.createReadStream.returns @stream
+
+		it "should put the file with s3.upload", (done) ->
+			@s3.upload.callsArgWith 1
+			@AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) =>
+				expect(err).to.not.be.ok
+				expect(@s3.upload.calledOnce, "called only once").to.be.true
+				expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream)
+							 , "called with correct arguments").to.be.true
+				done()
+
+		it "should dispatch the error from s3.upload", (done) ->
+			@s3.upload.callsArgWith 1, @error
+			@AWSSDKPersistorManager.sendFile @bucketName, @key, @fsPath, (err) =>
+				expect(err).to.equal @error
+				done()
+
+
+	describe "sendStream", ->
+		beforeEach ->
+			@stream = {}
+
+		it "should put the file with s3.upload", (done) ->
+			@s3.upload.callsArgWith 1
+			@AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) =>
+				expect(err).to.not.be.ok
+				expect(@s3.upload.calledOnce, "called only once").to.be.true
+				expect((@s3.upload.calledWith Bucket: @bucketName, Key: @key, Body: @stream),
+							 "called with correct arguments").to.be.true
+				done()
+
+		it "should dispatch the error from s3.upload", (done) ->
+			@s3.upload.callsArgWith 1, @error
+			@AWSSDKPersistorManager.sendStream @bucketName, @key, @stream, (err) =>
+				expect(err).to.equal @error
+				done()
+
+	describe "getFileStream", ->
+		beforeEach ->
+			@opts = {}
+			@stream = {}
+			@read_stream =
+				on: @read_stream_on = sinon.stub()
+			@object =
+				createReadStream: sinon.stub().returns @read_stream
+			@s3.getObject.returns @object
+
+		it "should return a stream from s3.getObject", (done) ->
+			@read_stream_on.withArgs('readable').callsArgWith 1
+
+			@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
+				expect(@read_stream_on.calledTwice)
+				expect(err).to.not.be.ok
+				expect(stream, "returned the stream").to.equal @read_stream
+				expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key),
+							 "called with correct arguments").to.be.true
+				done()
+
+		describe "with start and end options", ->
+			beforeEach ->
+				@opts =
+					start: 0
+					end: 8
+			it "should pass headers to the s3.GetObject", (done) ->
+				@read_stream_on.withArgs('readable').callsArgWith 1
+				@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
+					expect((@s3.getObject.calledWith Bucket: @bucketName, Key: @key, Range: 'bytes=0-8'),
+						"called with correct arguments").to.be.true
+				done()
+
+		describe "error conditions", ->
+			describe "when the file doesn't exist", ->
+				beforeEach ->
+					@error = new Error()
+					@error.code = 'NoSuchKey'
+				it "should produce a NotFoundError", (done) ->
+					@read_stream_on.withArgs('error').callsArgWith 1, @error
+					@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
+						expect(stream).to.not.be.ok
+						expect(err).to.be.ok
+						expect(err instanceof @Errors.NotFoundError, "error is a correct instance").to.equal true
+						done()
+
+			describe "when there is some other error", ->
+				beforeEach ->
+					@error = new Error()
+				it "should dispatch the error from s3 object stream", (done) ->
+					@read_stream_on.withArgs('error').callsArgWith 1, @error
+					@AWSSDKPersistorManager.getFileStream @bucketName, @key, @opts, (err, stream) =>
+						expect(stream).to.not.be.ok
+						expect(err).to.be.ok
+						expect(err).to.equal @error
+						done()
+
+	describe "copyFile", ->
+		beforeEach ->
+			@destKey = "some/key"
+			@stream = {}
+
+		it "should copy the file with s3.copyObject", (done) ->
+			@s3.copyObject.callsArgWith 1
+			@AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) =>
+				expect(err).to.not.be.ok
+				expect(@s3.copyObject.calledOnce, "called only once").to.be.true
+				expect((@s3.copyObject.calledWith Bucket: @bucketName, Key: @destKey, CopySource: @bucketName + '/' + @key),
+					"called with correct arguments").to.be.true
+				done()
+
+		it "should dispatch the error from s3.copyObject", (done) ->
+			@s3.copyObject.callsArgWith 1, @error
+			@AWSSDKPersistorManager.copyFile @bucketName, @key, @destKey, (err) =>
+				expect(err).to.equal @error
+				done()
+
+	describe "deleteFile", ->
+		it "should delete the file with s3.deleteObject", (done) ->
+			@s3.deleteObject.callsArgWith 1
+			@AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) =>
+				expect(err).to.not.be.ok
+				expect(@s3.deleteObject.calledOnce, "called only once").to.be.true
+				expect((@s3.deleteObject.calledWith Bucket: @bucketName, Key: @key),
+					"called with correct arguments").to.be.true
+				done()
+
+		it "should dispatch the error from s3.deleteObject", (done) ->
+			@s3.deleteObject.callsArgWith 1, @error
+			@AWSSDKPersistorManager.deleteFile @bucketName, @key, (err) =>
+				expect(err).to.equal @error
+				done()
+
+	describe "deleteDirectory", ->
+
+		it "should list the directory content using s3.listObjects", (done) ->
+			@s3.listObjects.callsArgWith 1, null, Contents: []
+			@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
+				expect(err).to.not.be.ok
+				expect(@s3.listObjects.calledOnce, "called only once").to.be.true
+				expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key),
+					"called with correct arguments").to.be.true
+				done()
+
+		it "should dispatch the error from s3.listObjects", (done) ->
+			@s3.listObjects.callsArgWith 1, @error
+			@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
+				expect(err).to.equal @error
+				done()
+
+		describe "with directory content", ->
+			beforeEach ->
+				@fileList = [
+					Key: 'foo'
+				, Key: 'bar'
+				, Key: 'baz'
+				]
+
+			it "should forward the file keys to s3.deleteObjects", (done) ->
+				@s3.listObjects.callsArgWith 1, null, Contents: @fileList
+				@s3.deleteObjects.callsArgWith 1
+				@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
+					expect(err).to.not.be.ok
+					expect(@s3.deleteObjects.calledOnce, "called only once").to.be.true
+					expect((@s3.deleteObjects.calledWith
+							Bucket: @bucketName
+							Delete:
+								Quiet: true
+								Objects: @fileList),
+						"called with correct arguments").to.be.true
+					done()
+
+			it "should dispatch the error from s3.deleteObjects", (done) ->
+				@s3.listObjects.callsArgWith 1, null, Contents: @fileList
+				@s3.deleteObjects.callsArgWith 1, @error
+				@AWSSDKPersistorManager.deleteDirectory @bucketName, @key, (err) =>
+					expect(err).to.equal @error
+					done()
+
+
+	describe "checkIfFileExists", ->
+
+		it "should check for the file with s3.headObject", (done) ->
+			@s3.headObject.callsArgWith 1, null, {}
+			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
+				expect(err).to.not.be.ok
+				expect(@s3.headObject.calledOnce, "called only once").to.be.true
+				expect((@s3.headObject.calledWith Bucket: @bucketName, Key: @key),
+					"called with correct arguments").to.be.true
+				done()
+
+		it "should return false on an inexistant file", (done) ->
+			@s3.headObject.callsArgWith 1, null, {}
+			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
+				expect(exists).to.be.false
+				done()
+
+		it "should return true on an existing file", (done) ->
+			@s3.headObject.callsArgWith 1, null, ETag: "etag"
+			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
+				expect(exists).to.be.true
+				done()
+
+		it "should dispatch the error from s3.headObject", (done) ->
+			@s3.headObject.callsArgWith 1, @error
+			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
+				expect(err).to.equal @error
+				done()