Merge pull request #20 from heukirne/projectsize

Project Size endpoint added

services/filestore/app.coffee

@@ -85,6 +85,8 @@ app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey
 app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, express.bodyParser(), fileController.copyFile
 app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile
 
+app.get "/project/:project_id/size", keyBuilder.publicProjectKey, fileController.directorySize
+
 app.get "/heapdump", (req, res)->
 	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)->
 		res.send filename

services/filestore/app/coffee/AWSSDKPersistorManager.coffee

@@ -85,3 +85,17 @@ module.exports =
 				return callback err
 			callback null, data.ETag?
 
+	directorySize:(bucketName, key, callback)->
+		logger.log bucketName:bucketName, key:key, "get project size in s3"
+		s3.listObjects {Bucket: bucketName, Prefix: key}, (err, data) ->
+			if err?
+				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in s3"
+				return callback err
+			if data.Contents.length == 0
+				logger.log bucketName:bucketName, key:key, "the directory is empty"
+				return callback()
+			totalSize = 0
+			_.each data.Contents, (entry)->
+				totalSize += entry.Size
+			callback null, totalSize
+

services/filestore/app/coffee/FSPersistorManager.coffee

@@ -87,3 +87,19 @@ module.exports =
     fs.exists "#{location}/#{filteredName}", (exists) ->
       logger.log location:location, name:filteredName, exists:exists, "checked if file exists"
       callback null, exists
+
+  directorySize:(location, name, callback)->
+    filteredName = filterName name.replace(/\/$/,'')
+    logger.log location:location, name:filteredName, "get project size in file system"
+    fs.readdir "#{location}/#{filteredName}", (err, files) ->
+      if err?
+        logger.err err:err, location:location, name:filteredName, "something went wrong listing prefix in aws"
+        return callback(err)
+      totalSize = 0
+      _.each files, (entry)->
+        fd = fs.openSync "#{location}/#{filteredName}/#{entry}", 'r'
+        fileStats = fs.fstatSync(fd)
+        totalSize += fileStats.size
+        fs.closeSync fd
+      logger.log totalSize:totalSize, "total size", files:files
+      callback null, totalSize

services/filestore/app/coffee/FileController.coffee

@@ -83,3 +83,14 @@ module.exports = FileController =
 		else
 			range = parsed[0]
 			{start: range.start, end: range.end}
+
+	directorySize: (req, res)->
+		metrics.inc "projectSize"
+		{project_id, bucket} = req
+		logger.log project_id:project_id, bucket:bucket, "reciving request to project size"
+		FileHandler.getDirectorySize bucket, project_id, (err, size)->
+			if err?
+				logger.log err: err, project_id: project_id, bucket: bucket, "error inserting file"
+				res.send 500
+			else
+				res.json {'total bytes' : size}

services/filestore/app/coffee/FileHandler.coffee

@@ -87,3 +87,10 @@ module.exports =
 			if err?
 				return callback(err)
 			LocalFileWriter.writeStream fileStream, key, callback
+
+	getDirectorySize: (bucket, project_id, callback)->
+		logger.log bucket:bucket, project_id:project_id, "getting project size"
+		PersistorManager.directorySize bucket, project_id, (err, size)->
+			if err?
+				logger.err  bucket:bucket, project_id:project_id, "error getting size"
+			callback err, size

services/filestore/app/coffee/KeyBuilder.coffee

@@ -42,4 +42,9 @@ module.exports =
 		opts = req.query
 		next()
 
+	publicProjectKey: (req, res, next)->
+		{project_id} = req.params
+		req.project_id = project_id
+		req.bucket = settings.filestore.stores.user_files
+		next()
 	

services/filestore/app/coffee/S3PersistorManager.coffee

@@ -137,3 +137,19 @@ module.exports =
 			exists = res.statusCode == 200
 			logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3"
 			callback(err, exists)
+
+	directorySize:(bucketName, key, callback)->
+		logger.log bucketName:bucketName, key:key, "get project size in s3"
+		s3Client = knox.createClient
+			key: settings.filestore.s3.key
+			secret: settings.filestore.s3.secret
+			bucket: bucketName
+		s3Client.list prefix:key, (err, data)->
+			if err?
+				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws"
+				return callback(err)
+			totalSize = 0
+			_.each data.Contents, (entry)->
+				totalSize += entry.Size
+			logger.log totalSize:totalSize, "total size"
+			callback null, totalSize

services/filestore/test/unit/coffee/AWSSDKPersistorManagerTests.coffee

@@ -247,3 +247,26 @@ describe "AWSSDKPersistorManager", ->
 			@AWSSDKPersistorManager.checkIfFileExists @bucketName, @key, (err, exists) =>
 				expect(err).to.equal @error
 				done()
+
+	describe "directorySize", ->
+
+		it "should list the directory content using s3.listObjects", (done) ->
+			@s3.listObjects.callsArgWith 1, null, Contents: []
+			@AWSSDKPersistorManager.directorySize @bucketName, @key, (err) =>
+				expect(err).to.not.be.ok
+				expect(@s3.listObjects.calledOnce, "called only once").to.be.true
+				expect((@s3.listObjects.calledWith Bucket: @bucketName, Prefix: @key),
+					"called with correct arguments").to.be.true
+				done()
+
+		it "should dispatch the error from s3.listObjects", (done) ->
+			@s3.listObjects.callsArgWith 1, @error
+			@AWSSDKPersistorManager.directorySize @bucketName, @key, (err) =>
+				expect(err).to.equal @error
+				done()
+
+		it "should sum directory files sizes", (done) ->
+			@s3.listObjects.callsArgWith 1, null, Contents: [ { Size: 1024 }, { Size: 2048 }]
+			@AWSSDKPersistorManager.directorySize @bucketName, @key, (err, size) =>
+				expect(size).to.equal 3072
+				done()

services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee

@@ -18,6 +18,10 @@ describe "FSPersistorManagerTests", ->
       unlink:sinon.stub()
       rmdir:sinon.stub()
       exists:sinon.stub()
+      readdir:sinon.stub()
+      openSync:sinon.stub()
+      fstatSync:sinon.stub()
+      closeSync:sinon.stub()
     @Rimraf = sinon.stub()
     @LocalFileWriter =
       writeStream: sinon.stub()
@@ -208,3 +212,18 @@ describe "FSPersistorManagerTests", ->
       @FSPersistorManager.checkIfFileExists @location, @name1, (err,exists) =>
         exists.should.be.false
         done()
+
+  describe "directorySize", ->
+
+    it "should propogate the error", (done) ->
+      @Fs.readdir.callsArgWith(1, @error)
+      @FSPersistorManager.directorySize @location, @name1, (err, totalsize) =>
+        err.should.equal @error
+        done()
+
+    it "should sum directory files size", (done) ->
+      @Fs.readdir.callsArgWith(1, null, [ {'file1'}, {'file2'} ])
+      @Fs.fstatSync.returns({size : 1024})
+      @FSPersistorManager.directorySize @location, @name1, (err, totalsize) =>
+        expect(totalsize).to.equal 2048
+        done()

services/filestore/test/unit/coffee/FileControllerTests.coffee

@@ -22,6 +22,7 @@ describe "FileController", ->
 			getFile: sinon.stub()
 			deleteFile: sinon.stub()
 			insertFile: sinon.stub()
+			getDirectorySize: sinon.stub()
 		@LocalFileWriter = {}
 		@controller = SandboxedModule.require modulePath, requires:
 			"./LocalFileWriter":@LocalFileWriter
@@ -152,3 +153,18 @@ describe "FileController", ->
 			result = @controller._get_range('carrots=0-200')
 			expect(result).to.equal null
 			done()
+
+	describe "directorySize", ->
+
+		it "should return total directory size bytes", (done) ->
+			@FileHandler.getDirectorySize.callsArgWith(2, null, 1024)
+			@controller.directorySize @req, json:(result)=>
+				expect(result['total bytes']).to.equal 1024
+				done()
+
+		it "should send a 500 if there was an error", (done)->
+			@FileHandler.getDirectorySize.callsArgWith(2, "error")
+			@res.send = (code)->
+				code.should.equal 500
+				done()
+			@controller.directorySize @req, @res

services/filestore/test/unit/coffee/FileHandlerTests.coffee

@@ -20,6 +20,7 @@ describe "FileHandler", ->
 			deleteDirectory: sinon.stub()
 			sendStream: sinon.stub()
 			insertFile: sinon.stub()
+			directorySize: sinon.stub()
 		@LocalFileWriter =
 			writeStream: sinon.stub()
 			deleteFile: sinon.stub()
@@ -192,3 +193,13 @@ describe "FileHandler", ->
 				@FileConverter.convert.calledWith(@stubbedPath, @format).should.equal true
 				@LocalFileWriter.deleteFile.calledWith(@stubbedPath).should.equal true
 				done()
+
+	describe "getDirectorySize", ->
+
+		beforeEach ->
+			@PersistorManager.directorySize.callsArgWith(2)
+
+		it "should call the filestore manager to get directory size", (done)->
+			@handler.getDirectorySize @bucket, @key, =>
+				@PersistorManager.directorySize.calledWith(@bucket, @key).should.equal true
+				done()

services/filestore/test/unit/coffee/S3PersistorManagerTests.coffee

@@ -256,3 +256,16 @@ describe "S3PersistorManagerTests", ->
 			@S3PersistorManager.checkIfFileExists @bucketName, @key, (err)=>
 				err.should.equal @error
 				done()
+
+	describe "directorySize", ->
+
+		beforeEach ->
+			@S3PersistorManager = SandboxedModule.require modulePath, requires: @requires
+
+		it "should sum directory files size", (done) ->
+			data =
+				Contents: [ {Size: 1024}, {Size: 2048} ]
+			@stubbedKnoxClient.list.callsArgWith(1, null, data)
+			@S3PersistorManager.directorySize @bucketName, @key, (err, totalSize)=>
+				totalSize.should.equal 3072
+				done()