Merge branch 'master' of https://github.com/sharelatex/filestore-sharelatex

services/filestore/Gruntfile.coffee

@@ -52,7 +52,10 @@ module.exports = (grunt) ->
 				options:
 					ext:"*.coffee"
 
-
+		execute:
+			app:
+				src: "app.js"
+				
 		concurrent:
 			dev:
 				tasks: ['nodemon', 'watch']
@@ -79,13 +82,15 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-concurrent'
 	grunt.loadNpmTasks 'grunt-mocha-test'
 	grunt.loadNpmTasks 'grunt-forever'
+	grunt.loadNpmTasks 'grunt-bunyan'
+	grunt.loadNpmTasks 'grunt-execute'
 
 	grunt.registerTask "test:unit", ["coffee", "mochaTest:unit"]
 	grunt.registerTask "test:acceptence", ["coffee", "mochaTest:acceptence"]
 	grunt.registerTask "test:acceptance", ["test:acceptence"]
 
 	grunt.registerTask "ci", "test:unit"
-	grunt.registerTask 'default', ['coffee', 'concurrent']
+	grunt.registerTask 'default', ['coffee', 'bunyan','execute']
 
 	grunt.registerTask "compile", "coffee"
 	grunt.registerTask "install", "compile"

services/filestore/app.coffee

@@ -13,6 +13,7 @@ streamBuffers = require("stream-buffers")
 Metrics = require "metrics-sharelatex"
 Metrics.initialize("filestore")
 Metrics.open_sockets.monitor(logger)
+Metrics.event_loop?.monitor(logger)
 
 app.configure ->
 	app.use express.bodyParser()
@@ -74,8 +75,17 @@ app.put "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileContro
 app.del "/project/:project_id/file/:file_id", keyBuilder.userFileKey, fileController.deleteFile
 
 app.get  "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.getFile
+app.get  "/template/:template_id/v/:version/:format/:sub_type", keyBuilder.templateFileKey, fileController.getFile
 app.post "/template/:template_id/v/:version/:format", keyBuilder.templateFileKey, fileController.insertFile
 
+
+app.get  "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.getFile
+app.post "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.insertFile
+
+app.put "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.copyFile
+app.del "/project/:project_id/public/:public_file_id", keyBuilder.publicFileKey, fileController.deleteFile
+
+
 app.get "/heapdump", (req, res)->
 	require('heapdump').writeSnapshot '/tmp/' + Date.now() + '.filestore.heapsnapshot', (err, filename)->
 		res.send filename
@@ -125,7 +135,7 @@ beginShutdown = () ->
 		logger.log "server will stop accepting connections"
 
 server.listen port, host, ->
-	logger.log("filestore listening on #{host}:#{port}")
+	logger.info "Filestore starting up, listening on #{host}:#{port}"
 
 process.on 'SIGTERM', () ->
 	logger.log("filestore got SIGTERM, shutting down gracefully")

services/filestore/app/coffee/FileConverter.coffee

@@ -39,7 +39,7 @@ module.exports =
 		args = "nice convert -flatten -background white -density 300 -define pdf:fit-page=#{width} #{sourcePath} -resize #{width} #{destPath}"
 		safe_exec args, childProcessOpts, (err, stdout, stderr)->
 			if err?
-				logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to preview"
+				logger.err err:err, stderr:stderr, sourcePath:sourcePath, "something went wrong converting file to thumbnail"
 			else
 				logger.log  sourcePath:sourcePath, destPath:destPath, "finished thumbnailing file"
 			callback(err, destPath)	

services/filestore/app/coffee/KeyBuilder.coffee

@@ -23,9 +23,20 @@ module.exports =
 		req.bucket = settings.filestore.stores.user_files
 		next()
 		
+	publicFileKey: (req, res, next)->
+		{project_id, public_file_id} = req.params
+		if not settings.filestore.stores.public_files?
+			res.status(501).send("public files not available")
+		else
+			req.key = "#{project_id}/#{public_file_id}"
+			req.bucket = settings.filestore.stores.public_files
+			next()
+
 	templateFileKey: (req, res, next)->
-		{template_id, format, version} = req.params
+		{template_id, format, version, sub_type} = req.params
 		req.key = "#{template_id}/v/#{version}/#{format}"
+		if sub_type?
+			req.key = "#{req.key}/#{sub_type}"
 		req.bucket = settings.filestore.stores.template_files
 		req.version = version
 		opts = req.query

services/filestore/app/coffee/LocalFileWriter.coffee

@@ -14,7 +14,6 @@ module.exports =
 		fsPath = @_getPath(key)
 		logger.log fsPath:fsPath, "writing file locally"
 		writeStream = fs.createWriteStream(fsPath)
-		stream.pipe writeStream
 		writeStream.on "finish", ->
 			timer.done()
 			logger.log fsPath:fsPath, "finished writing file locally"
@@ -25,6 +24,7 @@ module.exports =
 		stream.on "error", (err)->
 			logger.log err:err, fsPath:fsPath, "problem writing file locally, with read stream"
 			callback err
+		stream.pipe writeStream
 
 	deleteFile: (fsPath, callback)->
 		logger.log fsPath:fsPath, "removing local temp file"

services/filestore/app/coffee/S3PersistorManager.coffee

@@ -104,6 +104,9 @@ module.exports =
 			secret: settings.filestore.s3.secret
 			bucket: bucketName
 		s3Client.list prefix:key, (err, data)->
+			if err?
+				logger.err err:err, bucketName:bucketName, key:key, "something went wrong listing prefix in aws"
+				return callback(err)
 			keys = _.map data.Contents, (entry)->
 				return entry.Key
 			s3Client.deleteMultiple keys, callback

services/filestore/config/settings.defaults.coffee

@@ -21,6 +21,7 @@ module.exports =
 			# Must contain full path, e.g. "/var/lib/sharelatex/data".
 			# This path must exist, not be tmpfs and be writable to by the user sharelatex is run as.
 			user_files: Path.resolve(__dirname + "/../user_files")
+			public_files: Path.resolve(__dirname + "/../public_files")
 			template_files: Path.resolve(__dirname + "/../template_files")
 		# if you are using S3, then fill in your S3 details below
 		# s3:

services/filestore/package.json

@@ -8,8 +8,11 @@
   },
   "dependencies": {
     "async": "~0.2.10",
+    "bunyan": "^1.3.5",
     "coffee-script": "~1.7.1",
     "express": "~3.4.8",
+    "grunt-bunyan": "^0.5.0",
+    "grunt-execute": "^0.2.2",
     "grunt-mocha-test": "~0.8.2",
     "heapdump": "^0.3.2",
     "knox": "~0.9.1",