Merge pull request #41 from sharelatex/csh-ho-docker-issue-1338-bulk-upgrade

Services bulk upgrade - filestore

services/filestore/README.md

@@ -5,6 +5,18 @@ An API for CRUD operations on binary files stored in S3
 
 [![Build Status](https://travis-ci.org/sharelatex/filestore-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/filestore-sharelatex)
 
+filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, presenting a RESTful HTTP interface to the CLSIs on port 3009 by default. Urls are mapped to node functions in https://github.com/sharelatex/filestore-sharelatex/blob/master/app.coffee . URLs are of the form:
+
+* `/project/:project_id/file/:file_id`
+* `/template/:template_id/v/:version/:format`
+* `/project/:project_id/public/:public_file_id`
+* `/project/:project_id/size`
+* `/bucket/:bucket/key/*`
+* `/heapdump`
+* `/shutdown`
+* `/status` - returns `filestore sharelatex up` or `server is being shut down` (HTTP 500)
+* `/health_check` 
+
 License
 -------
 

services/filestore/app.coffee

@@ -1,3 +1,5 @@
+Metrics = require "metrics-sharelatex"
+Metrics.initialize("filestore")
 express = require('express')
 bodyParser = require "body-parser"
 logger = require('logger-sharelatex')
@@ -15,16 +17,12 @@ app = express()
 if settings.sentry?.dsn?
 	logger.initializeErrorReporting(settings.sentry.dsn)
 
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("filestore")
 Metrics.open_sockets.monitor(logger)
 Metrics.event_loop?.monitor(logger)
 Metrics.memory.monitor(logger)
 
 app.use Metrics.http.monitor(logger)
 
-Metrics.inc "startup"
-
 app.use (req, res, next)->
 	Metrics.inc "http-request"
 	next()

services/filestore/app/coffee/FSPersistorManager.coffee

@@ -13,10 +13,19 @@ module.exports =
   sendFile: ( location, target, source, callback = (err)->) ->
     filteredTarget = filterName target
     logger.log location:location, target:filteredTarget, source:source, "sending file"
-    fs.rename source, "#{location}/#{filteredTarget}", (err) ->
-      if err!=null
+    done = _.once (err) ->
+      if err?
         logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file"
-      callback err
+      callback(err)
+    # actually copy the file (instead of moving it) to maintain consistent behaviour
+    # between the different implementations
+    sourceStream = fs.createReadStream source
+    sourceStream.on 'error', done
+    targetStream = fs.createWriteStream "#{location}/#{filteredTarget}"
+    targetStream.on 'error', done
+    targetStream.on 'finish', () ->
+      done()
+    sourceStream.pipe targetStream
 
   sendStream: ( location, target, sourceStream, callback = (err)->) ->
     logger.log location:location, target:target, "sending file stream"
@@ -26,7 +35,10 @@ module.exports =
       if err?
         logger.err  location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
         return callback err
-      @sendFile location, target, fsPath, callback
+      @sendFile location, target, fsPath, (err) -> 
+        # delete the temporary file created above and return the original error
+        LocalFileWriter.deleteFile fsPath, () ->
+          callback(err)
 
   # opts may be {start: Number, end: Number}
   getFileStream: (location, name, opts, _callback = (err, res)->) ->

services/filestore/app/coffee/FileController.coffee

@@ -29,10 +29,10 @@ module.exports = FileController =
 			logger.log start: range.start, end: range.end, "getting range of bytes from file"
 		FileHandler.getFile bucket, key, options, (err, fileStream)->
 			if err?
-				logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
 				if err instanceof Errors.NotFoundError
 					return res.send 404
 				else
+					logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
 					return res.send 500
 			else if req.query.cacheWarm
 				logger.log key:key, bucket:bucket, format:format, style:style, "request is only for cache warm so not sending stream"

services/filestore/app/coffee/FileHandler.coffee

@@ -6,6 +6,7 @@ FileConverter = require("./FileConverter")
 KeyBuilder = require("./KeyBuilder")
 async = require("async")
 ImageOptimiser = require("./ImageOptimiser")
+Errors = require('./Errors')
 
 module.exports = FileHandler =
 
@@ -32,7 +33,7 @@ module.exports = FileHandler =
 
 	_getStandardFile: (bucket, key, opts, callback)->
 		PersistorManager.getFileStream bucket, key, opts, (err, fileStream)->
-			if err?
+			if err? and !(err instanceof Errors.NotFoundError)
 				logger.err  bucket:bucket, key:key, opts:FileHandler._scrubSecrets(opts), "error getting fileStream"
 			callback err, fileStream
 
@@ -64,7 +65,24 @@ module.exports = FileHandler =
 				LocalFileWriter.deleteFile convertedFsPath, ->
 				LocalFileWriter.deleteFile originalFsPath, ->
 				return callback(err)
-			PersistorManager.getFileStream bucket, convertedKey, opts, callback
+			# Send back the converted file from the local copy to avoid problems
+			# with the file not being present in S3 yet.  As described in the
+			# documentation below, we have already made a 'HEAD' request in
+			# checkIfFileExists so we only have "eventual consistency" if we try
+			# to stream it from S3 here.  This was a cause of many 403 errors.
+			#
+			# "Amazon S3 provides read-after-write consistency for PUTS of new
+			# objects in your S3 bucket in all regions with one caveat. The
+			# caveat is that if you make a HEAD or GET request to the key name
+			# (to find if the object exists) before creating the object, Amazon
+			# S3 provides eventual consistency for read-after-write.""
+			# https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
+			LocalFileWriter.getStream convertedFsPath, (err, readStream) ->
+				return callback(err) if err?
+				readStream.on 'end', () ->
+					logger.log {convertedFsPath: convertedFsPath}, "deleting temporary file"
+					LocalFileWriter.deleteFile convertedFsPath, ->
+				callback(null, readStream)
 
 	_convertFile: (bucket, originalKey, opts, callback)->
 		@_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)->

services/filestore/app/coffee/LocalFileWriter.coffee

@@ -5,6 +5,7 @@ _ = require("underscore")
 logger = require("logger-sharelatex")
 metrics = require("metrics-sharelatex")
 Settings = require("settings-sharelatex")
+Errors = require "./Errors"
 
 module.exports = 
 
@@ -26,6 +27,22 @@ module.exports =
 			callback err
 		stream.pipe writeStream
 
+	getStream: (fsPath, _callback = (err, res)->) ->
+		callback = _.once _callback
+		timer = new metrics.Timer("readingFile")
+		logger.log fsPath:fsPath, "reading file locally"
+		readStream = fs.createReadStream(fsPath)
+		readStream.on "end", ->
+			timer.done()
+			logger.log fsPath:fsPath, "finished reading file locally"
+		readStream.on "error", (err)->
+			logger.err err:err, fsPath:fsPath, "problem reading file locally, with read stream"
+			if err.code == 'ENOENT'
+				callback new Errors.NotFoundError(err.message), null
+			else
+				callback err
+		callback null, readStream
+
 	deleteFile: (fsPath, callback)->
 		if !fsPath? or fsPath == ""
 			return callback()

services/filestore/app/coffee/S3PersistorManager.coffee

@@ -42,9 +42,8 @@ module.exports =
 			if res.statusCode != 200
 				logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file"
 				return callback("non 200 response from s3 on put file")
-			LocalFileWriter.deleteFile fsPath, (err)->
-				logger.log res:res,  bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
-				callback(err)
+			logger.log res:res,  bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
+			callback(err)
 		putEventEmiter.on "error", (err)->
 			logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file"
 			callback err
@@ -57,7 +56,10 @@ module.exports =
 			if err?
 				logger.err  bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
 				return callback(err)
-			@sendFile bucketName, key, fsPath, callback
+			@sendFile bucketName, key, fsPath, (err) ->
+				# delete the temporary file created above and return the original error
+				LocalFileWriter.deleteFile fsPath, () ->
+					callback(err)
 
 	# opts may be {start: Number, end: Number}
 	getFileStream: (bucketName, key, opts, callback = (err, res)->)->
@@ -74,7 +76,9 @@ module.exports =
 		s3Stream = s3Client.get(key, headers)
 		s3Stream.end()
 		s3Stream.on 'response', (res) ->
-			if res.statusCode == 404
+			if res.statusCode in [403, 404]
+				# S3 returns a 403 instead of a 404 when the user doesn't have
+				# permission to list the bucket contents.
 				logger.log bucketName:bucketName, key:key, "file not found in s3"
 				return callback new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null
 			else if res.statusCode not in [200, 206]

services/filestore/npm-shrinkwrap.json

@@ -3,24 +3,36 @@
   "version": "0.1.4",
   "dependencies": {
     "@google-cloud/common": {
-      "version": "0.23.0",
-      "from": "@google-cloud/common@>=0.23.0 <0.24.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.23.0.tgz"
+      "version": "0.27.0",
+      "from": "@google-cloud/common@>=0.27.0 <0.28.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz"
     },
     "@google-cloud/debug-agent": {
-      "version": "3.0.0",
+      "version": "3.0.1",
       "from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.1.tgz",
       "dependencies": {
         "coffeescript": {
           "version": "2.3.2",
           "from": "coffeescript@>=2.0.0 <3.0.0",
           "resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz"
+        }
+      }
+    },
+    "@google-cloud/profiler": {
+      "version": "0.2.3",
+      "from": "@google-cloud/profiler@>=0.2.3 <0.3.0",
+      "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz",
+      "dependencies": {
+        "@google-cloud/common": {
+          "version": "0.26.2",
+          "from": "@google-cloud/common@>=0.26.0 <0.27.0",
+          "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz"
         },
-        "lodash": {
-          "version": "4.17.11",
-          "from": "lodash@>=4.12.0 <5.0.0",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz"
+        "through2": {
+          "version": "3.0.0",
+          "from": "through2@>=3.0.0 <4.0.0",
+          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz"
         }
       }
     },
@@ -35,41 +47,14 @@
       "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz"
     },
     "@google-cloud/trace-agent": {
-      "version": "3.3.1",
+      "version": "3.5.0",
       "from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.3.1.tgz",
+      "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.0.tgz",
       "dependencies": {
         "@google-cloud/common": {
-          "version": "0.26.2",
-          "from": "@google-cloud/common@>=0.26.0 <0.27.0",
-          "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz"
-        },
-        "gcp-metadata": {
-          "version": "0.9.0",
-          "from": "gcp-metadata@>=0.9.0 <0.10.0",
-          "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.0.tgz"
-        },
-        "google-auth-library": {
-          "version": "2.0.1",
-          "from": "google-auth-library@>=2.0.0 <3.0.0",
-          "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.1.tgz",
-          "dependencies": {
-            "gcp-metadata": {
-              "version": "0.7.0",
-              "from": "gcp-metadata@^0.7.0",
-              "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
-            }
-          }
-        },
-        "lru-cache": {
-          "version": "4.1.4",
-          "from": "lru-cache@>=4.1.3 <5.0.0",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.4.tgz"
-        },
-        "through2": {
-          "version": "3.0.0",
-          "from": "through2@>=3.0.0 <4.0.0",
-          "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz"
+          "version": "0.28.0",
+          "from": "@google-cloud/common@>=0.28.0 <0.29.0",
+          "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.28.0.tgz"
         },
         "uuid": {
           "version": "3.3.2",
@@ -78,6 +63,61 @@
         }
       }
     },
+    "@protobufjs/aspromise": {
+      "version": "1.1.2",
+      "from": "@protobufjs/aspromise@>=1.1.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz"
+    },
+    "@protobufjs/base64": {
+      "version": "1.1.2",
+      "from": "@protobufjs/base64@>=1.1.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz"
+    },
+    "@protobufjs/codegen": {
+      "version": "2.0.4",
+      "from": "@protobufjs/codegen@>=2.0.4 <3.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz"
+    },
+    "@protobufjs/eventemitter": {
+      "version": "1.1.0",
+      "from": "@protobufjs/eventemitter@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz"
+    },
+    "@protobufjs/fetch": {
+      "version": "1.1.0",
+      "from": "@protobufjs/fetch@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz"
+    },
+    "@protobufjs/float": {
+      "version": "1.0.2",
+      "from": "@protobufjs/float@>=1.0.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz"
+    },
+    "@protobufjs/inquire": {
+      "version": "1.1.0",
+      "from": "@protobufjs/inquire@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz"
+    },
+    "@protobufjs/path": {
+      "version": "1.1.2",
+      "from": "@protobufjs/path@>=1.1.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz"
+    },
+    "@protobufjs/pool": {
+      "version": "1.1.0",
+      "from": "@protobufjs/pool@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz"
+    },
+    "@protobufjs/utf8": {
+      "version": "1.1.0",
+      "from": "@protobufjs/utf8@>=1.1.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz"
+    },
+    "@sindresorhus/is": {
+      "version": "0.13.0",
+      "from": "@sindresorhus/is@>=0.13.0 <0.14.0",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz"
+    },
     "@sinonjs/commons": {
       "version": "1.3.0",
       "from": "@sinonjs/commons@>=1.2.0 <2.0.0",
@@ -105,6 +145,11 @@
       "from": "@types/caseless@*",
       "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz"
     },
+    "@types/console-log-level": {
+      "version": "1.4.0",
+      "from": "@types/console-log-level@>=1.4.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz"
+    },
     "@types/duplexify": {
       "version": "3.6.0",
       "from": "@types/duplexify@>=3.5.0 <4.0.0",
@@ -115,16 +160,26 @@
       "from": "@types/form-data@*",
       "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz"
     },
+    "@types/long": {
+      "version": "4.0.0",
+      "from": "@types/long@>=4.0.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz"
+    },
     "@types/node": {
-      "version": "10.12.10",
+      "version": "10.12.18",
       "from": "@types/node@*",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz"
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz"
     },
     "@types/request": {
       "version": "2.48.1",
       "from": "@types/request@>=2.47.0 <3.0.0",
       "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz"
     },
+    "@types/semver": {
+      "version": "5.5.0",
+      "from": "@types/semver@>=5.5.0 <6.0.0",
+      "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz"
+    },
     "@types/tough-cookie": {
       "version": "2.3.4",
       "from": "@types/tough-cookie@*",
@@ -254,6 +309,11 @@
       "from": "bignumber.js@>=7.0.0 <8.0.0",
       "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz"
     },
+    "bindings": {
+      "version": "1.3.1",
+      "from": "bindings@>=1.2.1 <2.0.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz"
+    },
     "bintrees": {
       "version": "1.0.1",
       "from": "bintrees@1.0.1",
@@ -431,10 +491,10 @@
       "from": "deep-eql@>=3.0.1 <4.0.0",
       "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz"
     },
-    "define-properties": {
-      "version": "1.1.3",
-      "from": "define-properties@>=1.1.2 <2.0.0",
-      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz"
+    "delay": {
+      "version": "4.1.0",
+      "from": "delay@>=4.0.1 <5.0.0",
+      "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz"
     },
     "delayed-stream": {
       "version": "0.0.5",
@@ -497,16 +557,6 @@
       "from": "ent@>=2.2.0 <3.0.0",
       "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz"
     },
-    "es-abstract": {
-      "version": "1.12.0",
-      "from": "es-abstract@>=1.5.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz"
-    },
-    "es-to-primitive": {
-      "version": "1.2.0",
-      "from": "es-to-primitive@>=1.1.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz"
-    },
     "es6-promise": {
       "version": "4.2.5",
       "from": "es6-promise@>=4.0.3 <5.0.0",
@@ -609,9 +659,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.5.10",
+      "version": "1.6.0",
       "from": "follow-redirects@>=1.3.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.0.tgz",
       "dependencies": {
         "debug": {
           "version": "3.1.0",
@@ -657,15 +707,15 @@
       "from": "fs.realpath@>=1.0.0 <2.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
     },
-    "function-bind": {
-      "version": "1.1.1",
-      "from": "function-bind@>=1.1.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz"
+    "gaxios": {
+      "version": "1.0.4",
+      "from": "gaxios@>=1.0.2 <2.0.0",
+      "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.0.4.tgz"
     },
     "gcp-metadata": {
-      "version": "0.7.0",
-      "from": "gcp-metadata@>=0.7.0 <0.8.0",
-      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
+      "version": "0.9.3",
+      "from": "gcp-metadata@>=0.9.0 <0.10.0",
+      "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz"
     },
     "get-func-name": {
       "version": "2.0.0",
@@ -689,33 +739,26 @@
       "optional": true
     },
     "google-auth-library": {
-      "version": "1.6.1",
-      "from": "google-auth-library@>=1.6.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.6.1.tgz",
+      "version": "2.0.2",
+      "from": "google-auth-library@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz",
       "dependencies": {
         "gcp-metadata": {
-          "version": "0.6.3",
-          "from": "gcp-metadata@>=0.6.3 <0.7.0",
-          "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz"
+          "version": "0.7.0",
+          "from": "gcp-metadata@>=0.7.0 <0.8.0",
+          "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
         },
         "lru-cache": {
-          "version": "4.1.4",
-          "from": "lru-cache@>=4.1.3 <5.0.0",
-          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.4.tgz"
+          "version": "5.1.1",
+          "from": "lru-cache@>=5.0.0 <6.0.0",
+          "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz"
         }
       }
     },
     "google-p12-pem": {
-      "version": "1.0.2",
+      "version": "1.0.3",
       "from": "google-p12-pem@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.2.tgz",
-      "dependencies": {
-        "pify": {
-          "version": "3.0.0",
-          "from": "pify@>=3.0.0 <4.0.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz"
-        }
-      }
+      "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz"
     },
     "graceful-fs": {
       "version": "4.1.15",
@@ -920,11 +963,6 @@
         }
       }
     },
-    "has": {
-      "version": "1.0.3",
-      "from": "has@>=1.0.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz"
-    },
     "has-ansi": {
       "version": "0.1.0",
       "from": "has-ansi@>=0.1.0 <0.2.0",
@@ -935,11 +973,6 @@
       "from": "has-flag@>=3.0.0 <4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz"
     },
-    "has-symbols": {
-      "version": "1.0.0",
-      "from": "has-symbols@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz"
-    },
     "hawk": {
       "version": "0.10.2",
       "from": "hawk@>=0.10.2 <0.11.0",
@@ -956,9 +989,9 @@
       "resolved": "https://registry.npmjs.org/heapdump/-/heapdump-0.3.12.tgz"
     },
     "hex2dec": {
-      "version": "1.1.0",
+      "version": "1.1.1",
       "from": "hex2dec@>=1.0.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.0.tgz"
+      "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.1.tgz"
     },
     "hoek": {
       "version": "0.7.6",
@@ -1018,35 +1051,15 @@
       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz"
     },
     "is": {
-      "version": "3.2.1",
-      "from": "is@>=3.2.1 <4.0.0",
-      "resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz"
+      "version": "3.3.0",
+      "from": "is@>=3.2.0 <4.0.0",
+      "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz"
     },
     "is-buffer": {
       "version": "1.1.6",
       "from": "is-buffer@>=1.1.5 <2.0.0",
       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz"
     },
-    "is-callable": {
-      "version": "1.1.4",
-      "from": "is-callable@>=1.1.3 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz"
-    },
-    "is-date-object": {
-      "version": "1.0.1",
-      "from": "is-date-object@>=1.0.1 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz"
-    },
-    "is-regex": {
-      "version": "1.0.4",
-      "from": "is-regex@>=1.0.4 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz"
-    },
-    "is-symbol": {
-      "version": "1.0.2",
-      "from": "is-symbol@>=1.0.2 <2.0.0",
-      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz"
-    },
     "isarray": {
       "version": "1.0.0",
       "from": "isarray@>=1.0.0 <2.0.0",
@@ -1136,15 +1149,15 @@
       "from": "lodash.get@>=4.4.2 <5.0.0",
       "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz"
     },
-    "lodash.isstring": {
-      "version": "4.0.1",
-      "from": "lodash.isstring@>=4.0.1 <5.0.0",
-      "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz"
+    "lodash.pickby": {
+      "version": "4.6.0",
+      "from": "lodash.pickby@>=4.6.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz"
     },
     "logger-sharelatex": {
-      "version": "1.5.7",
-      "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7",
-      "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#13562f8866708fc86aef8202bf5a2ce4d1c6eed7",
+      "version": "1.5.9",
+      "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9",
+      "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#e8e1b95052f62e107336053e4a983f81cdbdf589",
       "dependencies": {
         "coffee-script": {
           "version": "1.12.4",
@@ -1158,6 +1171,11 @@
       "from": "lolex@>=3.0.0 <4.0.0",
       "resolved": "https://registry.npmjs.org/lolex/-/lolex-3.0.0.tgz"
     },
+    "long": {
+      "version": "4.0.0",
+      "from": "long@>=4.0.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz"
+    },
     "lru-cache": {
       "version": "2.7.3",
       "from": "lru-cache@>=2.0.0 <3.0.0",
@@ -1194,14 +1212,19 @@
       "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
     },
     "metrics-sharelatex": {
-      "version": "2.0.7",
-      "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7",
-      "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3c7dd668d1153c13acee9cceb3a8ce24495b7c86",
+      "version": "2.0.12",
+      "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
+      "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3ac1621ef049e2f2d88a83b3a41011333d609662",
       "dependencies": {
         "coffee-script": {
           "version": "1.6.0",
           "from": "coffee-script@1.6.0",
           "resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz"
+        },
+        "underscore": {
+          "version": "1.6.0",
+          "from": "underscore@>=1.6.0 <1.7.0",
+          "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
         }
       }
     },
@@ -1402,16 +1425,6 @@
       "from": "oauth-sign@>=0.2.0 <0.3.0",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.2.0.tgz"
     },
-    "object-keys": {
-      "version": "1.0.12",
-      "from": "object-keys@>=1.0.12 <2.0.0",
-      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz"
-    },
-    "object.getownpropertydescriptors": {
-      "version": "2.0.3",
-      "from": "object.getownpropertydescriptors@>=2.0.3 <3.0.0",
-      "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz"
-    },
     "on-finished": {
       "version": "2.3.0",
       "from": "on-finished@>=2.3.0 <2.4.0",
@@ -1423,15 +1436,25 @@
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz"
     },
     "p-limit": {
-      "version": "2.0.0",
+      "version": "2.1.0",
       "from": "p-limit@>=2.0.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz"
+      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz"
     },
     "p-try": {
       "version": "2.0.0",
       "from": "p-try@>=2.0.0 <3.0.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz"
     },
+    "parse-duration": {
+      "version": "0.1.1",
+      "from": "parse-duration@>=0.1.1 <0.2.0",
+      "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz"
+    },
+    "parse-ms": {
+      "version": "2.0.0",
+      "from": "parse-ms@>=2.0.0 <3.0.0",
+      "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.0.0.tgz"
+    },
     "parseurl": {
       "version": "1.3.2",
       "from": "parseurl@>=1.3.2 <1.4.0",
@@ -1467,26 +1490,31 @@
       "from": "pngcrush@0.0.3",
       "resolved": "https://registry.npmjs.org/pngcrush/-/pngcrush-0.0.3.tgz"
     },
+    "pretty-ms": {
+      "version": "4.0.0",
+      "from": "pretty-ms@>=4.0.0 <5.0.0",
+      "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz"
+    },
     "process-nextick-args": {
       "version": "2.0.0",
       "from": "process-nextick-args@>=2.0.0 <2.1.0",
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz"
     },
     "prom-client": {
-      "version": "11.2.0",
+      "version": "11.2.1",
       "from": "prom-client@>=11.1.3 <12.0.0",
-      "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.0.tgz"
+      "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.1.tgz"
+    },
+    "protobufjs": {
+      "version": "6.8.8",
+      "from": "protobufjs@>=6.8.6 <6.9.0",
+      "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz"
     },
     "proxy-addr": {
       "version": "2.0.4",
       "from": "proxy-addr@>=2.0.4 <2.1.0",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz"
     },
-    "pseudomap": {
-      "version": "1.0.2",
-      "from": "pseudomap@>=1.0.2 <2.0.0",
-      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz"
-    },
     "punycode": {
       "version": "1.3.2",
       "from": "punycode@1.3.2",
@@ -1576,9 +1604,9 @@
       "resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz"
     },
     "resolve": {
-      "version": "1.8.1",
+      "version": "1.9.0",
       "from": "resolve@>=1.5.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz"
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz"
     },
     "response": {
       "version": "0.14.0",
@@ -1764,15 +1792,20 @@
       "from": "supports-color@>=0.2.0 <0.3.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz"
     },
+    "symbol-observable": {
+      "version": "1.2.0",
+      "from": "symbol-observable@>=1.2.0 <2.0.0",
+      "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz"
+    },
     "tdigest": {
       "version": "0.1.1",
       "from": "tdigest@>=0.1.1 <0.2.0",
       "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz"
     },
     "teeny-request": {
-      "version": "3.9.1",
-      "from": "teeny-request@>=3.6.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.9.1.tgz",
+      "version": "3.11.3",
+      "from": "teeny-request@>=3.11.1 <4.0.0",
+      "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz",
       "dependencies": {
         "uuid": {
           "version": "3.3.2",
@@ -1841,11 +1874,6 @@
       "from": "util-deprecate@>=1.0.1 <1.1.0",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
     },
-    "util.promisify": {
-      "version": "1.0.0",
-      "from": "util.promisify@>=1.0.0 <2.0.0",
-      "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz"
-    },
     "utils-merge": {
       "version": "1.0.1",
       "from": "utils-merge@1.0.1",

services/filestore/package.json

@@ -30,8 +30,8 @@
     "fs-extra": "^1.0.0",
     "heapdump": "^0.3.2",
     "knox": "~0.9.1",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7",
+    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9",
+    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
     "mocha": "5.2.0",
     "node-transloadit": "0.0.4",
     "node-uuid": "~1.4.1",

services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee

@@ -25,6 +25,7 @@ describe "FSPersistorManagerTests", ->
     @Rimraf = sinon.stub()
     @LocalFileWriter =
       writeStream: sinon.stub()
+      deleteFile: sinon.stub()
     @requires =
       "./LocalFileWriter":@LocalFileWriter
       "fs":@Fs
@@ -41,10 +42,32 @@ describe "FSPersistorManagerTests", ->
     @FSPersistorManager = SandboxedModule.require modulePath, requires: @requires
 
   describe "sendFile", ->
-    it "should put the file", (done) ->
-      @Fs.rename.callsArgWith(2,@error)
+    beforeEach ->
+      @Fs.createReadStream = sinon.stub().returns({
+        on: ->
+        pipe: ->
+      })
+
+    it "should copy the file", (done) ->
+      @Fs.createWriteStream =sinon.stub().returns({
+        on: (event, handler) ->
+          process.nextTick(handler) if event is 'finish'
+      })
+      @FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
+        @Fs.createReadStream.calledWith(@name2).should.equal true
+        @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
+        done()
+
+    it "should return an error if the file cannot be stored", (done) ->
+      @Fs.createWriteStream =sinon.stub().returns({
+        on: (event, handler) =>
+         if event is 'error'
+          process.nextTick () =>
+            handler(@error)
+      })
       @FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
-        @Fs.rename.calledWith( @name2, "#{@location}/#{@name1Filtered}" ).should.equal true
+        @Fs.createReadStream.calledWith(@name2).should.equal true
+        @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
         err.should.equal @error
         done()
 
@@ -52,6 +75,7 @@ describe "FSPersistorManagerTests", ->
     beforeEach ->
       @FSPersistorManager.sendFile = sinon.stub().callsArgWith(3)
       @LocalFileWriter.writeStream.callsArgWith(2, null, @name1)
+      @LocalFileWriter.deleteFile.callsArg(1)
       @SourceStream =
         on:->
 

services/filestore/test/unit/coffee/FileHandlerTests.coffee

@@ -23,6 +23,7 @@ describe "FileHandler", ->
 			directorySize: sinon.stub()
 		@LocalFileWriter =
 			writeStream: sinon.stub()
+			getStream: sinon.stub()
 			deleteFile: sinon.stub()
 		@FileConverter =
 			convert: sinon.stub()
@@ -152,17 +153,20 @@ describe "FileHandler", ->
 
 		it "should _convertFile ", (done)->
 			@stubbedStream = {"something":"here"}
+			@localStream = {
+				on: ->
+			}
 			@PersistorManager.sendFile = sinon.stub().callsArgWith(3)
-			@PersistorManager.getFileStream = sinon.stub().callsArgWith(3, null, @stubbedStream)
+			@LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, @localStream)
 			@convetedKey = @key+"converted"
 			@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
 			@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
 			@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, (err, fsStream)=>
 				@handler._convertFile.called.should.equal true
 				@PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
-				@PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
 				@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
-				fsStream.should.equal @stubbedStream
+				@LocalFileWriter.getStream.calledWith(@stubbedPath).should.equal true
+				fsStream.should.equal @localStream
 				done()
 
 	describe "_convertFile", ->

services/filestore/test/unit/coffee/LocalFileWriterTests.coffee

@@ -15,8 +15,11 @@ describe "LocalFileWriter", ->
 			on: (type, cb)->
 				if type == "finish"
 					cb()
+		@readStream = 
+			on: ->
 		@fs = 
 			createWriteStream : sinon.stub().returns(@writeStream)
+			createReadStream: sinon.stub().returns(@readStream)
 			unlink: sinon.stub()
 		@settings =
 			path:
@@ -56,6 +59,18 @@ describe "LocalFileWriter", ->
 				fsPath.should.equal @stubbedFsPath
 				done()
 
+	describe "getStream", ->
+
+		it "should read the stream from the file ", (done)->
+			@writer.getStream @stubbedFsPath, (err, stream)=>
+				@fs.createReadStream.calledWith(@stubbedFsPath).should.equal true
+				done()
+
+		it "should send the stream in the callback", (done)->
+			@writer.getStream @stubbedFsPath, (err, readStream)=>
+				readStream.should.equal @readStream
+				done()
+
 	describe "delete file", ->
 
 		it "should unlink the file", (done)->