Merge branch 'master' into ja_email_tokens

services/web/app/coffee/Features/Authentication/AuthenticationManager.coffee

@@ -1,4 +1,3 @@
-Settings = require 'settings-sharelatex'
 User = require("../../models/User").User
 {db, ObjectId} = require("../../infrastructure/mongojs")
 crypto = require 'crypto'

services/web/app/coffee/Features/Blog/BlogController.coffee

@@ -4,15 +4,13 @@ logger = require("logger-sharelatex")
 _ = require("underscore")
 ErrorController = require "../Errors/ErrorController"
 
-extensionsToProxy = [".png", ".xml", ".jpeg", ".json", ".zip", ".eps"]
-
 module.exports = BlogController = 
 
 	getPage: (req, res, next)->
 		url = req.url?.toLowerCase()
 		blogUrl = "#{settings.apis.blog.url}#{url}"
 
-		extensionsToProxy = [".png", ".xml", ".jpeg", ".json", ".zip", ".eps"]
+		extensionsToProxy = [".png", ".xml", ".jpeg", ".json", ".zip", ".eps", ".gif"]
 
 		shouldProxy = _.find extensionsToProxy, (extension)->
 			url.indexOf(extension) != -1

services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee

@@ -1,6 +1,5 @@
 request = require 'request'
 request = request.defaults()
-async = require 'async'
 settings = require 'settings-sharelatex'
 _ = require 'underscore'
 async = require 'async'

services/web/app/coffee/Features/Editor/EditorController.coffee

@@ -33,10 +33,12 @@ module.exports = EditorController =
 		logger.log {project_id, folder_id, docName, source}, "sending new doc to project"
 		Metrics.inc "editor.add-doc"
 		ProjectEntityHandler.addDoc project_id, folder_id, docName, docLines, (err, doc, folder_id)=>
+			if err?
+				logger.err err:err, project_id:project_id, docName:docName, "error adding doc without lock"
+				return callback(err)
 			EditorRealTimeController.emitToRoom(project_id, 'reciveNewDoc', folder_id, doc, source)
 			callback(err, doc)
 
-
 	addFile: (project_id, folder_id, fileName, path, source, callback = (error, file)->)->
 		LockManager.getLock project_id, (err)->
 			if err?
@@ -46,20 +48,20 @@ module.exports = EditorController =
 				LockManager.releaseLock project_id, ->
 					callback(error, file)	
 
-
 	addFileWithoutLock: (project_id, folder_id, fileName, path, source, callback = (error, file)->)->
 		fileName = fileName.trim()
 		logger.log {project_id, folder_id, fileName, path}, "sending new file to project"
 		Metrics.inc "editor.add-file"
 		ProjectEntityHandler.addFile project_id, folder_id, fileName, path, (err, fileRef, folder_id)=>
+			if err?
+				logger.err err:err, project_id:project_id, folder_id:folder_id, fileName:fileName, "error adding file without lock"
+				return callback(err)
 			EditorRealTimeController.emitToRoom(project_id, 'reciveNewFile', folder_id, fileRef, source)
 			callback(err, fileRef)
 
 	replaceFile: (project_id, file_id, fsPath, source, callback = (error) ->)->
 		ProjectEntityHandler.replaceFile project_id, file_id, fsPath, callback
 
-
-
 	addFolder : (project_id, folder_id, folderName, source, callback = (error, folder)->)->
 		LockManager.getLock project_id, (err)->
 			if err?
@@ -74,6 +76,9 @@ module.exports = EditorController =
 		logger.log {project_id, folder_id, folderName, source}, "sending new folder to project"
 		Metrics.inc "editor.add-folder"
 		ProjectEntityHandler.addFolder project_id, folder_id, folderName, (err, folder, folder_id)=>
+			if err?
+				logger.err err:err, project_id:project_id, folder_id:folder_id, folderName:folderName, "error adding folder without lock"
+				return callback(err)
 			@p.notifyProjectUsersOfNewFolder project_id, folder_id, folder, (error) ->
 				callback error, folder
 
@@ -90,6 +95,9 @@ module.exports = EditorController =
 	mkdirpWithoutLock: (project_id, path, callback)->
 		logger.log project_id:project_id, path:path, "making directories if they don't exist"
 		ProjectEntityHandler.mkdirp project_id, path, (err, newFolders, lastFolder)=>
+			if err?
+				logger.err err:err, project_id:project_id, path:path, "error mkdirp without lock"
+				return callback(err)
 			self = @
 			jobs = _.map newFolders, (folder, index)->
 				return (cb)->
@@ -109,7 +117,10 @@ module.exports = EditorController =
 	deleteEntityWithoutLock: (project_id, entity_id, entityType, source, callback)->
 		logger.log {project_id, entity_id, entityType, source}, "start delete process of entity"
 		Metrics.inc "editor.delete-entity"
-		ProjectEntityHandler.deleteEntity project_id, entity_id, entityType, =>
+		ProjectEntityHandler.deleteEntity project_id, entity_id, entityType, (err)->
+			if err?
+				logger.err err:err, project_id:project_id, entity_id:entity_id, entityType:entityType, "error deleting entity"
+				return callback(err)
 			logger.log project_id:project_id, entity_id:entity_id, entityType:entityType, "telling users entity has been deleted"
 			EditorRealTimeController.emitToRoom(project_id, 'removeEntity', entity_id, source)
 			if callback?
@@ -143,19 +154,33 @@ module.exports = EditorController =
 		newName = sanitize.escape(newName)
 		Metrics.inc "editor.rename-entity"
 		logger.log entity_id:entity_id, entity_id:entity_id, entity_id:entity_id, "reciving new name for entity for project"
-		ProjectEntityHandler.renameEntity project_id, entity_id, entityType, newName, =>
+		ProjectEntityHandler.renameEntity project_id, entity_id, entityType, newName, ->
+			if err?
+				logger.err err:err, project_id:project_id, entity_id:entity_id, entityType:entityType, newName:newName, "error renaming entity"
+				return callback(err)
 			if newName.length > 0
 				EditorRealTimeController.emitToRoom project_id, 'reciveEntityRename', entity_id, newName
 				callback?()
-#
+
 	moveEntity: (project_id, entity_id, folder_id, entityType, callback)->
 		Metrics.inc "editor.move-entity"
-		ProjectEntityHandler.moveEntity project_id, entity_id, folder_id, entityType, =>
-			EditorRealTimeController.emitToRoom project_id, 'reciveEntityMove', entity_id, folder_id
-			callback?()
+		LockManager.getLock project_id, (err)->
+			if err?
+				logger.err err:err, project_id:project_id, "could not get lock for move entity"
+				return callback(err)
+			ProjectEntityHandler.moveEntity project_id, entity_id, folder_id, entityType, =>
+				if err?
+					logger.err err:err, project_id:project_id, entity_id:entity_id, folder_id:folder_id, "error moving entity"
+					return callback(err)
+				LockManager.releaseLock project_id, ->
+					EditorRealTimeController.emitToRoom project_id, 'reciveEntityMove', entity_id, folder_id
+					callback?()
 
 	renameProject: (project_id, newName, callback = (err) ->) ->
-		ProjectDetailsHandler.renameProject project_id, newName, =>
+		ProjectDetailsHandler.renameProject project_id, newName, ->
+			if err?
+				logger.err err:err, project_id:project_id, newName:newName, "error renaming project"
+				return callback(err)
 			EditorRealTimeController.emitToRoom project_id, 'projectNameUpdated', newName
 			callback()
 

services/web/app/coffee/Features/Editor/EditorHttpController.coffee

@@ -69,11 +69,16 @@ module.exports = EditorHttpController =
 		project_id = req.params.Project_id
 		name = req.body.name
 		parent_folder_id = req.body.parent_folder_id
+		logger.log project_id:project_id, name:name, parent_folder_id:parent_folder_id, "getting request to add doc to project"
 		if !EditorHttpController._nameIsAcceptableLength(name)
 			return res.sendStatus 400
 		EditorController.addDoc project_id, parent_folder_id, name, [], "editor", (error, doc) ->
-			return next(error) if error?
-			res.json doc
+			if error == "project_has_to_many_files"
+				res.status(400).json(req.i18n.translate("project_has_to_many_files"))
+			else if error?
+				next(error)
+			else
+				res.json doc
 
 	addFolder: (req, res, next) ->
 		project_id = req.params.Project_id
@@ -82,8 +87,12 @@ module.exports = EditorHttpController =
 		if !EditorHttpController._nameIsAcceptableLength(name)
 			return res.sendStatus 400
 		EditorController.addFolder project_id, parent_folder_id, name, "editor", (error, doc) ->
-			return next(error) if error?
-			res.json doc
+			if error == "project_has_to_many_files"
+				res.status(400).json(req.i18n.translate("project_has_to_many_files"))
+			else if error?
+				next(error)
+			else
+				res.json doc
 
 	renameEntity: (req, res, next) ->
 		project_id  = req.params.Project_id

services/web/app/coffee/Features/Email/EmailBuilder.coffee

@@ -1,5 +1,4 @@
 _ = require('underscore')
-
 PersonalEmailLayout = require("./Layouts/PersonalEmailLayout")
 NotificationEmailLayout = require("./Layouts/NotificationEmailLayout")
 settings = require("settings-sharelatex")

services/web/app/coffee/Features/Email/EmailSender.coffee

@@ -1,7 +1,6 @@
 logger = require('logger-sharelatex')
 metrics = require('../../infrastructure/Metrics')
 Settings = require('settings-sharelatex')
-metrics = require("../../infrastructure/Metrics")
 nodemailer = require("nodemailer")
 sesTransport = require('nodemailer-ses-transport')
 _ = require("underscore")
@@ -25,7 +24,6 @@ else if Settings?.email?.parameters?
 
 
 	logger.log "using smtp for email"
-	console.log smtp
 	nm_client = nodemailer.createTransport(smtp)
 else
 	nm_client = client

services/web/app/coffee/Features/FileStore/FileStoreHandler.coffee

@@ -6,24 +6,32 @@ settings = require("settings-sharelatex")
 oneMinInMs = 60 * 1000
 fiveMinsInMs = oneMinInMs * 5
 
-module.exports =
+module.exports = FileStoreHandler =
 
 	uploadFileFromDisk: (project_id, file_id, fsPath, callback)->
-		logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "uploading file from disk"
-		readStream = fs.createReadStream(fsPath)
-		opts =
-			method: "post"
-			uri: @_buildUrl(project_id, file_id)
-			timeout:fiveMinsInMs
-		writeStream = request(opts)
-		readStream.pipe writeStream
-		writeStream.on "end", callback
-		readStream.on "error", (err)->
-			logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the read stream of uploadFileFromDisk"
-			callback err
-		writeStream.on "error", (err)->
-			logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the write stream of uploadFileFromDisk"
-			callback err
+		fs.lstat fsPath, (err, stat)->
+			if err?
+				logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "error stating file"
+				callback(err)
+			if !stat.isFile()
+				logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "tried to upload symlink, not contining"
+				return callback(new Error("can not upload symlink"))
+
+			logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "uploading file from disk"
+			readStream = fs.createReadStream(fsPath)
+			opts =
+				method: "post"
+				uri: FileStoreHandler._buildUrl(project_id, file_id)
+				timeout:fiveMinsInMs
+			writeStream = request(opts)
+			readStream.pipe writeStream
+			writeStream.on "end", callback
+			readStream.on "error", (err)->
+				logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the read stream of uploadFileFromDisk"
+				callback err
+			writeStream.on "error", (err)->
+				logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the write stream of uploadFileFromDisk"
+				callback err
 
 	getFileStream: (project_id, file_id, query, callback)->
 		logger.log project_id:project_id, file_id:file_id, query:query, "getting file stream from file store"

services/web/app/coffee/Features/Project/ProjectController.coffee

@@ -61,7 +61,7 @@ module.exports = ProjectController =
 	deleteProject: (req, res) ->
 		project_id = req.params.Project_id
 		forever    = req.query?.forever?
-		logger.log project_id: project_id, forever: forever, "received request to delete project"
+		logger.log project_id: project_id, forever: forever, "received request to archive project"
 
 		if forever
 			doDelete = projectDeleter.deleteProject

services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee

@@ -35,7 +35,9 @@ module.exports =
 			self._buildTemplate "mainbasic.tex", owner_id, projectName, (error, docLines)->
 				return callback(error) if error?
 				ProjectEntityHandler.addDoc project._id, project.rootFolder[0]._id, "main.tex", docLines, (error, doc)->
-					return callback(error) if error?
+					if error?
+						logger.err err:error, "error adding doc when creating basic project"
+						return callback(error)
 					ProjectEntityHandler.setRootDoc project._id, doc._id, (error) ->
 						callback(error, project)
 

services/web/app/coffee/Features/Project/ProjectDeleter.coffee

@@ -1,4 +1,5 @@
 Project = require('../../models/Project').Project
+ProjectGetter = require("./ProjectGetter")
 logger = require('logger-sharelatex')
 documentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
 tagsHandler = require("../Tags/TagsHandler")
@@ -34,10 +35,10 @@ module.exports = ProjectDeleter =
 			Project.remove _id: project_id, callback
 
 	archiveProject: (project_id, callback = (error) ->)->
-		logger.log project_id:project_id, "deleting project"
-		Project.findById project_id, (err, project)=>
+		logger.log project_id:project_id, "archived project from user request"
+		ProjectGetter.getProject project_id, {owner_ref:true, collaberator_refs:true, readOnly_refs:true}, (err, project)=>
 			if err? or !project?
-				logger.err err:err, project_id:project_id, "error getting project to delete it"
+				logger.err err:err, project_id:project_id, "error getting project to archived it"
 				callback(err)
 			else
 				async.series [
@@ -52,8 +53,10 @@ module.exports = ProjectDeleter =
 						Project.update {_id:project_id}, { $set: { archived: true }}, cb
 				], (err)->
 					if err?
-						logger.err err:err, "problem deleting project"
-					callback(err)
+						logger.err err:err, "problem archived project"
+						return callback(err)
+					logger.log project_id:project_id, "succesfully archived project from user request"
+					callback()
 
 	restoreProject: (project_id, callback = (error) ->) ->
 		Project.update {_id:project_id}, { $unset: { archived: true }}, callback

services/web/app/coffee/Features/Project/ProjectDetailsHandler.coffee

@@ -9,7 +9,7 @@ PublicAccessLevels = require("../Authorization/PublicAccessLevels")
 module.exports = 
 
 	getDetails: (project_id, callback)->
-		ProjectGetter.getProjectWithoutDocLines project_id, (err, project)->
+		ProjectGetter.getProject project_id, {name:true, description:true, compiler:true, features:true, owner_ref:true}, (err, project)->
 			if err?
 				logger.err err:err, project_id:project_id, "error getting project"
 				return callback(err)
@@ -38,7 +38,7 @@ module.exports =
 
 	renameProject: (project_id, newName, callback = ->)->
 		logger.log project_id: project_id, newName:newName, "renaming project"
-		ProjectGetter.getProject project_id, {"name":1}, (err, project)->
+		ProjectGetter.getProject project_id, {name:true}, (err, project)->
 			if err? or !project?
 				logger.err err:err,  project_id:project_id, "error getting project or could not find it todo project rename"
 				return callback(err)

services/web/app/coffee/Features/Project/ProjectDuplicator.coffee

@@ -4,61 +4,86 @@ projectLocator = require('./ProjectLocator')
 projectOptionsHandler = require('./ProjectOptionsHandler')
 DocumentUpdaterHandler = require("../DocumentUpdater/DocumentUpdaterHandler")
 DocstoreManager = require "../Docstore/DocstoreManager"
-Project = require("../../models/Project").Project
+ProjectGetter = require("./ProjectGetter")
 _ = require('underscore')
 async = require('async')
+logger = require("logger-sharelatex")
 
-module.exports =
-	duplicate: (owner, originalProjectId, newProjectName, callback)->
-		DocumentUpdaterHandler.flushProjectToMongo originalProjectId, (err) ->
-			return callback(err) if err?
-			Project.findById originalProjectId, (err, originalProject) ->
-				return callback(err) if err?
-				projectCreationHandler.createBlankProject owner._id, newProjectName, (err, newProject)->
-					return callback(err) if err?
-					projectLocator.findRootDoc {project:originalProject}, (err, originalRootDoc)->
-						return callback(err) if err?
-						DocstoreManager.getAllDocs originalProjectId, (err, docContentsArray) ->
-							return callback(err) if err?
-
-							docContents = {}
-							for docContent in docContentsArray
-								docContents[docContent._id] = docContent
-
-							projectOptionsHandler.setCompiler newProject._id, originalProject.compiler
-
-							setRootDoc = _.once (doc_id)->
-								projectEntityHandler.setRootDoc newProject, doc_id
-
-							copyDocs = (originalFolder, newParentFolder, callback)->
-								jobs = originalFolder.docs.map (doc)->
-									return (callback)->
-										content = docContents[doc._id.toString()]
-										return callback(new Error("doc_id not found: #{doc._id}")) if !content?
-										projectEntityHandler.addDoc newProject._id, newParentFolder._id, doc.name, content.lines, (err, newDoc)->
-											if originalRootDoc? and newDoc.name == originalRootDoc.name
-												setRootDoc newDoc._id
-											callback()
-								async.series jobs, callback
-
-							copyFiles = (originalFolder, newParentFolder, callback)->
-								jobs = originalFolder.fileRefs.map (file)->
-									return (callback)->
-										projectEntityHandler.copyFileFromExistingProject newProject, newParentFolder._id, originalProject._id, file, callback
-								async.parallelLimit jobs, 5, callback
-
-							copyFolder = (folder, desFolder, callback)->
-								jobs = folder.folders.map (childFolder)->
-									return (callback)->
-										projectEntityHandler.addFolder newProject, desFolder._id, childFolder.name, (err, newFolder)->
-											copyFolder childFolder, newFolder, callback
-								jobs.push (cb)->
-									copyDocs folder, desFolder, cb
-								jobs.push (cb)->
-									copyFiles folder, desFolder, cb
-
-								async.series jobs, callback
-
-							copyFolder originalProject.rootFolder[0], newProject.rootFolder[0], ->
-								callback(err, newProject)
 
+module.exports = ProjectDuplicator =
+
+	_copyDocs: (newProject, originalRootDoc, originalFolder, desFolder, docContents, callback)->
+		setRootDoc = _.once (doc_id)->
+			projectEntityHandler.setRootDoc newProject._id, doc_id
+
+		jobs = originalFolder.docs.map (doc)->
+			return (cb)->
+				content = docContents[doc._id.toString()]
+				projectEntityHandler.addDocWithProject newProject, desFolder._id, doc.name, content.lines, (err, newDoc)->
+					if err?
+						logger.err err:err, "error copying doc"
+						return callback(err)
+					if originalRootDoc? and newDoc.name == originalRootDoc.name
+						setRootDoc newDoc._id
+					cb()
+
+		async.series jobs, callback
+
+	_copyFiles: (newProject, originalProject_id, originalFolder, desFolder, callback)->
+		jobs = originalFolder.fileRefs.map (file)->
+			return (cb)->
+				projectEntityHandler.copyFileFromExistingProjectWithProject newProject, desFolder._id, originalProject_id, file, cb
+		async.parallelLimit jobs, 5, callback
+
+
+	_copyFolderRecursivly: (newProject_id, originalProject_id, originalRootDoc, originalFolder, desFolder, docContents, callback)->
+		ProjectGetter.getProject newProject_id, {rootFolder:true, name:true}, (err, newProject)->
+			if err?
+				logger.err project_id:newProject_id, "could not get project"
+				return cb(err)
+
+			jobs = originalFolder.folders.map (childFolder)->
+				return (cb)->
+					projectEntityHandler.addFolderWithProject newProject, desFolder?._id, childFolder.name, (err, newFolder)->
+						return cb(err) if err?
+						ProjectDuplicator._copyFolderRecursivly newProject_id, originalProject_id, originalRootDoc, childFolder, newFolder, docContents, cb
+
+			jobs.push (cb)->
+				ProjectDuplicator._copyFiles newProject, originalProject_id, originalFolder, desFolder, cb
+			jobs.push (cb)->
+				ProjectDuplicator._copyDocs newProject, originalRootDoc, originalFolder, desFolder, docContents, cb
+
+			async.series jobs, callback
+
+	duplicate: (owner, originalProject_id, newProjectName, callback)->
+
+		jobs = 
+			flush: (cb)->
+				DocumentUpdaterHandler.flushProjectToMongo originalProject_id, cb
+			originalProject: (cb)->
+				ProjectGetter.getProject originalProject_id, {compiler:true, rootFolder:true, rootDoc_id:true}, cb
+			newProject: (cb)->
+				projectCreationHandler.createBlankProject owner._id, newProjectName, cb
+			originalRootDoc: (cb)->
+				projectLocator.findRootDoc {project_id:originalProject_id}, cb
+			docContentsArray: (cb)-> 
+				DocstoreManager.getAllDocs originalProject_id, cb
+
+		async.series jobs, (err, results)->
+			if err?
+				logger.err err:err, originalProject_id:originalProject_id, "error duplicating project"
+				return callback(err)
+			{originalProject, newProject, originalRootDoc, docContentsArray} = results
+
+			originalRootDoc = originalRootDoc[0]
+
+			docContents = {}
+			for docContent in docContentsArray
+				docContents[docContent._id] = docContent
+			
+			projectOptionsHandler.setCompiler newProject._id, originalProject.compiler, ->
+
+			ProjectDuplicator._copyFolderRecursivly newProject._id, originalProject_id, originalRootDoc, originalProject.rootFolder[0], newProject.rootFolder[0], docContents, ->
+				if err?
+					logger.err err:err, originalProject_id:originalProject_id,  newProjectName:newProjectName, "error cloning project"
+				callback(err, newProject)

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -1,4 +1,5 @@
 Project = require('../../models/Project').Project
+settings = require "settings-sharelatex"
 Doc = require('../../models/Doc').Doc
 Folder = require('../../models/Folder').Folder
 File = require('../../models/File').File
@@ -75,23 +76,21 @@ module.exports = ProjectEntityHandler =
 		documentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler')
 		documentUpdaterHandler.flushProjectToMongo project_id, (error) ->
 			return callback(error) if error?
-			Project.findById project_id, (error, project) ->
+			ProjectGetter.getProject project_id, {name:true}, (error, project) ->
 				return callback(error) if error?
 				requests = []
 				self.getAllDocs project_id, (error, docs) ->
 					return callback(error) if error?
 					for docPath, doc of docs
 						do (docPath, doc) ->
-							requests.push (callback) ->
-								tpdsUpdateSender.addDoc {project_id:project_id, doc_id:doc._id, path:docPath, project_name:project.name, rev:doc.rev||0},
-									callback
+							requests.push (cb) ->
+								tpdsUpdateSender.addDoc {project_id:project_id, doc_id:doc._id, path:docPath, project_name:project.name, rev:doc.rev||0}, cb
 					self.getAllFiles project_id, (error, files) ->
 						return callback(error) if error?
 						for filePath, file of files
 							do (filePath, file) ->
-								requests.push (callback) ->
-									tpdsUpdateSender.addFile {project_id:project_id, file_id:file._id, path:filePath, project_name:project.name, rev:file.rev},
-										callback
+								requests.push (cb) ->
+									tpdsUpdateSender.addFile {project_id:project_id, file_id:file._id, path:filePath, project_name:project.name, rev:file.rev}, cb
 						async.series requests, (err) ->
 							logger.log project_id:project_id, "finished flushing project to tpds"
 							callback(err)
@@ -110,27 +109,36 @@ module.exports = ProjectEntityHandler =
 			options = {}
 		DocstoreManager.getDoc project_id, doc_id, options, callback
 
+
 	addDoc: (project_id, folder_id, docName, docLines, callback = (error, doc, folder_id) ->)=>
 		ProjectGetter.getProjectWithOnlyFolders project_id, (err, project) ->
-			logger.log project: project._id, folder_id: folder_id, doc_name: docName, "adding doc"
-			return callback(err) if err?
-			confirmFolder project, folder_id, (folder_id)=>
-				doc = new Doc name: docName
-				# Put doc in docstore first, so that if it errors, we don't have a doc_id in the project
-				# which hasn't been created in docstore.
-				DocstoreManager.updateDoc project._id.toString(), doc._id.toString(), docLines, (err, modified, rev) ->
-					return callback(err) if err? 
-					Project.putElement project._id, folder_id, doc, "doc", (err, result)=>
-						return callback(err) if err?
-						tpdsUpdateSender.addDoc {
-							project_id:   project._id,
-							doc_id:		  doc._id	
-							path:         result.path.fileSystem,
-							project_name: project.name,
-							rev:          0
-						}, (err) ->
-							return callback(err) if err?
-							callback(null, doc, folder_id)
+			if err?
+				logger.err project_id:project_id, err:err, "error getting project for add doc"
+				return callback(err)
+			ProjectEntityHandler.addDocWithProject project, folder_id, docName, docLines, callback
+
+	addDocWithProject: (project, folder_id, docName, docLines, callback = (error, doc, folder_id) ->)=>
+		project_id = project._id
+		logger.log project_id: project_id, folder_id: folder_id, doc_name: docName, "adding doc to project with project"
+		confirmFolder project, folder_id, (folder_id)=>
+			doc = new Doc name: docName
+			# Put doc in docstore first, so that if it errors, we don't have a doc_id in the project
+			# which hasn't been created in docstore.
+			DocstoreManager.updateDoc project_id.toString(), doc._id.toString(), docLines, (err, modified, rev) ->
+				return callback(err) if err? 
+
+				ProjectEntityHandler._putElement project, folder_id, doc, "doc", (err, result)=>
+					return callback(err) if err?
+					tpdsUpdateSender.addDoc {
+						project_id:   project_id,
+						doc_id:		  doc?._id
+						path:         result?.path?.fileSystem,
+						project_name: project.name,
+						rev:          0
+					}, (err) ->
+						if err?
+							logger.err err:err, "error adding doc to tpdsworker, contining anyway"
+						callback(null, doc, folder_id)
 
 	restoreDoc: (project_id, doc_id, name, callback = (error, doc, folder_id) ->) ->
 		# getDoc will return the deleted doc's lines, but we don't actually remove
@@ -139,22 +147,34 @@ module.exports = ProjectEntityHandler =
 			return callback(error) if error?
 			ProjectEntityHandler.addDoc project_id, null, name, lines, callback
 
-	addFile: (project_or_id, folder_id, fileName, path, callback = (error, fileRef, folder_id) ->)->
-		ProjectGetter.getProjectWithOnlyFolders project_or_id, (err, project) ->
-			logger.log project_id: project._id, folder_id: folder_id, file_name: fileName, path:path, "adding file"
-			return callback(err) if err?
-			confirmFolder project, folder_id, (folder_id)->
-				fileRef = new File name : fileName
-				FileStoreHandler.uploadFileFromDisk project._id, fileRef._id, path, (err)->
+	addFile: (project_id, folder_id, fileName, path, callback = (error, fileRef, folder_id) ->)->
+		ProjectGetter.getProjectWithOnlyFolders project_id, (err, project) ->
+			if err?
+				logger.err project_id:project_id, err:err, "error getting project for add file"
+				return callback(err)
+			ProjectEntityHandler.addFileWithProject project, folder_id, fileName, path, callback
+
+	addFileWithProject: (project, folder_id, fileName, path, callback = (error, fileRef, folder_id) ->)->
+		project_id = project._id
+		logger.log project_id: project._id, folder_id: folder_id, file_name: fileName, path:path, "adding file"
+		return callback(err) if err?
+		confirmFolder project, folder_id, (folder_id)->
+			fileRef = new File name : fileName
+			FileStoreHandler.uploadFileFromDisk project._id, fileRef._id, path, (err)->
+				if err?
+					logger.err err:err, project_id: project._id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
+					return callback(err)
+				ProjectEntityHandler._putElement project, folder_id, fileRef, "file", (err, result)=>
 					if err?
-						logger.err err:err, project_id: project._id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error uploading image to s3"
+						logger.err err:err, project_id: project._id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error adding file with project"
 						return callback(err)
-					Project.putElement project._id, folder_id, fileRef, "file", (err, result)=>
-						tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result.path.fileSystem, project_name:project.name, rev:fileRef.rev}, ->
-							callback(err, fileRef, folder_id)
+					tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result?.path?.fileSystem, project_name:project.name, rev:fileRef.rev}, (err)->
+						if err?
+							logger.err err:err, project_id: project._id, folder_id: folder_id, file_name: fileName, fileRef:fileRef, "error sending file to tpdsworker"
+						callback(null, fileRef, folder_id)
 
-	replaceFile: (project_or_id, file_id, fsPath, callback)->
-		Project.getProject project_or_id, "", (err, project) ->
+	replaceFile: (project_id, file_id, fsPath, callback)->
+		ProjectGetter.getProject project_id, {name:true}, (err, project) ->
 			return callback(err) if err?
 			findOpts = 
 				project_id:project._id
@@ -182,21 +202,36 @@ module.exports = ProjectEntityHandler =
 						Project.update conditons, update, {}, (err, second)->
 							callback()
 
-	copyFileFromExistingProject: (project_or_id, folder_id, originalProject_id, origonalFileRef, callback = (error, fileRef, folder_id) ->)->
-		Project.getProject project_or_id, "", (err, project) ->
-			logger.log project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "copying file in s3"
-			return callback(err) if err?
-			confirmFolder project, folder_id, (folder_id)=>
-				if !origonalFileRef?
-					logger.err project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "file trying to copy is null"
-					return callback()
-				fileRef = new File name : origonalFileRef.name
-				FileStoreHandler.copyFile originalProject_id, origonalFileRef._id, project._id, fileRef._id, (err)->
+	copyFileFromExistingProject: (project_id, folder_id, originalProject_id, origonalFileRef, callback = (error, fileRef, folder_id) ->)->
+		logger.log project_id:project_id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "copying file in s3"
+		ProjectGetter.getProject project_id, {name:true}, (err, project) ->
+			if err?
+				logger.err project_id:project_id, err:err, "error getting project for copy file from existing project"
+				return callback(err)
+			ProjectEntityHandler.copyFileFromExistingProjectWithProject project, folder_id, originalProject_id, origonalFileRef, callback
+
+
+	copyFileFromExistingProjectWithProject: (project, folder_id, originalProject_id, origonalFileRef, callback = (error, fileRef, folder_id) ->)->
+		project_id = project._id
+		logger.log project_id:project_id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "copying file in s3 with project"
+		return callback(err) if err?
+		confirmFolder project, folder_id, (folder_id)=>
+			if !origonalFileRef?
+				logger.err project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "file trying to copy is null"
+				return callback()
+			fileRef = new File name : origonalFileRef.name
+			FileStoreHandler.copyFile originalProject_id, origonalFileRef._id, project._id, fileRef._id, (err)->
+				if err?
+					logger.err err:err, project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "error coping file in s3"
+					return callback(err)
+				ProjectEntityHandler._putElement project, folder_id, fileRef, "file", (err, result)=>
 					if err?
-						logger.err err:err, project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "error coping file in s3"
-					Project.putElement project._id, folder_id, fileRef, "file", (err, result)=>
-						tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result.path.fileSystem, rev:fileRef.rev, project_name:project.name}, (error) ->
-							callback(error, fileRef, folder_id)
+						logger.err err:err, project_id:project._id, folder_id:folder_id, "error putting element as part of copy"
+						return callback(err)
+					tpdsUpdateSender.addFile {project_id:project._id, file_id:fileRef._id, path:result?.path?.fileSystem, rev:fileRef.rev, project_name:project.name}, (err) ->
+						if err?
+							logger.err err:err,  project_id:project._id, folder_id:folder_id, originalProject_id:originalProject_id, origonalFileRef:origonalFileRef, "error sending file to tpds worker"
+						callback(null, fileRef, folder_id)
 
 	mkdirp: (project_id, path, callback = (err, newlyCreatedFolders, lastFolderInPath)->)->
 		self = @
@@ -204,7 +239,7 @@ module.exports = ProjectEntityHandler =
 		folders = _.select folders, (folder)->
 			return folder.length != 0
 
-		ProjectGetter.getProjectWithoutDocLines project_id, (err, project)=>
+		ProjectGetter.getProjectWithOnlyFolders project_id, (err, project)=>
 			if path == '/'
 				logger.log project_id: project._id, "mkdir is only trying to make path of / so sending back root folder"
 				return callback(null, [], project.rootFolder[0])
@@ -217,7 +252,7 @@ module.exports = ProjectEntityHandler =
 				if parentFolder?  
 					parentFolder_id = parentFolder._id
 				builtUpPath = "#{builtUpPath}/#{folderName}"
-				projectLocator.findElementByPath project_id, builtUpPath, (err, foundFolder)=>
+				projectLocator.findElementByPath project, builtUpPath, (err, foundFolder)=>
 					if !foundFolder?
 						logger.log path:path, project_id:project._id, folderName:folderName, "making folder from mkdirp"
 						@addFolder project_id, parentFolder_id, folderName, (err, newFolder, parentFolder_id)->
@@ -235,16 +270,23 @@ module.exports = ProjectEntityHandler =
 				folders = _.select folders, (folder)->
 					!folder.filterOut
 				callback(null, folders, lastFolder)
-	
-	addFolder: (project_or_id, parentFolder_id, folderName, callback) ->
-		folder = new Folder name: folderName
-		Project.getProject project_or_id, "", (err, project) ->
-			return callback(err) if err?
-			confirmFolder project, parentFolder_id, (parentFolder_id)=>
-				logger.log project: project_or_id, parentFolder_id:parentFolder_id, folderName:folderName, "new folder added"
-				Project.putElement project._id, parentFolder_id, folder, "folder", (err, result)=>
-					if callback?
-						callback(err, folder, parentFolder_id)
+
+	addFolder: (project_id, parentFolder_id, folderName, callback) ->
+		ProjectGetter.getProjectWithOnlyFolders project_id, (err, project)=>
+			if err?
+				logger.err project_id:project_id, err:err, "error getting project for add folder"
+				return callback(err)
+			ProjectEntityHandler.addFolderWithProject project, parentFolder_id, folderName, callback
+
+	addFolderWithProject: (project, parentFolder_id, folderName, callback = (err, folder, parentFolder_id)->) ->
+		confirmFolder project, parentFolder_id, (parentFolder_id)=>
+			folder = new Folder name: folderName
+			logger.log project: project._id, parentFolder_id:parentFolder_id, folderName:folderName, "adding new folder"
+			ProjectEntityHandler._putElement project, parentFolder_id, folder, "folder", (err, result)=>
+				if err?
+					logger.err err:err, project_id:project._id, "error adding folder to project"
+					return callback(err)
+				callback(err, folder, parentFolder_id)
 
 	updateDocLines : (project_id, doc_id, lines, callback = (error) ->)->
 		ProjectGetter.getProjectWithoutDocLines project_id, (err, project)->
@@ -281,7 +323,7 @@ module.exports = ProjectEntityHandler =
 			logger.err err: "No entityType set", project_id: project_id, entity_id: entity_id
 			return callback("No entityType set")
 		entityType = entityType.toLowerCase()
-		Project.findById project_id, (err, project)=>
+		ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project)=>
 			return callback(err) if err?
 			projectLocator.findElement {project:project, element_id:entity_id, type:entityType}, (err, entity, path)->
 				return callback(err) if err?
@@ -302,15 +344,18 @@ module.exports = ProjectEntityHandler =
 					return callback(error) if error?
 					self._removeElementFromMongoArray Project, project_id, path.mongo, (err)->
 						return callback(err) if err?
-						Project.putElement project_id, destinationFolder_id, entity, entityType, (err, result)->
+						# We've updated the project structure by removing the element, so must refresh it.
+						ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project)=>
 							return callback(err) if err?
-							opts = 
-								project_id:project_id
-								project_name:project.name
-								startPath:path.fileSystem
-								endPath:result.path.fileSystem,
-								rev:entity.rev
-							tpdsUpdateSender.moveEntity opts, callback
+							ProjectEntityHandler._putElement project, destinationFolder_id, entity, entityType, (err, result)->
+								return callback(err) if err?
+								opts = 
+									project_id:project_id
+									project_name:project.name
+									startPath:path.fileSystem
+									endPath:result.path.fileSystem,
+									rev:entity.rev
+								tpdsUpdateSender.moveEntity opts, callback
 
 	deleteEntity: (project_id, entity_id, entityType, callback = (error) ->)->
 		self = @
@@ -319,7 +364,7 @@ module.exports = ProjectEntityHandler =
 			logger.err err: "No entityType set", project_id: project_id, entity_id: entity_id
 			return callback("No entityType set")
 		entityType = entityType.toLowerCase()
-		Project.findById project_id, (err, project)=>
+		ProjectGetter.getProject project_id, {name:true, rootFolder:true}, (err, project)=>
 			return callback(error) if error?
 			projectLocator.findElement {project: project, element_id: entity_id, type: entityType}, (error, entity, path)=>
 				return callback(error) if error?
@@ -338,7 +383,7 @@ module.exports = ProjectEntityHandler =
 			logger.err err: "No entityType set", project_id: project_id, entity_id: entity_id
 			return callback("No entityType set")
 		entityType = entityType.toLowerCase()
-		Project.findById project_id, (err, project)=>
+		ProjectGetter.getProject project_id, {rootFolder:true, name:true}, (err, project)=>
 			projectLocator.findElement {project:project, element_id:entity_id, type:entityType}, (err, entity, path, folder)=>
 				if err?
 					return callback err
@@ -426,6 +471,72 @@ module.exports = ProjectEntityHandler =
 			}
 		}, {}, callback
 
+
+	_countElements : (project, callback)->
+	
+		countFolder = (folder, cb = (err, count)->)->
+
+			jobs = _.map folder?.folders, (folder)->
+				(asyncCb)-> countFolder folder, asyncCb
+
+			async.series jobs, (err, subfolderCounts)->
+				total = 0
+
+				if subfolderCounts?.length > 0
+					total = _.reduce subfolderCounts, (a, b)-> return a + b
+				if folder?.folders?.length?
+					total += folder?.folders?.length
+				if folder?.docs?.length?
+					total += folder?.docs?.length
+				if folder?.fileRefs?.length?
+					total += folder?.fileRefs?.length
+				cb(null, total)
+
+		countFolder project.rootFolder[0], callback
+
+	_putElement: (project, folder_id, element, type, callback = (err, path)->)->
+
+		sanitizeTypeOfElement = (elementType)->
+			lastChar = elementType.slice -1
+			if lastChar != "s"
+				elementType +="s"
+			if elementType == "files"
+				elementType = "fileRefs"
+			return elementType
+
+		if !element? or !element._id?
+			e = new Error("no element passed to be inserted")
+			logger.err project_id:project._id, folder_id:folder_id, element:element, type:type, "failed trying to insert element as it was null"
+			return callback(e)
+		type = sanitizeTypeOfElement type
+
+		if !folder_id?
+			folder_id = project.rootFolder[0]._id
+		ProjectEntityHandler._countElements project, (err, count)->
+			if count > settings.maxEntitiesPerProject
+				logger.warn project_id:project._id, "project too big, stopping insertions"
+				return callback("project_has_to_many_files")
+			projectLocator.findElement {project:project, element_id:folder_id, type:"folders"}, (err, folder, path)=>
+				if err?
+					logger.err err:err, project_id:project._id, folder_id:folder_id, type:type, element:element, "error finding folder for _putElement"
+					return callback(err)
+				newPath =
+					fileSystem: "#{path.fileSystem}/#{element.name}"
+					mongo: path.mongo
+				id = element._id+''
+				element._id = require('mongoose').Types.ObjectId(id)
+				conditions = _id:project._id
+				mongopath = "#{path.mongo}.#{type}"
+				update = "$push":{}
+				update["$push"][mongopath] = element
+				logger.log project_id: project._id, element_id: element._id, fileType: type, folder_id: folder_id, mongopath:mongopath, "adding element to project"
+				Project.update conditions, update, {}, (err)->
+					if err?
+						logger.err err: err, project_id: project._id, 'error saving in putElement project'
+						return callback(err)
+					callback(err, {path:newPath})
+
+
 confirmFolder = (project, folder_id, callback)->
 	logger.log folder_id:folder_id, project_id:project._id, "confirming folder in project"
 	if folder_id+'' == 'undefined'

services/web/app/coffee/Features/Project/ProjectGetter.coffee

@@ -3,31 +3,53 @@ db = mongojs.db
 ObjectId = mongojs.ObjectId
 async = require "async"
 Project = require("../../models/Project").Project
+Errors = require("../../errors")
+logger = require("logger-sharelatex")
 
 module.exports = ProjectGetter =
 	EXCLUDE_DEPTH: 8
 
+
 	getProjectWithoutDocLines: (project_id, callback=(error, project) ->) ->
 		excludes = {}
-		for i in [1..@EXCLUDE_DEPTH]
+		for i in [1..ProjectGetter.EXCLUDE_DEPTH]
 			excludes["rootFolder#{Array(i).join(".folder")}.docs.lines"] = 0
 		db.projects.find _id: ObjectId(project_id.toString()), excludes, (error, projects = []) ->
 			callback error, projects[0]
 
 	getProjectWithOnlyFolders: (project_id, callback=(error, project) ->) ->
 		excludes = {}
-		for i in [1..@EXCLUDE_DEPTH]
+		for i in [1..ProjectGetter.EXCLUDE_DEPTH]
 			excludes["rootFolder#{Array(i).join(".folder")}.docs"] = 0
 			excludes["rootFolder#{Array(i).join(".folder")}.fileRefs"] = 0
 		db.projects.find _id: ObjectId(project_id.toString()), excludes, (error, projects = []) ->
 			callback error, projects[0]
 
+
 	getProject: (query, projection, callback = (error, project) ->) ->
+		if !query?
+			return callback("no query provided")
+
+		if typeof(projection) == "function"
+			callback = projection
+
 		if typeof query == "string"
 			query = _id: ObjectId(query)
 		else if query instanceof ObjectId
 			query = _id: query
-		db.projects.findOne query, projection, callback
+		else if query?.toString().length == 24 # sometimes mongoose ids are hard to identify, this will catch them
+			query = _id:  ObjectId(query.toString())
+		else
+			err = new Error("malformed get request")
+			logger.log query:query, err:err, type:typeof(query), "malformed get request"
+			return callback(err)
+
+		db.projects.find query, projection, (err, project)->
+			if err?
+				logger.err err:err, query:query, projection:projection, "error getting project"
+				return callback(err)
+			callback(null, project?[0])
+
 	
 	findAllUsersProjects: (user_id, fields, callback = (error, ownedProjects, readAndWriteProjects, readOnlyProjects) ->) ->
 		CollaboratorsHandler = require "../Collaborators/CollaboratorsHandler"

services/web/app/coffee/Features/Project/ProjectLocator.coffee

@@ -1,11 +1,12 @@
 Project = require('../../models/Project').Project
+ProjectGetter = require("./ProjectGetter")
 Errors = require "../../errors"
 _ = require('underscore')
 logger = require('logger-sharelatex')
 async = require('async')
 ProjectGetter = require "./ProjectGetter"
 
-module.exports =
+module.exports = ProjectLocator =
 	findElement: (options, _callback = (err, element, path, parentFolder)->)->
 		callback = (args...) ->
 			_callback(args...)
@@ -51,7 +52,7 @@ module.exports =
 		if project?
 			startSearch(project)
 		else
-			Project.findById project_id, (err, project)->
+			ProjectGetter.getProject project_id, {rootFolder:true, rootDoc_id:true}, (err, project)->
 				return callback(err) if err?
 				if !project?
 					return callback(new Errors.NotFoundError("project not found"))
@@ -67,8 +68,12 @@ module.exports =
 		if project?
 			getRootDoc project
 		else
-			Project.findById project_id, (err, project)->
-				getRootDoc project
+			ProjectGetter.getProject project_id, {rootFolder:true, rootDoc_id:true}, (err, project)->
+				if err?
+					logger.err err:err, "error getting project"
+					return callback(err)
+				else
+					getRootDoc project
 
 	findElementByPath: (project_or_id, needlePath, callback = (err, foundEntity)->)->
 

services/web/app/coffee/Features/Spelling/SpellingController.coffee

@@ -12,5 +12,4 @@ module.exports = SpellingController =
 		request(url: Settings.apis.spelling.url + url, method: req.method, headers: req.headers, json: req.body, timeout:TEN_SECONDS)
 		.on "error", (error) ->
 			logger.error err: error, "Spelling API error"
-			res.sendStatus 500
 		.pipe(res)

services/web/app/coffee/Features/Subscription/SubscriptionController.coffee

@@ -1,7 +1,6 @@
 AuthenticationController = require '../Authentication/AuthenticationController'
 SubscriptionHandler  = require './SubscriptionHandler'
 PlansLocator = require("./PlansLocator")
-SubscriptionFormatters = require("./SubscriptionFormatters")
 SubscriptionViewModelBuilder = require('./SubscriptionViewModelBuilder')
 LimitationsManager = require("./LimitationsManager")
 RecurlyWrapper = require './RecurlyWrapper'

services/web/app/coffee/Features/Subscription/SubscriptionGroupController.coffee

@@ -1,10 +1,7 @@
 SubscriptionGroupHandler = require("./SubscriptionGroupHandler")
 logger = require("logger-sharelatex")
 SubscriptionLocator = require("./SubscriptionLocator")
-
 ErrorsController = require("../Errors/ErrorController")
-settings = require("settings-sharelatex")
-
 SubscriptionDomainHandler = require("./SubscriptionDomainHandler")
 _ = require("underscore")
 
@@ -90,11 +87,12 @@ module.exports =
 		logger.log subscription_id:subscription_id, user_id:req?.session?.user?._id, email:email, "starting the completion of joining group"
 		SubscriptionGroupHandler.processGroupVerification email, subscription_id, req.query?.token, (err)->
 			if err? and err == "token_not_found"
-				res.redirect "/user/subscription/#{subscription_id}/group/invited?expired=true"
+				return res.redirect "/user/subscription/#{subscription_id}/group/invited?expired=true"
 			else if err?
-				res.sendStatus 500
+				return res.sendStatus 500
 			else
-				res.redirect "/user/subscription/#{subscription_id}/group/successful-join"
+				logger.log subscription_id:subscription_id, email:email, "user successful completed join of group subscription"
+				return res.redirect "/user/subscription/#{subscription_id}/group/successful-join"
 
 	renderSuccessfulJoinPage: (req, res)->
 		subscription_id = req.params.subscription_id

services/web/app/coffee/Features/Subscription/SubscriptionGroupHandler.coffee

@@ -14,9 +14,10 @@ NotificationsBuilder = require("../Notifications/NotificationsBuilder")
 module.exports = SubscriptionGroupHandler =
 
 	addUserToGroup: (adminUserId, newEmail, callback)->
+		logger.log adminUserId:adminUserId, newEmail:newEmail, "adding user to group"
 		UserCreator.getUserOrCreateHoldingAccount newEmail, (err, user)->
 			if err?
-				logger.err err:err, "error creating user for holding account"
+				logger.err err:err,  adminUserId:adminUserId, newEmail:newEmail, "error creating user for holding account"
 				return callback(err)
 			if !user?
 				msg = "no user returned whenc reating holidng account or getting user"
@@ -24,8 +25,10 @@ module.exports = SubscriptionGroupHandler =
 				return callback(msg)
 			LimitationsManager.hasGroupMembersLimitReached adminUserId, (err, limitReached, subscription)->
 				if err?
+					logger.err err:err, adminUserId:adminUserId, newEmail:newEmail, "error checking if limit reached for group plan"
 					return callback(err)
 				if limitReached
+					logger.err adminUserId:adminUserId, newEmail:newEmail, "group subscription limit reached not adding user to group"
 					return callback(limitReached:limitReached)
 				SubscriptionUpdater.addUserToGroup adminUserId, user._id, (err)->
 					if err?
@@ -74,8 +77,8 @@ module.exports = SubscriptionGroupHandler =
 			EmailHandler.sendEmail "completeJoinGroupAccount", opts, callback
 
 	processGroupVerification: (userEmail, subscription_id, token, callback)->
+		logger.log userEmail:userEmail, subscription_id:subscription_id, "processing group verification for user"
 		OneTimeTokenHandler.getValueFromTokenAndExpire token, (err, token_subscription_id)->
-			
 			if err?  or subscription_id != token_subscription_id
 				logger.err userEmail:userEmail, token:token, "token value not found for processing group verification"
 				return callback("token_not_found")
@@ -84,7 +87,7 @@ module.exports = SubscriptionGroupHandler =
 					logger.err err:err, subscription:subscription, userEmail:userEmail, subscription_id:subscription_id, "error getting subscription"
 					return callback(err)
 				if !subscription?
-					logger.warn subscription_id:subscription_id, "no subscription found"
+					logger.warn subscription_id:subscription_id, userEmail:userEmail, "no subscription found"
 					return callback()
 				SubscriptionGroupHandler.addUserToGroup subscription?.admin_id, userEmail, callback
 

services/web/app/coffee/Features/Subscription/UserFeaturesUpdater.coffee

@@ -1,4 +1,3 @@
-Settings = require "settings-sharelatex"
 logger = require("logger-sharelatex")
 User = require('../../models/User').User
 PlansLocator = require("./PlansLocator")

services/web/app/coffee/Features/ThirdPartyDataStore/TpdsUpdateSender.coffee

@@ -43,6 +43,9 @@ module.exports = TpdsUpdateSender =
 
 	_addEntity: (options, callback = (err)->)->
 		getProjectsUsersIds options.project_id, (err, user_id, allUserIds)->
+			if err?
+				logger.err err:err, options:options, "error getting projects user ids"
+				return callback(err)
 			logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, rev:options.rev, "sending file to third party data store"
 			postOptions =
 				method : "post"
@@ -54,6 +57,9 @@ module.exports = TpdsUpdateSender =
 				title: "addFile"
 				streamOrigin : options.streamOrigin
 			TpdsUpdateSender._enqueue options.project_id, "pipeStreamFrom", postOptions, (err)->
+				if err?
+					logger.err err:err,  project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, rev:options.rev, "error sending file to third party data store queued up for processing"
+					return callback(err)
 				logger.log project_id: options.project_id, user_id:user_id, path: options.path, uri:options.uri, rev:options.rev, "sending file to third party data store queued up for processing"
 				callback(err)
 	

services/web/app/coffee/Features/Uploads/ArchiveManager.coffee

@@ -3,21 +3,21 @@ logger  = require "logger-sharelatex"
 metrics = require "../../infrastructure/Metrics"
 fs      = require "fs"
 Path    = require "path"
+_ = require("underscore")
+
+ONE_MEG = 1024 * 1024
 
 module.exports = ArchiveManager =
-	extractZipArchive: (source, destination, _callback = (err) ->) ->
-		callback = (args...) ->
-			_callback(args...)
-			_callback = () ->
 
-		timer = new metrics.Timer("unzipDirectory")
-		logger.log source: source, destination: destination, "unzipping file"
 
-		unzip = child.spawn("unzip", [source, "-d", destination])
+	_isZipTooLarge: (source, callback = (err, isTooLarge)->)->
+		callback = _.once callback
+
+		unzip = child.spawn("unzip", ["-l", source])
 
-		# don't remove this line, some zips need
-		# us to listen on this for some unknow reason
+		output = ""
 		unzip.stdout.on "data", (d)->
+			output += d
 
 		error = null
 		unzip.stderr.on "data", (chunk) ->
@@ -31,11 +31,68 @@ module.exports = ArchiveManager =
 			callback(err)
 
 		unzip.on "exit", () ->
-			timer.done()
 			if error?
 				error = new Error(error)
-				logger.error err:error, source: source, destination: destination, "error unzipping file"
-			callback(error)
+				logger.error err:error, source: source, destination: destination, "error checking zip size"
+
+			lines = output.split("\n")
+			lastLine = lines[lines.length - 2]?.trim()
+			totalSizeInBytes = lastLine?.split(" ")?[0]
+
+			totalSizeInBytes = parseInt(totalSizeInBytes)
+
+			if !totalSizeInBytes? or isNaN(totalSizeInBytes)
+				logger.err source:source, "error getting bytes of zip"
+				return callback(new Error("something went wrong"))
+
+			isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
+
+			callback(error, isTooLarge)
+
+
+
+
+					
+	extractZipArchive: (source, destination, _callback = (err) ->) ->
+		callback = (args...) ->
+			_callback(args...)
+			_callback = () ->
+
+		ArchiveManager._isZipTooLarge source, (err, isTooLarge)->
+			if err?
+				logger.err err:err, "error checking size of zip file"
+				return callback(err)
+
+			if isTooLarge
+				return callback(new Error("zip_too_large"))		
+
+
+			timer = new metrics.Timer("unzipDirectory")
+			logger.log source: source, destination: destination, "unzipping file"
+
+			unzip = child.spawn("unzip", [source, "-d", destination])
+
+			# don't remove this line, some zips need
+			# us to listen on this for some unknow reason
+			unzip.stdout.on "data", (d)->
+
+			error = null
+			unzip.stderr.on "data", (chunk) ->
+				error ||= ""
+				error += chunk
+
+			unzip.on "error", (err) ->
+				logger.error {err, source, destination}, "unzip failed"
+				if err.code == "ENOENT"
+					logger.error "unzip command not found. Please check the unzip command is installed"
+				callback(err)
+
+			unzip.on "exit", () ->
+				timer.done()
+				if error?
+					error = new Error(error)
+					logger.error err:error, source: source, destination: destination, "error unzipping file"
+				callback(error)
 	
 	findTopLevelDirectory: (directory, callback = (error, topLevelDir) ->) ->
 		fs.readdir directory, (error, files) ->

services/web/app/coffee/Features/Uploads/FileSystemImportManager.coffee

@@ -4,76 +4,108 @@ _     = require "underscore"
 FileTypeManager  = require "./FileTypeManager"
 EditorController = require "../Editor/EditorController"
 ProjectLocator   = require "../Project/ProjectLocator"
+logger = require("logger-sharelatex")
 
 module.exports = FileSystemImportManager =
 	addDoc: (user_id, project_id, folder_id, name, path, replace, callback = (error, doc)-> )->
-		fs.readFile path, "utf8", (error, content = "") ->
-			return callback(error) if error?
-			content = content.replace(/\r/g, "")
-			lines = content.split("\n")
-			if replace
+		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
+			if !isSafe
+				logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, name:name, path:path, "add doc is from symlink, stopping process"
+				return callback("path is symlink")
+			fs.readFile path, "utf8", (error, content = "") ->
+				return callback(error) if error?
+				content = content.replace(/\r/g, "")
+				lines = content.split("\n")
+				if replace
+					ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
+						return callback(error) if error?
+						return callback(new Error("Couldn't find folder")) if !folder?
+						existingDoc = null
+						for doc in folder.docs
+							if doc.name == name
+								existingDoc = doc
+								break
+						if existingDoc?
+							EditorController.setDoc project_id, existingDoc._id, user_id, lines, "upload", callback
+						else
+							EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
+				else
+					EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
+
+	addFile: (user_id, project_id, folder_id, name, path, replace, callback = (error, file)-> )->
+		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
+			if !isSafe
+				logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, name:name, path:path, "add file is from symlink, stopping insert"
+				return callback("path is symlink")
+
+			if !replace
+				EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
+			else
 				ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
 					return callback(error) if error?
 					return callback(new Error("Couldn't find folder")) if !folder?
-					existingDoc = null
-					for doc in folder.docs
-						if doc.name == name
-							existingDoc = doc
+					existingFile = null
+					for fileRef in folder.fileRefs
+						if fileRef.name == name
+							existingFile = fileRef
 							break
-					if existingDoc?
-						EditorController.setDoc project_id, existingDoc._id, user_id, lines, "upload", callback
+					if existingFile?
+						EditorController.replaceFile project_id, existingFile._id, path, "upload", callback
 					else
-						EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
-			else
-				EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
-
-	addFile: (user_id, project_id, folder_id, name, path, replace, callback = (error, file)-> )->
-		if replace
-			ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
-				return callback(error) if error?
-				return callback(new Error("Couldn't find folder")) if !folder?
-				existingFile = null
-				for fileRef in folder.fileRefs
-					if fileRef.name == name
-						existingFile = fileRef
-						break
-				if existingFile?
-					EditorController.replaceFile project_id, existingFile._id, path, "upload", callback
-				else
-					EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
-		else
-			EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
+						EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
 
 	addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
-		EditorController.addFolderWithoutLock project_id, folder_id, name, "upload", (error, new_folder) =>
-			return callback(error) if error?
-			@addFolderContents user_id, project_id, new_folder._id, path, replace, (error) ->
+		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
+			if !isSafe
+				logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, path:path, "add folder is from symlink, stopping insert"
+				return callback("path is symlink")
+			EditorController.addFolderWithoutLock project_id, folder_id, name, "upload", (error, new_folder) =>
 				return callback(error) if error?
-				callback null, new_folder
+				FileSystemImportManager.addFolderContents user_id, project_id, new_folder._id, path, replace, (error) ->
+					return callback(error) if error?
+					callback null, new_folder
 
 	addFolderContents: (user_id, project_id, parent_folder_id, folderPath, replace, callback = (error)-> ) ->
-		fs.readdir folderPath, (error, entries = []) =>
-			return callback(error) if error?
-			jobs = _.map entries, (entry) =>
-				(callback) =>
-					FileTypeManager.shouldIgnore entry, (error, ignore) =>
+		FileSystemImportManager._isSafeOnFileSystem folderPath, (err, isSafe)->
+			if !isSafe
+				logger.log user_id:user_id, project_id:project_id, parent_folder_id:parent_folder_id, folderPath:folderPath, "add folder contents is from symlink, stopping insert"
+				return callback("path is symlink")
+			fs.readdir folderPath, (error, entries = []) =>
+				return callback(error) if error?
+				jobs = _.map entries, (entry) =>
+					(callback) =>
+						FileTypeManager.shouldIgnore entry, (error, ignore) =>
+							return callback(error) if error?
+							if !ignore
+								FileSystemImportManager.addEntity user_id, project_id, parent_folder_id, entry, "#{folderPath}/#{entry}", replace, callback
+							else
+								callback()
+				async.parallelLimit jobs, 5, callback
+
+	addEntity: (user_id, project_id, folder_id, name, path, replace, callback = (error, entity)-> ) ->
+		FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
+			if !isSafe
+				logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, path:path, "add entry is from symlink, stopping insert"
+				return callback("path is symlink")
+
+			FileTypeManager.isDirectory path, (error, isDirectory) =>
+				return callback(error) if error?
+				if isDirectory
+					FileSystemImportManager.addFolder user_id, project_id, folder_id, name, path, replace, callback
+				else
+					FileTypeManager.isBinary name, path, (error, isBinary) =>
 						return callback(error) if error?
-						if !ignore
-							@addEntity user_id, project_id, parent_folder_id, entry, "#{folderPath}/#{entry}", replace, callback
+						if isBinary
+							FileSystemImportManager.addFile user_id, project_id, folder_id, name, path, replace, callback
 						else
-							callback()
-			async.parallelLimit jobs, 5, callback
+							FileSystemImportManager.addDoc user_id, project_id, folder_id, name, path, replace, callback
 
-	addEntity: (user_id, project_id, folder_id, name, path, replace, callback = (error, entity)-> ) ->
-		FileTypeManager.isDirectory path, (error, isDirectory) =>
-			return callback(error) if error?
-			if isDirectory
-				@addFolder user_id, project_id, folder_id, name, path, replace, callback
-			else
-				FileTypeManager.isBinary name, path, (error, isBinary) =>
-					return callback(error) if error?
-					if isBinary
-						@addFile user_id, project_id, folder_id, name, path, replace, callback
-					else
-						@addDoc user_id, project_id, folder_id, name, path, replace, callback
+
+	_isSafeOnFileSystem: (path, callback = (err, isSafe)->)->
+		fs.lstat path, (err, stat)->
+			if err?
+				logger.err err:err, "error with path symlink check"
+				return callback(err)
+			isSafe = stat.isFile() or stat.isDirectory()
+			callback(err, isSafe)
 

services/web/app/coffee/Features/Uploads/ProjectUploadController.coffee

@@ -34,6 +34,7 @@ module.exports = ProjectUploadController =
 		if !name? or name.length == 0 or name.length > 150
 			logger.err project_id:project_id, name:name, "bad name when trying to upload file"
 			return res.send success: false
+		logger.log folder_id:folder_id, project_id:project_id, "getting upload file request"
 		user_id = req.session.user._id
 		FileSystemImportManager.addEntity user_id, project_id, folder_id, name, path, true, (error, entity) ->
 			fs.unlink path, ->

services/web/app/coffee/Features/User/UserController.coffee

@@ -1,3 +1,4 @@
+UserHandler = require("./UserHandler")
 UserDeleter = require("./UserDeleter")
 UserLocator = require("./UserLocator")
 User = require("../../models/User").User
@@ -10,7 +11,7 @@ AuthenticationManager = require("../Authentication/AuthenticationManager")
 UserUpdater = require("./UserUpdater")
 settings = require "settings-sharelatex"
 
-module.exports =
+module.exports = UserController =
 
 	deleteUser: (req, res)->
 		user_id = req.session.user._id
@@ -67,7 +68,14 @@ module.exports =
 							else
 								message = req.i18n.translate("problem_changing_email_address")
 							return res.send 500, {message:message}
-						res.sendStatus(200)
+						User.findById user_id, (err, user)->
+							if err?
+								logger.err err:err, user_id:user_id, "error getting user for email update"
+								return res.send 500
+							UserHandler.populateGroupLicenceInvite user, (err)-> #need to refresh this in the background
+								if err?
+									logger.err err:err, "error populateGroupLicenceInvite"
+								res.sendStatus(200)
 
 	logout : (req, res)->
 		metrics.inc "user.logout"

services/web/app/coffee/Features/User/UserHandler.coffee

@@ -6,7 +6,8 @@ logger = require("logger-sharelatex")
 
 module.exports = UserHandler =
 
-	_populateGroupLicenceInvite: (user, callback)->
+	populateGroupLicenceInvite: (user, callback)->
+		logger.log user_id:user._id, "populating any potential group licence invites"
 		licence = SubscriptionDomainHandler.getLicenceUserCanJoin user
 		if !licence?
 			return callback()
@@ -21,5 +22,5 @@ module.exports = UserHandler =
 				NotificationsBuilder.groupPlan(user, licence).create(callback)
 
 	setupLoginData: (user, callback = ->)->
-		@_populateGroupLicenceInvite user, callback
+		@populateGroupLicenceInvite user, callback
 

services/web/app/coffee/Features/Wiki/WikiController.coffee

@@ -4,7 +4,7 @@ logger = require("logger-sharelatex")
 ErrorController = require "../Errors/ErrorController"
 _ = require("underscore")
 AuthenticationController = require("../Authentication/AuthenticationController")
-
+async = require("async")
 other_lngs = ["es"]
 
 module.exports = WikiController = 
@@ -28,19 +28,22 @@ module.exports = WikiController =
 				lngPage = "#{page}_#{req.lng}"
 			else
 				lngPage = page
-
-			WikiController._getPageContent "Contents", (error, contents) ->
+			jobs =
+				contents: (cb)-> 
+					WikiController._getPageContent "Contents", cb
+				pageData: (cb)->
+					WikiController._getPageContent lngPage, cb
+			async.parallel jobs, (error, results)->
 				return next(error) if error?
-				WikiController._getPageContent lngPage, (error, pageData) ->
-					return next(error) if error?
-					if pageData.content?.length > 280
-						if _.include(other_lngs, req.lng)
-							pageData.title = pageData.title.slice(0, pageData.title.length - (req.lng.length+1) )
+				{pageData, contents} = results
+				if pageData.content?.length > 280
+					if _.include(other_lngs, req.lng)
+						pageData.title = pageData.title.slice(0, pageData.title.length - (req.lng.length+1) )
+					WikiController._renderPage(pageData, contents, res)
+				else
+					WikiController._getPageContent page, (error, pageData) ->
+						return next(error) if error?
 						WikiController._renderPage(pageData, contents, res)
-					else
-						WikiController._getPageContent page, (error, pageData) ->
-							return next(error) if error?
-							WikiController._renderPage(pageData, contents, res)
 
 
 					
@@ -62,7 +65,6 @@ module.exports = WikiController =
 			result = 
 				content: data?.parse?.text?['*']
 				title: data?.parse?.title
-
 			callback null, result
 
 

services/web/app/coffee/models/Project.coffee

@@ -43,54 +43,6 @@ ProjectSchema.statics.getProject = (project_or_id, fields, callback)->
 			return callback(new Errors.NotFoundError(e.message))
 		this.findById project_or_id, fields, callback
 
-sanitizeTypeOfElement = (elementType)->
-	lastChar = elementType.slice -1
-	if lastChar != "s"
-		elementType +="s"
-	if elementType == "files"
-		elementType = "fileRefs"
-	return elementType
-
-ProjectSchema.statics.putElement = (project_id, folder_id, element, type, callback)->
-	if !element?
-		e = new Error("no element passed to be inserted")
-		logger.err project_id:project_id, folder_id:folder_id, element:element, type:type, "failed trying to insert element as it was null"
-		return callback(e)
-	type = sanitizeTypeOfElement type
-	this.findById project_id, (err, project)=>
-		if err?
-			callback(err)
-		if !folder_id?
-			folder_id = project.rootFolder[0]._id
-		require('../Features/Project/ProjectLocator').findElement {project:project, element_id:folder_id, type:"folders"}, (err, folder, path)=>
-			newPath =
-				fileSystem: "#{path.fileSystem}/#{element.name}"
-				mongo: path.mongo # TODO: This is not correct
-			if err?
-				callback(err)
-			logger.log project_id: project_id, element_id: element._id, fileType: type, folder_id: folder_id, "adding element to project"
-			id = element._id+''
-			element._id = concreteObjectId(id)
-			conditions = _id:project_id
-			mongopath = "#{path.mongo}.#{type}"
-			update = "$push":{}
-			update["$push"][mongopath] = element
-			this.update conditions, update, {}, (err)->
-				if(err)
-					logger.err err: err, project: project, 'error saving in putElement project'
-				if callback?
-					callback(err, {path:newPath})
-
-getIndexOf = (searchEntity, id)->
-	length = searchEntity.length
-	count = 0
-	while(count < length)
-		if searchEntity[count]._id+"" == id+""
-			return count
-		count++
-
-
-
 applyToAllFilesRecursivly = ProjectSchema.statics.applyToAllFilesRecursivly = (folder, fun)->
 	_.each folder.fileRefs, (file)->
 		fun(file)

services/web/app/coffee/models/ProjectInvite.coffee

@@ -0,0 +1,23 @@
+mongoose = require 'mongoose'
+Settings = require 'settings-sharelatex'
+
+Schema = mongoose.Schema
+ObjectId = Schema.ObjectId
+
+ProjectInviteSchema = new Schema
+	project_id:     ObjectId
+	from_user_id:   ObjectId
+	privilegeLevel: String
+	# For existing users
+	to_user_id:     ObjectId
+	# For non-existant users
+	hashed_token:   String
+	email:          String
+
+conn = mongoose.createConnection(Settings.mongo.url, server: poolSize: Settings.mongo.poolSize || 10)
+
+ProjectInvite = conn.model('ProjectInvite', ProjectInviteSchema)
+
+mongoose.model 'ProjectInvite', ProjectInviteSchema
+exports.ProjectInvite = ProjectInvite
+exports.ProjectInviteSchema = ProjectInviteSchema

services/web/app/views/project/editor/editor.jade

@@ -20,6 +20,7 @@ div.full-size(
 			keybindings="settings.mode",
 			font-size="settings.fontSize",
 			auto-complete="settings.autoComplete",
+			spell-check="true",
 			spell-check-language="project.spellCheckLanguage",
 			highlights="onlineUserCursorHighlights[editor.open_doc_id]"
 			show-print-margin="false",

services/web/app/views/project/editor/file-tree.jade

@@ -309,6 +309,7 @@ script(type='text/ng-template', id='newDocModalTemplate')
 		h3 #{translate("new_file")}
 	.modal-body
 		form(novalidate, name="newDocForm")
+			div.alert.alert-danger(ng-if="error") {{error}}
 			input.form-control(
 				type="text",
 				placeholder="File Name",
@@ -333,6 +334,7 @@ script(type='text/ng-template', id='newFolderModalTemplate')
 	.modal-header
 		h3 #{translate("new_folder")}
 	.modal-body
+		div.alert.alert-danger(ng-if="error") {{error}}
 		form(novalidate, name="newFolderForm")
 			input.form-control(
 				type="text",

services/web/app/views/project/editor/pdf.jade

@@ -112,8 +112,10 @@ div.full-size.pdf(ng-controller="PdfController")
 					| #{translate("learn_how_to_make_documents_compile_quickly")}
 
 		.alert.alert-success(ng-show="pdf.timedout && !hasPremiumCompile")
-			p
+			p(ng-if="project.owner._id == user.id")
 				strong #{translate("upgrade_for_faster_compiles")}
+			p(ng-if="project.owner._id != user.id")
+				strong #{translate("ask_proj_owner_to_upgrade_for_faster_compiles")}
 			p #{translate("free_accounts_have_timeout_upgrade_to_increase")}
 			p Plus:
 			p
@@ -142,7 +144,7 @@ div.full-size.pdf(ng-controller="PdfController")
 						i.fa.fa-check  
 						|#{translate("compile_larger_projects")}
 
-			p(ng-controller="FreeTrialModalController")
+			p(ng-controller="FreeTrialModalController", ng-if="project.owner._id == user.id")
 				a.btn.btn-success.row-spaced-small(
 					href
 					ng-class="buttonClass"

services/web/app/views/project/editor/share.jade

@@ -125,7 +125,7 @@ script(type='text/ng-template', id='shareProjectModalTemplate')
 			span.text-danger.error(ng-show="state.error") #{translate("generic_something_went_wrong")}
 		button.btn.btn-primary(
 			ng-click="done()"
-		) #{translate("done")}
+		) #{translate("close")}
 
 script(type="text/ng-template", id="makePublicModalTemplate")
 	.modal-header

services/web/config/settings.defaults.coffee

@@ -129,6 +129,9 @@ module.exports =
 	# Same, but with http auth credentials.
 	httpAuthSiteUrl: 'http://#{httpAuthUser}:#{httpAuthPass}@localhost:3000'
 
+
+	maxEntitiesPerProject: 2000
+	
 	# Security
 	# --------
 	security:

services/web/public/coffee/ide/editor/directives/aceEditor.coffee

@@ -18,7 +18,7 @@ define [
 			url = ace.config._moduleUrl(args...) + "?fingerprint=#{window.aceFingerprint}"
 			return url
 
-	App.directive "aceEditor", ($timeout, $compile, $rootScope, event_tracking, localStorage) ->
+	App.directive "aceEditor", ($timeout, $compile, $rootScope, event_tracking, localStorage, $cacheFactory) ->
 		monkeyPatchSearch($rootScope, $compile)
 
 		return  {
@@ -29,6 +29,7 @@ define [
 				fontSize: "="
 				autoComplete: "="
 				sharejsDoc: "="
+				spellCheck: "="
 				spellCheckLanguage: "="
 				highlights: "="
 				text: "="
@@ -55,7 +56,9 @@ define [
 				scope.name = attrs.aceEditor
 
 				autoCompleteManager   = new AutoCompleteManager(scope, editor, element)
-				spellCheckManager     = new SpellCheckManager(scope, editor, element)
+				if scope.spellCheck # only enable spellcheck when explicitly required
+					spellCheckCache =  $cacheFactory("spellCheck-#{scope.name}", {capacity: 1000})
+					spellCheckManager = new SpellCheckManager(scope, editor, element, spellCheckCache)
 				undoManager           = new UndoManager(scope, editor, element)
 				highlightsManager     = new HighlightsManager(scope, editor, element)
 				cursorPositionManager = new CursorPositionManager(scope, editor, element, localStorage)

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee

@@ -45,12 +45,15 @@ define [
 			references = @$scope.$root._references
 			ReferencesCompleter =
 				getCompletions: (editor, session, pos, prefix, callback) ->
-					range = new Range(pos.row, 0, pos.row, pos.column)
-					lineUpToCursor = editor.getSession().getTextRange(range)
+					upToCursorRange = new Range(pos.row, 0, pos.row, pos.column)
+					lineUpToCursor = editor.getSession().getTextRange(upToCursorRange)
 					commandFragment = getLastCommandFragment(lineUpToCursor)
 					if commandFragment
-						citeMatch = commandFragment.match(/^~?\\([a-z]*cite[a-z]?){(.*,)?(\w*)/)
+						citeMatch = commandFragment.match(/^~?\\([a-z]*cite[a-z]?){([^}]*,)?(\w*)/)
 						if citeMatch
+							beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999)
+							lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange)
+							needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/)
 							commandName = citeMatch[1]
 							previousArgs = citeMatch[2]
 							currentArg = citeMatch[3]
@@ -59,8 +62,8 @@ define [
 							previousArgsCaption = if previousArgs.length > 8 then "…," else previousArgs
 							result = []
 							result.push {
-								caption: "\\#{commandName}{",
-								snippet: "\\#{commandName}{",
+								caption: "\\#{commandName}{}",
+								snippet: "\\#{commandName}{}",
 								meta: "reference",
 								score: 11000
 							}
@@ -68,8 +71,8 @@ define [
 								references.keys.forEach (key) ->
 									if !(key in [null, undefined])
 										result.push({
-											caption: "\\#{commandName}{#{previousArgsCaption}#{key}",
-											value: "\\#{commandName}{#{previousArgs}#{key}",
+											caption: "\\#{commandName}{#{previousArgsCaption}#{key}#{if needsClosingBrace then '}' else ''}",
+											value: "\\#{commandName}{#{previousArgs}#{key}#{if needsClosingBrace then '}' else ''}",
 											meta: "reference",
 											score: 10000
 										})

services/web/public/coffee/ide/editor/directives/aceEditor/spell-check/SpellCheckManager.coffee

@@ -5,7 +5,7 @@ define [
 	Range = ace.require("ace/range").Range
 
 	class SpellCheckManager
-		constructor: (@$scope, @editor, @element) ->
+		constructor: (@$scope, @editor, @element, @cache) ->
 			$(document.body).append @element.find(".spell-check-menu")
 			
 			@updatedLines = []
@@ -102,6 +102,8 @@ define [
 		learnWord: (highlight) ->
 			@apiRequest "/learn", word: highlight.word
 			@highlightedWordManager.removeWord highlight.word
+			language = @$scope.spellCheckLanguage
+			@cache?.put("#{language}:#{highlight.word}", true)
 
 		getHighlightedWordAtCursor: () ->
 			cursor = @editor.getCursorPosition()
@@ -143,24 +145,67 @@ define [
 		runSpellCheck: (linesToProcess) ->
 			{words, positions} = @getWords(linesToProcess)
 			language = @$scope.spellCheckLanguage
-			@apiRequest "/check", {language: language, words: words}, (error, result) =>
-				if error? or !result? or !result.misspellings?
-					return null
 
+			highlights = []
+			seen = {}
+			newWords = []
+			newPositions = []
+
+			# iterate through all words, building up a list of
+			# newWords/newPositions not in the cache
+			for word, i in words
+				key = "#{language}:#{word}"
+				seen[key] ?= @cache.get(key) # avoid hitting the cache unnecessarily
+				cached = seen[key]
+				if not cached?
+					newWords.push words[i]
+					newPositions.push positions[i]
+				else if cached is true
+					# word is correct
+				else
+					highlights.push
+						column: positions[i].column
+						row: positions[i].row
+						word: word
+						suggestions: cached
+			words = newWords
+			positions = newPositions
+
+			displayResult = (highlights) =>
 				if linesToProcess?
 					for shouldProcess, row in linesToProcess
 						@highlightedWordManager.clearRows(row, row) if shouldProcess
 				else
 					@highlightedWordManager.clearRows()
-
-				for misspelling in result.misspellings
-					word = words[misspelling.index]
-					position = positions[misspelling.index]
-					@highlightedWordManager.addHighlight
-						column: position.column
-						row: position.row
-						word: word
-						suggestions: misspelling.suggestions
+				for highlight in highlights
+					@highlightedWordManager.addHighlight highlight
+
+			if not words.length
+				displayResult highlights
+			else
+				@apiRequest "/check", {language: language, words: words}, (error, result) =>
+					if error? or !result? or !result.misspellings?
+						return null
+					mispelled = []
+					for misspelling in result.misspellings
+						word = words[misspelling.index]
+						position = positions[misspelling.index]
+						mispelled[misspelling.index] = true
+						highlights.push
+							column: position.column
+							row: position.row
+							word: word
+							suggestions: misspelling.suggestions
+						key = "#{language}:#{word}"
+						if not seen[key]
+							@cache.put key, misspelling.suggestions
+							seen[key] = true
+					for word, i in words when not mispelled[i]
+						key = "#{language}:#{word}"
+						if not seen[key]
+							@cache.put(key, true)
+							seen[key] = true
+					displayResult highlights
 
 		getWords: (linesToProcess) ->
 			lines = @editor.getValue().split("\n")

services/web/public/coffee/ide/file-tree/controllers/FileTreeController.coffee

@@ -61,6 +61,8 @@ define [
 				$scope.state.inflight = true
 				ide.fileTreeManager
 					.createDoc(name, parent_folder)
+					.error (e)->
+						$scope.error = e
 					.success () ->
 						$scope.state.inflight = false
 						$modalInstance.close()
@@ -90,6 +92,8 @@ define [
 				$scope.state.inflight = true
 				ide.fileTreeManager
 					.createFolder(name, parent_folder)
+					.error (e)->
+						$scope.error = e
 					.success () ->
 						$scope.state.inflight = false
 						$modalInstance.close()

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -1,7 +1,8 @@
 define [
 	"base"
 	"libs/latex-log-parser"
-], (App, LogParser) ->
+	"libs/bib-log-parser"
+], (App, LogParser, BibLogParser) ->
 	App.controller "PdfController", ($scope, $http, ide, $modal, synctex, event_tracking, localStorage) ->
 		autoCompile = true
 		$scope.$on "project:joined", () ->
@@ -12,7 +13,7 @@ define [
 
 		$scope.$on "pdf:error:display", () ->
 			$scope.pdf.error = true
-		
+
 		$scope.draft = localStorage("draft:#{$scope.project_id}") or false
 		$scope.$watch "draft", (new_value, old_value) ->
 			if new_value? and old_value != new_value
@@ -82,25 +83,42 @@ define [
 			qs = if outputFile?.build? then "?build=#{outputFile.build}" else ""
 			$http.get "/project/#{$scope.project_id}/output/output.log" + qs
 				.success (log) ->
+					#console.log ">>", log
 					$scope.pdf.rawLog = log
 					logEntries = LogParser.parse(log, ignoreDuplicates: true)
+					#console.log ">>", logEntries
 					$scope.pdf.logEntries = logEntries
 					$scope.pdf.logEntries.all = logEntries.errors.concat(logEntries.warnings).concat(logEntries.typesetting)
-
-					$scope.pdf.logEntryAnnotations = {}
-					for entry in logEntries.all
-						if entry.file?
-							entry.file = normalizeFilePath(entry.file)
-
-							entity = ide.fileTreeManager.findEntityByPath(entry.file)
-							if entity?
-								$scope.pdf.logEntryAnnotations[entity.id] ||= []
-								$scope.pdf.logEntryAnnotations[entity.id].push {
-									row: entry.line - 1
-									type: if entry.level == "error" then "error" else "warning"
-									text: entry.message
-								}
-
+					# # # #
+					proceed = () ->
+						$scope.pdf.logEntryAnnotations = {}
+						for entry in logEntries.all
+							if entry.file?
+								entry.file = normalizeFilePath(entry.file)
+								entity = ide.fileTreeManager.findEntityByPath(entry.file)
+								if entity?
+									$scope.pdf.logEntryAnnotations[entity.id] ||= []
+									$scope.pdf.logEntryAnnotations[entity.id].push {
+										row: entry.line - 1
+										type: if entry.level == "error" then "error" else "warning"
+										text: entry.message
+									}
+					# Get the biber log and parse it too
+					$http.get "/project/#{$scope.project_id}/output/output.blg" + qs
+						.success (log) ->
+							window._s = $scope
+							biberLogEntries = BibLogParser.parse(log, {})
+							if $scope.pdf.logEntries
+								entries = $scope.pdf.logEntries
+								all = biberLogEntries.errors.concat(biberLogEntries.warnings)
+								entries.all = entries.all.concat(all)
+								entries.errors = entries.errors.concat(biberLogEntries.errors)
+								entries.warnings = entries.warnings.concat(biberLogEntries.warnings)
+							proceed()
+						.error (e) ->
+							console.error ">> error", e
+							proceed()
+					# # # #
 				.error () ->
 					$scope.pdf.logEntries = []
 					$scope.pdf.rawLog = ""
@@ -127,7 +145,7 @@ define [
 		$scope.recompile = (options = {}) ->
 			return if $scope.pdf.compiling
 			$scope.pdf.compiling = true
-			
+
 			ide.$scope.$broadcast("flush-changes")
 
 			options.rootDocOverride_id = getRootDocOverride_id()
@@ -140,7 +158,7 @@ define [
 				.error () ->
 					$scope.pdf.compiling = false
 					$scope.pdf.error = true
-					
+
 		# This needs to be public.
 		ide.$scope.recompile = $scope.recompile
 
@@ -177,17 +195,17 @@ define [
 				.then (data) ->
 					{doc, line} = data
 					ide.editorManager.openDoc(doc, gotoLine: line)
-					
+
 		$scope.switchToFlatLayout = () ->
 			$scope.ui.pdfLayout = 'flat'
 			$scope.ui.view = 'pdf'
 			ide.localStorage "pdf.layout", "flat"
-			
+
 		$scope.switchToSideBySideLayout = () ->
 			$scope.ui.pdfLayout = 'sideBySide'
 			$scope.ui.view = 'editor'
 			localStorage "pdf.layout", "split"
-			
+
 		if pdfLayout = localStorage("pdf.layout")
 			$scope.switchToSideBySideLayout() if pdfLayout == "split"
 			$scope.switchToFlatLayout() if pdfLayout == "flat"
@@ -216,7 +234,7 @@ define [
 				if !path?
 					deferred.reject()
 					return deferred.promise
-				
+
 				# If the root file is folder/main.tex, then synctex sees the
 				# path as folder/./main.tex
 				rootDocDirname = ide.fileTreeManager.getRootDocDirname()
@@ -226,7 +244,7 @@ define [
 				{row, column} = cursorPosition
 
 				$http({
-						url: "/project/#{ide.project_id}/sync/code", 
+						url: "/project/#{ide.project_id}/sync/code",
 						method: "GET",
 						params: {
 							file: path
@@ -253,7 +271,7 @@ define [
 					position.offset.top = position.offset.top + 80
 
 				$http({
-						url: "/project/#{ide.project_id}/sync/pdf", 
+						url: "/project/#{ide.project_id}/sync/pdf",
 						method: "GET",
 						params: {
 							page: position.page + 1
@@ -316,4 +334,4 @@ define [
 
 		$scope.cancel = () ->
 			$modalInstance.dismiss('cancel')
-	]
+	]

services/web/public/coffee/ide/settings/controllers/ProjectNameController.coffee

@@ -17,6 +17,8 @@ define [
 			newName = $scope.inputs.name
 			if !newName? or newName.length == 0 or newName.length > MAX_PROJECT_NAME_LENGTH
 				return
+			if $scope.project.name == newName
+				return
 			$scope.project.name = newName
 			settings.saveProjectSettings({name: $scope.project.name})
 

services/web/public/coffee/main/universties-site.coffee

@@ -12,12 +12,13 @@ define [
 				console.log "email not set"
 				return
 			$scope.sending = true
+			ticketNumber = Math.floor((1 + Math.random()) * 0x10000).toString(32)
 			params = 
 				name: $scope.form.name || $scope.form.email
 				email: $scope.form.email
 				labels: $scope.form.source
 				message: "Please contact me with more details"
-				subject: $scope.form.subject
+				subject: $scope.form.subject + " - [#{ticketNumber}]"
 				about : "#{$scope.form.position || ''} #{$scope.form.university || ''}"
 
 			Groove.createTicket params, (err, json)->

services/web/public/js/libs/bib-log-parser.js

@@ -0,0 +1,190 @@
+// Generated by CoffeeScript 1.10.0
+define(function() {
+  var BAD_CROSS_REFERENCE_REGEX, BibLogParser, LINE_SPLITTER_REGEX, MESSAGE_LEVELS, MULTILINE_COMMAND_ERROR_REGEX, MULTILINE_ERROR_REGEX, MULTILINE_WARNING_REGEX, SINGLELINE_WARNING_REGEX, consume, errorParsers, warningParsers;
+  LINE_SPLITTER_REGEX = /^\[(\d+)].*>\s(INFO|WARN|ERROR)\s-\s(.*)$/;
+  MESSAGE_LEVELS = {
+    "INFO": "info",
+    "WARN": "warning",
+    "ERROR": "error"
+  };
+  BibLogParser = function(text, options) {
+    if (typeof text !== 'string') {
+      throw new Error("BibLogParser Error: text parameter must be a string");
+    }
+    this.text = text.replace(/(\r\n)|\r/g, '\n');
+    this.options = options || {};
+    this.lines = text.split('\n');
+  };
+  consume = function(logText, regex, process) {
+    var iterationCount, match, newEntry, re, result, text;
+    text = logText;
+    result = [];
+    re = regex;
+    iterationCount = 0;
+    while (match = re.exec(text)) {
+      iterationCount += 1;
+      if (iterationCount >= 10000) {
+        return result;
+      }
+      newEntry = process(match);
+      result.push(newEntry);
+      text = (match.input.slice(0, match.index)) + (match.input.slice(match.index + match[0].length + 1, match.input.length));
+    }
+    return [result, text];
+  };
+  MULTILINE_WARNING_REGEX = /^Warning--(.+)\n--line (\d+) of file (.+)$/m;
+  SINGLELINE_WARNING_REGEX = /^Warning--(.+)$/m;
+  MULTILINE_ERROR_REGEX = /^(.*)---line (\d+) of file (.*)\n([^]+?)\nI'm skipping whatever remains of this entry$/m;
+  BAD_CROSS_REFERENCE_REGEX = /^(A bad cross reference---entry ".+?"\nrefers to entry.+?, which doesn't exist)$/m;
+  MULTILINE_COMMAND_ERROR_REGEX = /^(.*)\n---line (\d+) of file (.*)\n([^]+?)\nI'm skipping whatever remains of this command$/m;
+  warningParsers = [
+    [
+      MULTILINE_WARNING_REGEX, function(match) {
+        var fileName, fullMatch, lineNumber, message;
+        fullMatch = match[0], message = match[1], lineNumber = match[2], fileName = match[3];
+        return {
+          file: fileName,
+          level: "warning",
+          message: message,
+          line: lineNumber,
+          raw: fullMatch
+        };
+      }
+    ], [
+      SINGLELINE_WARNING_REGEX, function(match) {
+        var fullMatch, message;
+        fullMatch = match[0], message = match[1];
+        return {
+          file: '',
+          level: "warning",
+          message: message,
+          line: '',
+          raw: fullMatch
+        };
+      }
+    ]
+  ];
+  errorParsers = [
+    [
+      MULTILINE_ERROR_REGEX, function(match) {
+        var fileName, firstMessage, fullMatch, lineNumber, secondMessage;
+        fullMatch = match[0], firstMessage = match[1], lineNumber = match[2], fileName = match[3], secondMessage = match[4];
+        return {
+          file: fileName,
+          level: "error",
+          message: firstMessage + '\n' + secondMessage,
+          line: lineNumber,
+          raw: fullMatch
+        };
+      }
+    ], [
+      BAD_CROSS_REFERENCE_REGEX, function(match) {
+        var fullMatch, message;
+        fullMatch = match[0], message = match[1];
+        return {
+          file: '',
+          level: "error",
+          message: message,
+          line: '',
+          raw: fullMatch
+        };
+      }
+    ], [
+      MULTILINE_COMMAND_ERROR_REGEX, function(match) {
+        var fileName, firstMessage, fullMatch, lineNumber, secondMessage;
+        fullMatch = match[0], firstMessage = match[1], lineNumber = match[2], fileName = match[3], secondMessage = match[4];
+        return {
+          file: fileName,
+          level: "error",
+          message: firstMessage + '\n' + secondMessage,
+          line: lineNumber,
+          raw: fullMatch
+        };
+      }
+    ]
+  ];
+  (function() {
+    this.parseBibtex = function() {
+      var allErrors, allWarnings, ref, ref1, remainingText, result;
+      result = {
+        all: [],
+        errors: [],
+        warnings: [],
+        files: [],
+        typesetting: []
+      };
+      ref = warningParsers.reduce(function(accumulator, parser) {
+        var _remainingText, currentWarnings, process, ref, regex, text, warnings;
+        currentWarnings = accumulator[0], text = accumulator[1];
+        regex = parser[0], process = parser[1];
+        ref = consume(text, regex, process), warnings = ref[0], _remainingText = ref[1];
+        return [currentWarnings.concat(warnings), _remainingText];
+      }, [[], this.text]), allWarnings = ref[0], remainingText = ref[1];
+      ref1 = errorParsers.reduce(function(accumulator, parser) {
+        var _remainingText, currentErrors, errors, process, ref1, regex, text;
+        currentErrors = accumulator[0], text = accumulator[1];
+        regex = parser[0], process = parser[1];
+        ref1 = consume(text, regex, process), errors = ref1[0], _remainingText = ref1[1];
+        return [currentErrors.concat(errors), _remainingText];
+      }, [[], remainingText]), allErrors = ref1[0], remainingText = ref1[1];
+      result.warnings = allWarnings;
+      result.errors = allErrors;
+      result.all = allWarnings.concat(allErrors);
+      return result;
+    };
+    this.parseBiber = function() {
+      var result;
+      result = {
+        all: [],
+        errors: [],
+        warnings: [],
+        files: [],
+        typesetting: []
+      };
+      this.lines.forEach(function(line) {
+        var _, fileName, fullLine, lineMatch, lineNumber, match, message, messageType, newEntry, realMessage;
+        match = line.match(LINE_SPLITTER_REGEX);
+        if (match) {
+          fullLine = match[0], lineNumber = match[1], messageType = match[2], message = match[3];
+          newEntry = {
+            file: '',
+            level: MESSAGE_LEVELS[messageType] || "INFO",
+            message: message,
+            line: '',
+            raw: fullLine
+          };
+          lineMatch = newEntry.message.match(/^BibTeX subsystem: \/.+\/(\w+\.\w+)_.+, line (\d+), (.+)$/);
+          if (lineMatch && lineMatch.length === 4) {
+            _ = lineMatch[0], fileName = lineMatch[1], lineNumber = lineMatch[2], realMessage = lineMatch[3];
+            newEntry.file = fileName;
+            newEntry.line = lineNumber;
+            newEntry.message = realMessage;
+          }
+          result.all.push(newEntry);
+          switch (newEntry.level) {
+            case 'error':
+              return result.errors.push(newEntry);
+            case 'warning':
+              return result.warnings.push(newEntry);
+          }
+        }
+      });
+      return result;
+    };
+    return this.parse = function() {
+      var firstLine;
+      firstLine = this.lines[0];
+      if (firstLine.match(/^.*INFO - This is Biber.*$/)) {
+        return this.parseBiber();
+      } else if (firstLine.match(/^This is BibTeX, Version.+$/)) {
+        return this.parseBibtex();
+      } else {
+        throw new Error("BibLogParser Error: cannot determine whether text is biber or bibtex output");
+      }
+    };
+  }).call(BibLogParser.prototype);
+  BibLogParser.parse = function(text, options) {
+    return new BibLogParser(text, options).parse();
+  };
+  return BibLogParser;
+});

services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee

@@ -8,7 +8,7 @@ path = require 'path'
 _ = require 'underscore'
 modulePath = path.join __dirname, '../../../../app/js/Features/DocumentUpdater/DocumentUpdaterHandler'
 
-describe 'Flushing documents :', ->
+describe 'DocumentUpdaterHandler - Flushing documents :', ->
 
 	beforeEach ->
 		@project_id = "project-id-923"
@@ -33,6 +33,9 @@ describe 'Flushing documents :', ->
 			"../../models/Project": Project: @Project={}
 			'../../Features/Project/ProjectLocator':{}
 			'redis-sharelatex' : createClient: () => @rclient
+			"../../infrastructure/Metrics": 
+				Timer:->
+					done:->
 
 	describe 'queueChange', ->
 		beforeEach ->

services/web/test/UnitTests/coffee/Editor/EditorControllerTests.coffee

@@ -518,12 +518,23 @@ describe "EditorController", ->
 			@folder_id = "313dasd21dasdsa"
 			@ProjectEntityHandler.moveEntity = sinon.stub().callsArgWith(4, @err)
 			@EditorRealTimeController.emitToRoom = sinon.stub()
+			@LockManager.releaseLock.callsArgWith(1)
+			@LockManager.getLock.callsArgWith(1)
 
 		it "should call the ProjectEntityHandler", (done)->
 			@EditorController.moveEntity @project_id, @entity_id, @folder_id, @entityType, =>
 				@ProjectEntityHandler.moveEntity.calledWith(@project_id, @entity_id, @folder_id, @entityType).should.equal true
 				done()
 
+		it "should take the lock", (done)->
+			@EditorController.moveEntity @project_id, @entity_id, @folder_id, @entityType, =>
+				@LockManager.getLock.calledWith(@project_id).should.equal true
+				done()
+
+		it "should release the lock", (done)->
+			@EditorController.moveEntity @project_id, @entity_id, @folder_id, @entityType, =>
+				@LockManager.releaseLock.calledWith(@project_id).should.equal true
+				done()
 
 		it "should emit the update to the room", (done)->
 			@EditorController.moveEntity @project_id, @entity_id, @folder_id, @entityType, =>

services/web/test/UnitTests/coffee/Editor/EditorRealTimeControllerTests.coffee

@@ -10,6 +10,7 @@ describe "EditorRealTimeController", ->
 				createClient: () ->
 					auth:->
 			"../../infrastructure/Server" : io: @io = {}
+			"settings-sharelatex":{redis:{}}
 		@EditorRealTimeController.rclientPub = publish: sinon.stub()
 		@EditorRealTimeController.rclientSub =
 			subscribe: sinon.stub()

services/web/test/UnitTests/coffee/Email/EmailBuilderTests.coffee

@@ -9,7 +9,7 @@ _ = require('underscore')
 _.templateSettings =
   interpolate: /\{\{(.+?)\}\}/g
 
-describe "Email Templator ", ->
+describe "EmailBuilder", ->
 
 	beforeEach ->
 

services/web/test/UnitTests/coffee/Email/EmailSenderTests.coffee

@@ -6,7 +6,7 @@ sinon = require('sinon')
 modulePath = path.join __dirname, "../../../../app/js/Features/Email/EmailSender.js"
 expect = require("chai").expect
 
-describe "Email", ->
+describe "EmailSender", ->
 
 	beforeEach ->
 
@@ -30,6 +30,9 @@ describe "Email", ->
 				log:->
 				warn:->
 				err:->
+			"../../infrastructure/Metrics": inc:->
+				
+
 
 		@opts =
 			to: "bob@bob.com"

services/web/test/UnitTests/coffee/FileStore/FileStoreHandlerTests.coffee

@@ -10,6 +10,10 @@ describe "FileStoreHandler", ->
 	beforeEach ->
 		@fs =
 			createReadStream : sinon.stub()
+			lstat: sinon.stub().callsArgWith(1, null, {
+				isFile:=> @isSafeOnFileSystem
+				isDirectory:-> return false
+			})
 		@writeStream =
 			my:"writeStream"
 			on: (type, cb)-> 
@@ -31,6 +35,7 @@ describe "FileStoreHandler", ->
 	describe "uploadFileFromDisk", ->
 		beforeEach ->
 			@request.returns(@writeStream)
+			@isSafeOnFileSystem = true
 
 		it "should create read stream", (done)->
 			@fs.createReadStream.returns 
@@ -74,6 +79,13 @@ describe "FileStoreHandler", ->
 				@handler._buildUrl.calledWith(@project_id, @file_id).should.equal true
 				done()
 
+		describe "symlink", ->
+			it "should not read file if it is symlink", (done)->
+				@isSafeOnFileSystem = false
+				@handler.uploadFileFromDisk @project_id, @file_id, @fsPath, =>
+					@fs.createReadStream.called.should.equal false
+					done()
+
 	describe "deleteFile", ->
 
 		it "should send a delete request to filestore api", (done)->

services/web/test/UnitTests/coffee/Project/ProjectControllerTests.coffee

@@ -55,6 +55,10 @@ describe "ProjectController", ->
 			"logger-sharelatex": 
 				log:->
 				err:->
+			"../../infrastructure/Metrics": 
+				Timer:->
+					done:->
+				inc:->
 			"./ProjectDeleter": @ProjectDeleter
 			"./ProjectDuplicator": @ProjectDuplicator
 			"./ProjectCreationHandler": @ProjectCreationHandler

services/web/test/UnitTests/coffee/Project/ProjectCreationHandlerTests.coffee

@@ -50,6 +50,9 @@ describe 'ProjectCreationHandler', ->
 			'./ProjectEntityHandler':@ProjectEntityHandler
 			"settings-sharelatex": @Settings = {}
 			'logger-sharelatex': {log:->}
+			"../../infrastructure/Metrics": inc:->
+				
+
 
 	describe 'Creating a Blank project', ->
 		beforeEach ->

services/web/test/UnitTests/coffee/Project/ProjectDeleterTests.coffee

@@ -25,6 +25,8 @@ describe 'ProjectDeleter', ->
 		@editorController = notifyUsersProjectHasBeenDeletedOrRenamed : sinon.stub().callsArgWith(1)
 		@TagsHandler = 
 			removeProjectFromAllTags: sinon.stub().callsArgWith(2)
+		@ProjectGetter =
+			getProject:sinon.stub()
 		@deleter = SandboxedModule.require modulePath, requires:
 			"../Editor/EditorController": @editorController
 			'../../models/Project':{Project:@Project}
@@ -32,6 +34,7 @@ describe 'ProjectDeleter', ->
 			"../Tags/TagsHandler":@TagsHandler
 			"../FileStore/FileStoreHandler": @FileStoreHandler = {}
 			"../Collaborators/CollaboratorsHandler": @CollaboratorsHandler = {}
+			"./ProjectGetter": @ProjectGetter
 			'logger-sharelatex':
 				log:->
 
@@ -92,6 +95,7 @@ describe 'ProjectDeleter', ->
 		beforeEach ->
 			@CollaboratorsHandler.getMemberIds = sinon.stub()
 			@CollaboratorsHandler.getMemberIds.withArgs(@project_id).yields(null, ["member-id-1", "member-id-2"])
+			@ProjectGetter.getProject.callsArgWith(2, null, @project)
 			@Project.update.callsArgWith(2)
 
 		it "should flushProjectToMongoAndDelete in doc updater", (done)->

services/web/test/UnitTests/coffee/Project/ProjectDetailsHandlerTests.coffee

@@ -5,7 +5,7 @@ sinon = require('sinon')
 assert = require("chai").assert
 require('chai').should()
 
-describe 'Project details handler', ->
+describe 'ProjectDetailsHandler', ->
 
 	beforeEach ->
 		@project_id = "321l3j1kjkjl"
@@ -50,7 +50,7 @@ describe 'Project details handler', ->
 
 		it "should return the error", (done)->
 			error = "some error"
-			@ProjectGetter.getProjectWithoutDocLines.callsArgWith(1, error)
+			@ProjectGetter.getProject.callsArgWith(2, error)
 			@handler.getDetails @project_id, (err)=>
 				err.should.equal error
 				done()

services/web/test/UnitTests/coffee/Project/ProjectDuplicatorTests.coffee

@@ -25,7 +25,7 @@ describe 'ProjectDuplicator', ->
 			folders:[@level1folder]
 			fileRefs:[{name:"file0", _id:"file0"}]
 		@project = 
-			_id: @project_id = "this_is_the_old_project"
+			_id: @old_project_id = "this_is_the_old_project_id"
 			rootDoc_id: "rootDoc_id"
 			rootFolder:[@rootFolder]
 			compiler: "this_is_a_Compiler"
@@ -45,7 +45,7 @@ describe 'ProjectDuplicator', ->
 
 		@owner = {_id:"this_is_the_owner"}
 		@stubbedNewProject = 
-			_id:"new_project_id"
+			_id: @new_project_id = "new_project_id"
 			readOnly_refs:[]
 			collaberator_refs:[]
 			rootFolder:[
@@ -59,15 +59,15 @@ describe 'ProjectDuplicator', ->
 		@newFolder = {_id: "newFolderId"}
 
 		@locator =
-			findRootDoc : sinon.stub().callsArgWith(1, null, @foundRootDoc)
+			findRootDoc : sinon.stub().callsArgWith(1, null, @foundRootDoc, {})
 
 		@projectOptionsHandler =
 			setCompiler : sinon.stub()								
 		@entityHandler =
-			addDoc: sinon.stub().callsArgWith(4, null, {name:"somDoc"})
-			copyFileFromExistingProject: sinon.stub().callsArgWith(4)
+			addDocWithProject: sinon.stub().callsArgWith(4, null, {name:"somDoc"})
+			copyFileFromExistingProjectWithProject: sinon.stub().callsArgWith(4)
 			setRootDoc: sinon.stub()
-			addFolder: sinon.stub().callsArgWith(3, null, @newFolder)
+			addFolderWithProject: sinon.stub().callsArgWith(3, null, @newFolder)
 
 		@DocumentUpdaterHandler =
 			flushProjectToMongo: sinon.stub().callsArg(1)
@@ -75,6 +75,12 @@ describe 'ProjectDuplicator', ->
 		@Project =
 			findById: sinon.stub().callsArgWith(1, null, @project)
 
+		@ProjectGetter =
+			getProject: sinon.stub()
+
+		@ProjectGetter.getProject.withArgs(@old_project_id, sinon.match.any).callsArgWith(2, null, @project)
+		@ProjectGetter.getProject.withArgs(@new_project_id, sinon.match.any).callsArgWith(2, null, @stubbedNewProject)
+
 		@duplicator = SandboxedModule.require modulePath, requires:
 			'../../models/Project':{Project:@Project}
 			"../DocumentUpdater/DocumentUpdaterHandler": @DocumentUpdaterHandler
@@ -83,63 +89,64 @@ describe 'ProjectDuplicator', ->
 			'./ProjectLocator': @locator
 			'./ProjectOptionsHandler': @projectOptionsHandler
 			"../Docstore/DocstoreManager": @DocstoreManager
+			"./ProjectGetter":@ProjectGetter
 			'logger-sharelatex':{log:->}
 
 	it "should look up the original project", (done) ->
 		newProjectName = "someProj"
-		@duplicator.duplicate @owner, @project_id, newProjectName, (err, newProject)=>
-			@Project.findById.calledWith(@project_id).should.equal true
+		@duplicator.duplicate @owner, @old_project_id, newProjectName, (err, newProject)=>
+			@ProjectGetter.getProject.calledWith(@old_project_id).should.equal true
 			done()
 
 	it "should flush the original project to mongo", (done) ->
 		newProjectName = "someProj"
-		@duplicator.duplicate @owner, @project_id, newProjectName, (err, newProject)=>
-			@DocumentUpdaterHandler.flushProjectToMongo.calledWith(@project_id).should.equal true
+		@duplicator.duplicate @owner, @old_project_id, newProjectName, (err, newProject)=>
+			@DocumentUpdaterHandler.flushProjectToMongo.calledWith(@old_project_id).should.equal true
 			done()
 
 	it 'should create a blank project', (done)->
 		newProjectName = "someProj"
-		@duplicator.duplicate @owner, @project_id, newProjectName, (err, newProject)=>
+		@duplicator.duplicate @owner, @old_project_id, newProjectName, (err, newProject)=>
 			newProject._id.should.equal @stubbedNewProject._id
 			@creationHandler.createBlankProject.calledWith(@owner._id, newProjectName).should.equal true
 			done()
 
 	it 'should use the same compiler', (done)->
-		@entityHandler.addDoc.callsArgWith(4, null, @rootFolder.docs[0])
-		@duplicator.duplicate @owner, @project_id, "", (err, newProject)=>
+		@entityHandler.addDocWithProject.callsArgWith(4, null, @rootFolder.docs[0])
+		@duplicator.duplicate @owner, @old_project_id, "", (err, newProject)=>
 			@projectOptionsHandler.setCompiler.calledWith(@stubbedNewProject._id, @project.compiler).should.equal true
 			done()
 	
-	it 'should use the same root doc', (done)->
-		@entityHandler.addDoc.callsArgWith(4, null, @rootFolder.docs[0])
-		@duplicator.duplicate @owner, @project_id, "", (err, newProject)=>
-			@entityHandler.setRootDoc.calledWith(@stubbedNewProject, @rootFolder.docs[0]._id).should.equal true
+	it 'should use the same root docccccccc', (done)->
+		@entityHandler.addDocWithProject.callsArgWith(4, null, @rootFolder.docs[0])
+		@duplicator.duplicate @owner, @old_project_id, "", (err, newProject)=>
+			@entityHandler.setRootDoc.calledWith(@stubbedNewProject._id, @rootFolder.docs[0]._id).should.equal true
 			done()
 
 	it 'should not copy the collaberators or read only refs', (done)->
-		@duplicator.duplicate @owner, @project_id, "", (err, newProject)=>
+		@duplicator.duplicate @owner, @old_project_id, "", (err, newProject)=>
 			newProject.collaberator_refs.length.should.equal 0
 			newProject.readOnly_refs.length.should.equal 0
 			done()	
 
 	it 'should copy all the folders', (done)->
-		@duplicator.duplicate @owner, @project_id, "", (err, newProject)=>
-			@entityHandler.addFolder.calledWith(@stubbedNewProject, @stubbedNewProject.rootFolder[0]._id, @level1folder.name).should.equal true
-			@entityHandler.addFolder.calledWith(@stubbedNewProject, @newFolder._id, @level2folder.name).should.equal true
-			@entityHandler.addFolder.callCount.should.equal 2
+		@duplicator.duplicate @owner, @old_project_id, "", (err, newProject)=>
+			@entityHandler.addFolderWithProject.calledWith(@stubbedNewProject, @stubbedNewProject.rootFolder[0]._id, @level1folder.name).should.equal true
+			@entityHandler.addFolderWithProject.calledWith(@stubbedNewProject, @newFolder._id, @level2folder.name).should.equal true
+			@entityHandler.addFolderWithProject.callCount.should.equal 2
 			done()
 
 	it 'should copy all the docs', (done)->
-		@duplicator.duplicate @owner, @project_id, "", (err, newProject)=>
-			@DocstoreManager.getAllDocs.calledWith(@project_id).should.equal true
-			@entityHandler.addDoc.calledWith(@stubbedNewProject._id, @stubbedNewProject.rootFolder[0]._id, @doc0.name, @doc0_lines).should.equal true
-			@entityHandler.addDoc.calledWith(@stubbedNewProject._id, @newFolder._id, @doc1.name, @doc1_lines).should.equal true
-			@entityHandler.addDoc.calledWith(@stubbedNewProject._id, @newFolder._id, @doc2.name, @doc2_lines).should.equal true
+		@duplicator.duplicate @owner, @old_project_id, "", (err, newProject)=>
+			@DocstoreManager.getAllDocs.calledWith(@old_project_id).should.equal true
+			@entityHandler.addDocWithProject.calledWith(@stubbedNewProject, @stubbedNewProject.rootFolder[0]._id, @doc0.name, @doc0_lines).should.equal true
+			@entityHandler.addDocWithProject.calledWith(@stubbedNewProject, @newFolder._id, @doc1.name, @doc1_lines).should.equal true
+			@entityHandler.addDocWithProject.calledWith(@stubbedNewProject, @newFolder._id, @doc2.name, @doc2_lines).should.equal true
 			done()
 
 	it 'should copy all the files', (done)->
-		@duplicator.duplicate @owner, @project_id, "", (err, newProject)=>
-			@entityHandler.copyFileFromExistingProject.calledWith(@stubbedNewProject, @stubbedNewProject.rootFolder[0]._id, @project._id, @rootFolder.fileRefs[0]).should.equal true
-			@entityHandler.copyFileFromExistingProject.calledWith(@stubbedNewProject, @newFolder._id, @project._id, @level1folder.fileRefs[0]).should.equal true
-			@entityHandler.copyFileFromExistingProject.calledWith(@stubbedNewProject, @newFolder._id, @project._id, @level2folder.fileRefs[0]).should.equal true
+		@duplicator.duplicate @owner, @old_project_id, "", (err, newProject)=>
+			@entityHandler.copyFileFromExistingProjectWithProject.calledWith(@stubbedNewProject, @stubbedNewProject.rootFolder[0]._id, @project._id, @rootFolder.fileRefs[0]).should.equal true
+			@entityHandler.copyFileFromExistingProjectWithProject.calledWith(@stubbedNewProject, @newFolder._id, @project._id, @level1folder.fileRefs[0]).should.equal true
+			@entityHandler.copyFileFromExistingProjectWithProject.calledWith(@stubbedNewProject, @newFolder._id, @project._id, @level2folder.fileRefs[0]).should.equal true
 			done()

services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee

@@ -28,6 +28,7 @@ describe 'ProjectEntityHandler', ->
 			folders:[
 				{name:"level1", folders:[]}
 			]
+		@ProjectUpdateStub = sinon.stub()
 		@ProjectModel = class Project
 			constructor:(options)->
 				@._id = project_id
@@ -35,6 +36,7 @@ describe 'ProjectEntityHandler', ->
 				@rev = 0
 			save:(callback)->callback()
 			rootFolder:[@rootFolder]
+		@ProjectModel.update = @ProjectUpdateStub
 
 		@DocModel = class Doc
 			constructor:(options)->
@@ -57,8 +59,13 @@ describe 'ProjectEntityHandler', ->
 		@ProjectModel.getProject = (project_id, fields, callback)=> callback(null, @project)
 		@ProjectGetter = 
 			getProjectWithOnlyFolders : (project_id, callback)=> callback(null, @project)
-		@ProjectModel.putElement = (project_id, folder_id, doc, type, callback)-> callback(null, {path:{fileSystem:"somehintg"}})
+			getProjectWithoutDocLines : (project_id, callback)=> callback(null, @project)
+			getProject:sinon.stub()
 		@projectUpdater = markAsUpdated:sinon.stub()
+		@projectLocator = 
+			findElement : sinon.stub()
+		@settings =
+			maxEntitiesPerProject:200
 		@ProjectEntityHandler = SandboxedModule.require modulePath, requires:
 			'../../models/Project': Project:@ProjectModel
 			'../../models/Doc': Doc:@DocModel
@@ -66,12 +73,13 @@ describe 'ProjectEntityHandler', ->
 			'../../models/File': File:@FileModel
 			'../FileStore/FileStoreHandler':@FileStoreHandler
 			'../ThirdPartyDataStore/TpdsUpdateSender':@tpdsUpdateSender
-			'./ProjectLocator':@projectLocator = {}
+			'./ProjectLocator': @projectLocator
 			'../../Features/DocumentUpdater/DocumentUpdaterHandler':@documentUpdaterHandler = {}
 			'../Docstore/DocstoreManager': @DocstoreManager = {}
-			'logger-sharelatex': @logger = {log:sinon.stub(), error: sinon.stub()}
+			'logger-sharelatex': @logger = {log:sinon.stub(), error: sinon.stub(), err:->}
 			'./ProjectUpdateHandler': @projectUpdater
 			"./ProjectGetter": @ProjectGetter
+			"settings-sharelatex":@settings
 
 
 	describe 'mkdirp', ->
@@ -79,7 +87,7 @@ describe 'ProjectEntityHandler', ->
 			@parentFolder_id = "1jnjknjk"
 			@newFolder = {_id:"newFolder_id_here"}
 			@lastFolder = {_id:"123das", folders:[]}
-			@ProjectGetter.getProjectWithoutDocLines = sinon.stub().callsArgWith(1, null, @project)
+			@ProjectGetter.getProjectWithOnlyFolders = sinon.stub().callsArgWith(1, null, @project)
 			@projectLocator.findElementByPath = (project_id, path, cb)=>
 				@parentFolder = {_id:"parentFolder_id_here"}
 				lastFolder = path.substring(path.lastIndexOf("/"))
@@ -140,6 +148,7 @@ describe 'ProjectEntityHandler', ->
 	describe 'deleteEntity', ->
 		entity_id = "4eecaffcbffa66588e000009"
 		beforeEach ->
+			@ProjectGetter.getProject.callsArgWith(2, null, @project)
 			@tpdsUpdateSender.deleteEntity = sinon.stub().callsArg(1)
 			@ProjectEntityHandler._removeElementFromMongoArray = sinon.stub().callsArg(3)
 			@ProjectEntityHandler._cleanUpEntity = sinon.stub().callsArg(3)
@@ -226,7 +235,8 @@ describe 'ProjectEntityHandler', ->
 				fileSystem: "/somewhere/else.txt"
 			}
 			@ProjectEntityHandler._removeElementFromMongoArray = sinon.stub().callsArg(3)
-			@ProjectModel.putElement = sinon.stub().callsArgWith(4, null, path: @pathAfterMove)
+			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, path: @pathAfterMove)
+			@ProjectGetter.getProject.callsArgWith(2, null, @project)
 			@tpdsUpdateSender.moveEntity = sinon.stub().callsArg(1)
 			
 		describe "moving a doc", ->
@@ -241,32 +251,14 @@ describe 'ProjectEntityHandler', ->
 				@ProjectEntityHandler.moveEntity project_id, @docId, folder_id, "docs", done
 
 			it 'should find the project then element', ->
-				@projectLocator.findElement
-					.calledWith({
-						element_id: @docId,
-						type: "docs",
-						project: @project
-					})
-					.should.equal true
+				@projectLocator.findElement.calledWith({element_id: @docId, type: "docs", project: @project }).should.equal true
 
 			it 'should remove the element from its current position', ->
 				@ProjectEntityHandler._removeElementFromMongoArray
-					.calledWith(
-						@ProjectModel,
-						project_id,
-						@path.mongo
-					)
-					.should.equal true
+					.calledWith(@ProjectModel, project_id, @path.mongo ).should.equal true
 					
 			it "should put the element back in the new folder", ->
-				@ProjectModel.putElement
-					.calledWith(
-						project_id,
-						folder_id,
-						@doc,
-						"docs"
-					)
-					.should.equal true
+				@ProjectEntityHandler._putElement.calledWith(@project, folder_id, @doc, "docs").should.equal true
 					
 			it 'should tell the third party data store', ->
 				@tpdsUpdateSender.moveEntity
@@ -327,9 +319,9 @@ describe 'ProjectEntityHandler', ->
 						.should.equal true
 						
 				it "should put the element back in the new folder", ->
-					@ProjectModel.putElement
+					@ProjectEntityHandler._putElement
 						.calledWith(
-							project_id,
+							@project,
 							@move_to_folder_id,
 							@folder,
 							"folder"
@@ -407,7 +399,7 @@ describe 'ProjectEntityHandler', ->
 			@lines = ['1234','abc']
 			@path = "/path/to/doc"
 
-			@ProjectModel.putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:@path}})
+			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:@path}})
 			@callback = sinon.stub()
 			@tpdsUpdateSender.addDoc = sinon.stub().callsArg(1)
 			@DocstoreManager.updateDoc = sinon.stub().callsArgWith(3, null, true, 0)
@@ -415,14 +407,12 @@ describe 'ProjectEntityHandler', ->
 			@ProjectEntityHandler.addDoc project_id, folder_id, @name, @lines, @callback
 
 			# Created doc
-			@doc = @ProjectModel.putElement.args[0][2]
+			@doc = @ProjectEntityHandler._putElement.args[0][2]
 			@doc.name.should.equal @name
 			expect(@doc.lines).to.be.undefined
 
 		it 'should call put element', ->
-			@ProjectModel.putElement
-				.calledWith(project_id, folder_id, @doc)
-				.should.equal true
+			@ProjectEntityHandler._putElement.calledWith(@project, folder_id, @doc).should.equal true
 
 		it 'should return doc and parent folder', ->
 			@callback.calledWith(null, @doc, folder_id).should.equal true
@@ -482,8 +472,8 @@ describe 'ProjectEntityHandler', ->
 			@ProjectEntityHandler.addFile project_id, folder_id, fileName, @filePath, (err, fileRef, parentFolder)->
 
 		it 'should put file into folder by calling put element', (done)->
-			@ProjectModel.putElement = (passedProject_id, passedFolder_id, passedFileRef, passedType, callback)->
-				passedProject_id.should.equal project_id
+			@ProjectEntityHandler._putElement = (passedProject, passedFolder_id, passedFileRef, passedType, callback)->
+				passedProject._id.should.equal project_id
 				passedFolder_id.should.equal folder_id
 				passedFileRef.name.should.equal fileName
 				passedType.should.equal 'file'
@@ -492,6 +482,7 @@ describe 'ProjectEntityHandler', ->
 			@ProjectEntityHandler.addFile project_id, folder_id, fileName, {}, (err, fileRef, parentFolder)->
 
 		it 'should return doc and parent folder', (done)->
+			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:"somehintg"}})
 			@ProjectEntityHandler.addFile project_id, folder_id, fileName, {}, (err, fileRef, parentFolder)->
 				parentFolder.should.equal folder_id
 				fileRef.name.should.equal fileName
@@ -502,10 +493,9 @@ describe 'ProjectEntityHandler', ->
 			opts =
 				path : "/somehwere/idsadsds"
 				project_id : project_id
-			@ProjectModel.putElement = (project_id, folder_id, doc, type, callback)-> callback(null, {path:{fileSystem:opts.path}})
+			@ProjectEntityHandler._putElement = (project_id, folder_id, doc, type, callback)-> callback(null, {path:{fileSystem:opts.path}})
 
 			@tpdsUpdateSender.addFile = (options)=>
-				console.log @project.name, options.project_name
 				options.project_id.should.equal project_id
 				options.path.should.equal opts.path
 				options.project_name.should.equal @project.name
@@ -525,6 +515,7 @@ describe 'ProjectEntityHandler', ->
 			@filePaths = {fileSystem:"/folder1/file.png", mongo:"folder.1.files.somewhere"}
 			@projectLocator.findElement = sinon.stub().callsArgWith(1, null, @fileRef, @filePaths)
 			@ProjectModel.update = (_, __, ___, cb)-> cb()
+			@ProjectGetter.getProject = sinon.stub().callsArgWith(2, null, @project)
 
 		it 'should find the file', (done)->
 
@@ -563,18 +554,20 @@ describe 'ProjectEntityHandler', ->
 			@ProjectModel.update = (conditions, update, options, callback)=>
 				conditions._id.should.equal project_id
 				differenceInMs = update.$set["#{@filePaths.mongo}.created"].getTime() - d.getTime()
-				differenceInMs.should.be.below(10)
+				differenceInMs.should.be.below(20)
 				done()
 
 			@ProjectEntityHandler.replaceFile project_id, @file_id, @fsPath, =>
 
 
-	describe 'adding a folder', ->
+	describe 'addFolder', ->
 		folderName = "folder1234"
+		beforeEach ->
+			@ProjectGetter.getProjectWithOnlyFolders = sinon.stub().callsArgWith(1, null, @project)
 
 		it 'should call put element', (done)->
-			@ProjectModel.putElement = (passedProject_id, passedFolder_id, passedFolder, passedType, callback)->
-				passedProject_id.should.equal project_id
+			@ProjectEntityHandler._putElement = (passedProject, passedFolder_id, passedFolder, passedType, callback)->
+				passedProject._id.should.equal project_id
 				passedFolder_id.should.equal folder_id
 				passedFolder.name.should.equal folderName
 				passedType.should.equal 'folder'
@@ -582,6 +575,7 @@ describe 'ProjectEntityHandler', ->
 			@ProjectEntityHandler.addFolder project_id, folder_id, folderName, (err, folder, parentFolder)->
 
 		it 'should return the folder and parent folder', (done)->
+			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4)
 			@ProjectEntityHandler.addFolder project_id, folder_id, folderName, (err, folder, parentFolder)->
 				parentFolder.should.equal folder_id
 				folder.name.should.equal folderName
@@ -803,27 +797,21 @@ describe 'ProjectEntityHandler', ->
 			@ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs)
 			@ProjectEntityHandler.getAllFiles = sinon.stub().callsArgWith(1, null, @files)
 
+			@ProjectGetter.getProject.callsArgWith(2, null, @project)
+
 			@ProjectEntityHandler.flushProjectToThirdPartyDataStore project_id, () -> done()
 
 		it "should flush the project from the doc updater", ->
-			@documentUpdaterHandler.flushProjectToMongo
-				.calledWith(project_id)
-				.should.equal true
+			@documentUpdaterHandler.flushProjectToMongo.calledWith(project_id).should.equal true
 
 		it "should look up the project in mongo", ->
-			@ProjectModel.findById
-				.calledWith(project_id)
-				.should.equal true
+			@ProjectGetter.getProject.calledWith(project_id).should.equal true
 
 		it "should get all the docs in the project", ->
-			@ProjectEntityHandler.getAllDocs
-				.calledWith(project_id)
-				.should.equal true
+			@ProjectEntityHandler.getAllDocs.calledWith(project_id).should.equal true
 
 		it "should get all the files in the project", ->
-			@ProjectEntityHandler.getAllFiles
-				.calledWith(project_id)
-				.should.equal true
+			@ProjectEntityHandler.getAllFiles.calledWith(project_id).should.equal true
 
 		it "should flush each doc to the TPDS", ->
 			for path, doc of @docs
@@ -867,20 +855,25 @@ describe 'ProjectEntityHandler', ->
 			@ProjectModel.update.calledWith({_id : @project_id}, {$unset : {rootDoc_id: true}})
 				.should.equal true
 
-	describe 'copying file', ->
+	describe 'copyFileFromExistingProject', ->
 		fileName = "something.jpg"
 		filePath = "dumpFolder/somewhere/image.jpeg"
 		oldProject_id = "123kljadas"
 		oldFileRef = {name:fileName, _id:"oldFileRef"}
+		beforeEach ->
+			@ProjectGetter.getProject = (project_id, fields, callback)=> callback(null, {name:@project.name, _id:@project._id})
+			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:"somehintg"}})
+
 
 		it 'should copy the file in FileStoreHandler', (done)->
+			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:"somehintg"}})
 			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)=>     
 				@FileStoreHandler.copyFile.calledWith(oldProject_id, oldFileRef._id, project_id, fileRef._id).should.equal true
 				done()
 
 		it 'should put file into folder by calling put element', (done)->
-			@ProjectModel.putElement = (passedProject_id, passedFolder_id, passedFileRef, passedType, callback)-> 
-				passedProject_id.should.equal project_id
+			@ProjectEntityHandler._putElement = (passedProject, passedFolder_id, passedFileRef, passedType, callback)-> 
+				passedProject._id.should.equal project_id
 				passedFolder_id.should.equal folder_id
 				passedFileRef.name.should.equal fileName
 				passedType.should.equal 'file'
@@ -899,7 +892,7 @@ describe 'ProjectEntityHandler', ->
 			opts =
 				path : "/somehwere/idsadsds"
 				project_id : project_id
-			@ProjectModel.putElement = (project_id, folder_id, doc, type, callback)-> callback(null, {path:{fileSystem:opts.path}})
+			@ProjectEntityHandler._putElement = (project_id, folder_id, doc, type, callback)-> callback(null, {path:{fileSystem:opts.path}})
 
 			@tpdsUpdateSender.addFile = (options)=>
 				options.project_id.should.equal project_id
@@ -921,6 +914,7 @@ describe 'ProjectEntityHandler', ->
 			@projectLocator.findElement = sinon.stub().callsArgWith(1, null, @entity = { _id: @entity_id, name:"old.tex", rev:4 }, @path)
 			@ProjectModel.update = sinon.stub().callsArgWith(3)
 			@tpdsUpdateSender.moveEntity = sinon.stub()
+			@ProjectGetter.getProject.callsArgWith(2, null, @project)
 
 		it "should update the name in mongo", (done)->
 
@@ -1008,4 +1002,121 @@ describe 'ProjectEntityHandler', ->
 				@ProjectEntityHandler.unsetRootDoc.called.should.equal false
 
 			it "should call the callback", ->
-				@callback.called.should.equal true
+				@callback.called.should.equal true
+
+
+	describe "_putElement", ->
+		beforeEach ->
+			@project_id = project_id
+			@project =
+				_id: ObjectId(project_id)
+				rootFolder: [_id:ObjectId()]
+			@folder =
+				_id: ObjectId()
+				name: "someFolder"
+			@doc = 
+				_id: ObjectId()
+				name: "new.tex"
+			@path = mongo: "mongo.path", fileSystem: "/file/system/old.tex"
+			@ProjectGetter.getProject.callsArgWith(2, null, @project)
+			@projectLocator.findElement.callsArgWith(1, null, @folder, @path)
+			@ProjectUpdateStub.callsArgWith(3)
+
+
+		describe "updating the project", ->
+			
+
+			it "should use the correct mongo path", (done)->
+				@ProjectEntityHandler._putElement @project, @folder._id, @doc, "docs", (err)=>
+					@ProjectModel.update.args[0][0]._id.should.equal @project._id
+					assert.deepEqual @ProjectModel.update.args[0][1].$push[@path.mongo+".docs"], @doc
+					done()
+
+			it "should add an s onto the type if not included", (done)->
+				@ProjectEntityHandler._putElement @project, @folder._id, @doc, "doc", (err)=>
+					assert.deepEqual @ProjectModel.update.args[0][1].$push[@path.mongo+".docs"], @doc
+					done()
+
+
+			it "should not call update if elemenet is null", (done)->
+				@ProjectEntityHandler._putElement @project, @folder._id, null, "doc", (err)=>
+					@ProjectModel.update.called.should.equal false
+					done()
+
+			it "should default to root folder insert", (done)->
+				@ProjectEntityHandler._putElement @project, null, @doc, "doc", (err)=>
+					@projectLocator.findElement.args[0][0].element_id.should.equal @project.rootFolder[0]._id
+					done()
+
+			it "should error if the element has no _id", (done)->
+				doc = 
+					name:"something"
+				@ProjectEntityHandler._putElement @project, @folder._id, doc, "doc", (err)=>
+					@ProjectModel.update.called.should.equal false
+					done()
+			
+
+
+		describe "_countElements", ->
+
+			beforeEach ->
+				@project.rootFolder[0].docs = [{_id:123}, {_id:345}]
+				@project.rootFolder[0].fileRefs = [{_id:123}, {_id:345}, {_id:456}]
+				@project.rootFolder[0].folders = [
+					{
+						docs:
+							[{_id:123}, {_id:345}, {_id:456}]
+						fileRefs:{}
+						folders: [
+							{
+								docs:[_id:1234], 
+								fileRefs:[{_id:23123}, {_id:123213}, {_id:2312}]
+								folders:[
+									{
+										docs:[{_id:321321}, {_id:123213}]
+										fileRefs:[{_id:312321}]
+										folders:[]
+									}
+								]
+							}
+						]
+					},{
+						docs:[{_id:123}, {_id:32131}]
+						fileRefs:[]
+						folders:[
+							{
+								docs:[{_id:3123}]
+								fileRefs:[{_id:321321}, {_id:321321}, {_id:313122}]
+								folders:0
+							}
+						]
+					}
+				]				
+
+			it "should return the correct number", (done)->
+				@ProjectEntityHandler._countElements @project, (err, count)->
+					count.should.equal 26
+					done()
+
+			it "should deal with null folders", (done)->
+				@project.rootFolder[0].folders[0].folders = undefined
+				@ProjectEntityHandler._countElements @project, (err, count)->
+					count.should.equal 17
+					done()				
+
+			it "should deal with null docs", (done)->
+				@project.rootFolder[0].folders[0].docs = undefined
+				@ProjectEntityHandler._countElements @project, (err, count)->
+					count.should.equal 23
+					done()				
+
+			it "should deal with null fileRefs", (done)->
+				@project.rootFolder[0].folders[0].folders[0].fileRefs = undefined
+				@ProjectEntityHandler._countElements @project, (err, count)->
+					count.should.equal 23
+					done()	
+
+
+
+
+

services/web/test/UnitTests/coffee/Project/ProjectGetterTests.coffee

@@ -18,66 +18,123 @@ describe "ProjectGetter", ->
 				ObjectId: ObjectId
 			"../../models/Project": Project: @Project = {}
 			"../Collaborators/CollaboratorsHandler": @CollaboratorsHandler = {}
+			"logger-sharelatex":
+				err:->
+				log:->
 
 	describe "getProjectWithoutDocLines", ->
 		beforeEach ->
 			@project =
-				_id: @project_id = "0123456789abcd9876543210"
+				_id: @project_id = "56d46b0a1d3422b87c5ebcb1"
 			@db.projects.find = sinon.stub().callsArgWith(2, null, [@project])
-			@ProjectGetter.getProjectWithoutDocLines @project_id, @callback
-
-		it "should call find with the project id", ->
-			@db.projects.find.calledWith(_id: ObjectId(@project_id)).should.equal true
-
-		it "should exclude the doc lines", ->
-			excludes =
-				"rootFolder.docs.lines": 0
-				"rootFolder.folder.docs.lines": 0
-				"rootFolder.folder.folder.docs.lines": 0
-				"rootFolder.folder.folder.folder.docs.lines": 0
-				"rootFolder.folder.folder.folder.folder.docs.lines": 0
-				"rootFolder.folder.folder.folder.folder.folder.docs.lines": 0
-				"rootFolder.folder.folder.folder.folder.folder.folder.docs.lines": 0
-				"rootFolder.folder.folder.folder.folder.folder.folder.folder.docs.lines": 0
-			@db.projects.find.calledWith(sinon.match.any, excludes)
-				.should.equal true
 
-		it "should call the callback with the project", ->
-			@callback.calledWith(null, @project).should.equal true
+		describe "passing an id", ->
+			beforeEach ->
+				@ProjectGetter.getProjectWithoutDocLines @project_id, @callback
+
+			it "should call find with the project id", ->
+				@db.projects.find.calledWith(_id: ObjectId(@project_id)).should.equal true
+
+			it "should exclude the doc lines", ->
+				excludes =
+					"rootFolder.docs.lines": 0
+					"rootFolder.folder.docs.lines": 0
+					"rootFolder.folder.folder.docs.lines": 0
+					"rootFolder.folder.folder.folder.docs.lines": 0
+					"rootFolder.folder.folder.folder.folder.docs.lines": 0
+					"rootFolder.folder.folder.folder.folder.folder.docs.lines": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.docs.lines": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.folder.docs.lines": 0
+				@db.projects.find.calledWith(sinon.match.any, excludes)
+					.should.equal true
+
+			it "should call the callback with the project", ->
+				@callback.calledWith(null, @project).should.equal true
+
 
 	describe "getProjectWithOnlyFolders", ->
-		beforeEach ->
+
+		beforeEach ()->
 			@project =
-				_id: @project_id = "0123456789abcd9876543210"
+				_id: @project_id = "56d46b0a1d3422b87c5ebcb1"
+			@db.projects.find = sinon.stub().callsArgWith(2, null, [@project])
+	
+		describe "passing an id", ->
+			beforeEach ->
+				@ProjectGetter.getProjectWithOnlyFolders @project_id, @callback
+
+			it "should call find with the project id", ->
+				@db.projects.find.calledWith(_id: ObjectId(@project_id)).should.equal true
+
+			it "should exclude the docs and files linesaaaa", ->
+				excludes =
+					"rootFolder.docs": 0
+					"rootFolder.fileRefs": 0
+					"rootFolder.folder.docs": 0
+					"rootFolder.folder.fileRefs": 0
+					"rootFolder.folder.folder.docs": 0
+					"rootFolder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.folder.fileRefs": 0
+				@db.projects.find.calledWith(sinon.match.any, excludes).should.equal true
+
+			it "should call the callback with the project", ->
+				@callback.calledWith(null, @project).should.equal true
+
+
+
+	describe "getProject", ->
+		beforeEach ()->
+			@project =
+				_id: @project_id = "56d46b0a1d3422b87c5ebcb1"
+			@db.projects.find = sinon.stub().callsArgWith(2, null, [@project])
+	
+		describe "passing an id", ->
+			beforeEach ->
+				@ProjectGetter.getProjectWithOnlyFolders @project_id, @callback
+
+			it "should call find with the project id", ->
+				@db.projects.find.calledWith(_id: ObjectId(@project_id)).should.equal true
+
+			it "should exclude the docs and files linesaaaa", ->
+				excludes =
+					"rootFolder.docs": 0
+					"rootFolder.fileRefs": 0
+					"rootFolder.folder.docs": 0
+					"rootFolder.folder.fileRefs": 0
+					"rootFolder.folder.folder.docs": 0
+					"rootFolder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.fileRefs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.folder.docs": 0
+					"rootFolder.folder.folder.folder.folder.folder.folder.folder.fileRefs": 0
+				@db.projects.find.calledWith(sinon.match.any, excludes).should.equal true
+
+			it "should call the callback with the project", ->
+				@callback.calledWith(null, @project).should.equal true
+
+
+
+	describe "getProject", ->
+		beforeEach ()->
+			@project =
+				_id: @project_id = "56d46b0a1d3422b87c5ebcb1"
 			@db.projects.find = sinon.stub().callsArgWith(2, null, [@project])
-			@ProjectGetter.getProjectWithOnlyFolders @project_id, @callback
-
-		it "should call find with the project id", ->
-			@db.projects.find.calledWith(_id: ObjectId(@project_id)).should.equal true
-
-		it "should exclude the docs and files lines", ->
-			excludes =
-				"rootFolder.docs": 0
-				"rootFolder.fileRefs": 0
-				"rootFolder.folder.docs": 0
-				"rootFolder.folder.fileRefs": 0
-				"rootFolder.folder.folder.docs": 0
-				"rootFolder.folder.folder.fileRefs": 0
-				"rootFolder.folder.folder.folder.docs": 0
-				"rootFolder.folder.folder.folder.fileRefs": 0
-				"rootFolder.folder.folder.folder.folder.docs": 0
-				"rootFolder.folder.folder.folder.folder.fileRefs": 0
-				"rootFolder.folder.folder.folder.folder.folder.docs": 0
-				"rootFolder.folder.folder.folder.folder.folder.fileRefs": 0
-				"rootFolder.folder.folder.folder.folder.folder.folder.docs": 0
-				"rootFolder.folder.folder.folder.folder.folder.folder.fileRefs": 0
-				"rootFolder.folder.folder.folder.folder.folder.folder.folder.docs": 0
-				"rootFolder.folder.folder.folder.folder.folder.folder.folder.fileRefs": 0
-			@db.projects.find.calledWith(sinon.match.any, excludes)
-				.should.equal true
 
-		it "should call the callback with the project", ->
-			@callback.calledWith(null, @project).should.equal true
 
 	describe "findAllUsersProjects", ->
 		beforeEach ->

services/web/test/UnitTests/coffee/Project/ProjectLocatorTests.coffee

@@ -38,10 +38,12 @@ describe 'ProjectLocator', ->
 
 		Project.findById = (project_id, callback)=>
 			callback(null, project)
+		@ProjectGetter = 
+			getProject: sinon.stub().callsArgWith(2, null, project)
 		@locator = SandboxedModule.require modulePath, requires:
 			'../../models/Project':{Project:Project}
 			'../../models/User':{User:@User}
-			'./ProjectGetter': @ProjectGetter = {}
+			"./ProjectGetter":@ProjectGetter
 			'logger-sharelatex':
 				log:->
 				err:->

services/web/test/UnitTests/coffee/Security/LoginRateLimiterTests.coffee

@@ -26,6 +26,7 @@ describe "LoginRateLimiter", ->
 
 		@LoginRateLimiter = SandboxedModule.require modulePath, requires:
 			'redis-sharelatex' : createClient: () => @rclient
+			"settings-sharelatex":{redis:{}}
 	
 	describe "processLoginRequest", ->
 

services/web/test/UnitTests/coffee/Subscription/UserFeaturesUpdaterTests.coffee

@@ -5,7 +5,7 @@ modulePath = "../../../../app/js/Features/Subscription/UserFeaturesUpdater"
 assert = require("chai").assert
 
 
-describe "user Features updater", ->
+describe "UserFeaturesUpdater", ->
 
 	beforeEach ->
 

services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsControllerTests.coffee

@@ -14,6 +14,8 @@ describe 'TpdsController', ->
 			'logger-sharelatex':
 				log:->
 				err:->
+			"../../infrastructure/Metrics": inc:->
+				
 		@user_id = "dsad29jlkjas"
 
 	describe 'getting an update', ->

services/web/test/UnitTests/coffee/ThirdPartyDataStore/TpdsUpdateSenderTests.coffee

@@ -42,6 +42,8 @@ describe 'TpdsUpdateSender', ->
 			'../../models/Project': Project:@Project
 			'request':@request
 			'../Collaborators/CollaboratorsHandler': @CollaboratorsHandler
+			"../../infrastructure/Metrics":
+				inc:-> 
 
 	describe "_enqueue", ->
 

services/web/test/UnitTests/coffee/ThirdPartyDataStore/UpdateMergerTests.coffee

@@ -20,9 +20,13 @@ describe 'UpdateMerger :', ->
 			'../Project/ProjectEntityHandler': @projectEntityHandler
 			'fs': @fs
 			'../Uploads/FileTypeManager':@FileTypeManager
+			'settings-sharelatex':{path:{dumpPath:"dump_here"}}
 			'logger-sharelatex':
 				log: ->
 				err: ->
+			"../../infrastructure/Metrics": 
+				Timer:->
+					done:->
 		@project_id = "project_id_here"
 		@user_id = "mock-user-id"
 		@source = "dropbox"

services/web/test/UnitTests/coffee/Uploads/ArchiveManagerTests.coffee

@@ -1,4 +1,5 @@
 sinon = require('sinon')
+expect = require("chai").expect
 chai = require('chai')
 should = chai.should()
 modulePath = "../../../../app/js/Features/Uploads/ArchiveManager.js"
@@ -9,12 +10,16 @@ describe "ArchiveManager", ->
 	beforeEach ->
 		@logger =
 			error: sinon.stub()
+			err:->
 			log: sinon.stub()
 		@process = new events.EventEmitter
 		@process.stdout = new events.EventEmitter
 		@process.stderr = new events.EventEmitter
+
 		@child =
 			spawn: sinon.stub().returns(@process)
+
+
 		@metrics =
 			Timer: class Timer
 				done: sinon.stub()
@@ -29,6 +34,7 @@ describe "ArchiveManager", ->
 			@source = "/path/to/zip/source.zip"
 			@destination = "/path/to/zip/destination"
 			@callback = sinon.stub()
+			@ArchiveManager._isZipTooLarge = sinon.stub().callsArgWith(1, null, false)
 
 		describe "successfully", ->
 			beforeEach (done) ->
@@ -58,6 +64,19 @@ describe "ArchiveManager", ->
 			it "should log out the error", ->
 				@logger.error.called.should.equal true
 
+		describe "with a zip that is too large", ->
+			beforeEach (done) ->
+				@ArchiveManager._isZipTooLarge = sinon.stub().callsArgWith(1, null, true)
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+
+			it "should return the callback with an error", ->
+				@callback.calledWithExactly(new Error("zip_too_large")).should.equal true
+
+			it "should not call spawn", ->
+				@child.spawn.called.should.equal false
+
 		describe "with an error on the process", ->
 			beforeEach (done) ->
 				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
@@ -71,6 +90,45 @@ describe "ArchiveManager", ->
 			it "should log out the error", ->
 				@logger.error.called.should.equal true
 	
+	describe "_isZipTooLarge", ->
+		beforeEach ->
+			@output = (totalSize)->"  Length     Date   Time    Name \n--------    ----   ----    ---- \n241  03-12-16 12:20   main.tex \n108801  03-12-16 12:20   ddd/x1J5kHh.jpg \n--------                   ------- \n#{totalSize}                   2 files\n"
+
+		it "should return false with small output", (done)->
+			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
+				isTooLarge.should.equal false
+				done()
+			@process.stdout.emit "data", @output("109042")
+			@process.emit "exit"
+
+		it "should return true with large bytes", (done)->
+			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
+				isTooLarge.should.equal true
+				done()
+			@process.stdout.emit "data", @output("1090000000000000042")
+			@process.emit "exit"
+
+		it "should return error on no data", (done)->
+			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
+				expect(error).to.exist
+				done()
+			@process.stdout.emit "data", ""
+			@process.emit "exit"
+
+		it "should return error if it didn't get a number", (done)->
+			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
+				expect(error).to.exist
+				done()
+			@process.stdout.emit "data", @output("total_size_string")
+			@process.emit "exit"
+
+		it "should return error if the is only a bit of data", (done)->
+			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
+				expect(error).to.exist
+				done()
+			@process.stdout.emit "data", "  Length     Date   Time    Name \n--------"
+			@process.emit "exit"
+
 	describe "findTopLevelDirectory", ->
 		beforeEach ->
 			@fs.readdir = sinon.stub()

services/web/test/UnitTests/coffee/Uploads/FileSystemImportManagerTests.coffee

@@ -18,12 +18,29 @@ describe "FileSystemImportManager", ->
 			"../Editor/EditorController": @EditorController = {}
 			"./FileTypeManager": @FileTypeManager = {}
 			"../Project/ProjectLocator": @ProjectLocator = {}
+			"logger-sharelatex":
+				log:->
+				err:->
 	
 	describe "addDoc", ->
 		beforeEach ->
 			@docContent = "one\ntwo\nthree"
 			@docLines = @docContent.split("\n")
 			@fs.readFile = sinon.stub().callsArgWith(2, null, @docContent)
+			@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
+
+
+		describe "when path is symlink", ->
+			beforeEach ->
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, false)
+				@EditorController.addDocWithoutLock = sinon.stub()
+				@FileSystemImportManager.addDoc @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
+
+			it "should not read the file from disk", ->
+				@fs.readFile.called.should.equal false
+
+			it "should not insert the doc", ->
+				@EditorController.addDocWithoutLock.called.should.equal false
 
 		describe "with replace set to false", ->
 			beforeEach ->
@@ -98,12 +115,24 @@ describe "FileSystemImportManager", ->
 	describe "addFile with replace set to false", ->
 		beforeEach ->
 			@EditorController.addFileWithoutLock = sinon.stub().callsArg(5)
+			@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 			@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
 
 		it "should add the file", ->
 			@EditorController.addFileWithoutLock.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload")
 				.should.equal true
 
+	describe "addFile with symlink", ->
+		beforeEach ->
+			@EditorController.addFileWithoutLock = sinon.stub()
+			@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, false)
+			@EditorController.replaceFile = sinon.stub()
+			@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
+
+		it "should node add the file", ->
+			@EditorController.addFileWithoutLock.called.should.equal false
+			@EditorController.replaceFile.called.should.equal false
+
 	describe "addFile with replace set to true", ->
 		describe "when the file doesn't exist", ->
 			beforeEach ->
@@ -113,6 +142,7 @@ describe "FileSystemImportManager", ->
 						name: "not-the-right-file.tex"
 					}]
 				}
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 				@ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder)
 				@EditorController.addFileWithoutLock = sinon.stub().callsArg(5)
 				@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, true, @callback
@@ -137,6 +167,7 @@ describe "FileSystemImportManager", ->
 						name: "not-the-right-file.tex"
 					}]
 				}
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 				@ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder)
 				@EditorController.replaceFile = sinon.stub().callsArg(4)
 				@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, true, @callback
@@ -151,20 +182,34 @@ describe "FileSystemImportManager", ->
 					.should.equal true
 
 	describe "addFolder", ->
+
 		beforeEach ->
 			@new_folder_id = "new-folder-id"
 			@EditorController.addFolderWithoutLock  = sinon.stub().callsArgWith(4, null, _id: @new_folder_id)
 			@FileSystemImportManager.addFolderContents = sinon.stub().callsArg(5)
-			@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
 
-		it "should add a folder to the project", ->
-			@EditorController.addFolderWithoutLock.calledWith(@project_id, @folder_id, @name, "upload")
-				.should.equal true
+		describe "successfully", ->
+			beforeEach ->
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
+				@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
 
-		it "should add the folders contents", ->
-			@FileSystemImportManager.addFolderContents.calledWith(@user_id, @project_id, @new_folder_id, @path_on_disk, @replace)
-				.should.equal true
+			it "should add a folder to the project", ->
+				@EditorController.addFolderWithoutLock.calledWith(@project_id, @folder_id, @name, "upload")
+					.should.equal true
+
+			it "should add the folders contents", ->
+				@FileSystemImportManager.addFolderContents.calledWith(@user_id, @project_id, @new_folder_id, @path_on_disk, @replace)
+					.should.equal true
+
+		describe "with symlink", ->
+			beforeEach ->
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, false)
+				@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
 
+			it "should not add a folder to the project", ->
+				@EditorController.addFolderWithoutLock.called.should.equal false
+				@FileSystemImportManager.addFolderContents.called.should.equal false
+				
 	describe "addFolderContents", ->
 		beforeEach ->
 			@folderEntries = ["path1", "path2", "path3"]
@@ -173,6 +218,7 @@ describe "FileSystemImportManager", ->
 			@FileSystemImportManager.addEntity = sinon.stub().callsArg(6)
 			@FileTypeManager.shouldIgnore = (path, callback) =>
 				callback null, @ignoredEntries.indexOf(require("path").basename(path)) != -1
+			@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 			@FileSystemImportManager.addFolderContents @user_id, @project_id, @folder_id, @path_on_disk, @replace, @callback
 
 		it "should call addEntity for each file in the folder which is not ignored", ->
@@ -193,6 +239,7 @@ describe "FileSystemImportManager", ->
 			beforeEach ->
 				@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, true)
 				@FileSystemImportManager.addFolder = sinon.stub().callsArg(6)
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 				@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
 
 			it "should call addFolder", ->
@@ -203,6 +250,7 @@ describe "FileSystemImportManager", ->
 			beforeEach ->
 				@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, false)
 				@FileTypeManager.isBinary = sinon.stub().callsArgWith(2, null, true)
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 				@FileSystemImportManager.addFile = sinon.stub().callsArg(6)
 				@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
 
@@ -215,6 +263,7 @@ describe "FileSystemImportManager", ->
 				@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, false)
 				@FileTypeManager.isBinary = sinon.stub().callsArgWith(2, null, false)
 				@FileSystemImportManager.addDoc = sinon.stub().callsArg(6)
+				@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
 				@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
 
 			it "should call addFile", ->

services/web/test/UnitTests/coffee/User/UserControllerTests.coffee

@@ -42,6 +42,8 @@ describe "UserController", ->
 			changeEmailAddress:sinon.stub()
 		@settings =
 			siteUrl: "sharelatex.example.com"
+		@UserHandler = 
+			populateGroupLicenceInvite:sinon.stub().callsArgWith(1)
 		@UserController = SandboxedModule.require modulePath, requires:
 			"./UserLocator": @UserLocator
 			"./UserDeleter": @UserDeleter
@@ -53,9 +55,10 @@ describe "UserController", ->
 			"../Authentication/AuthenticationManager": @AuthenticationManager
 			"../Referal/ReferalAllocator":@ReferalAllocator
 			"../Subscription/SubscriptionDomainHandler":@SubscriptionDomainHandler
+			"./UserHandler":@UserHandler
 			"settings-sharelatex": @settings
 			"logger-sharelatex": {log:->}
-
+			"../../infrastructure/Metrics": inc:->
 
 		@req = 
 			session: 
@@ -151,7 +154,14 @@ describe "UserController", ->
 				done()
 			@UserController.updateUserSettings @req, @res
 
-
+		it "should call populateGroupLicenceInvite", (done)->
+			@req.body.email = @newEmail.toUpperCase()
+			@UserUpdater.changeEmailAddress.callsArgWith(2)
+			@res.sendStatus = (code)=>
+				code.should.equal 200
+				@UserHandler.populateGroupLicenceInvite.calledWith(@user).should.equal true
+				done()
+			@UserController.updateUserSettings @req, @res
 
 	describe "logout", ->
 
@@ -178,7 +188,6 @@ describe "UserController", ->
 				.should.equal true
 		
 		it "should return the user and activation url", ->
-			console.log @res.json.args
 			@res.json
 				.calledWith({
 					email: @email,

services/web/test/UnitTests/coffee/User/UserHandlerTests.coffee

@@ -29,12 +29,12 @@ describe "UserHandler", ->
 			"../Subscription/SubscriptionDomainHandler":@SubscriptionDomainHandler
 			"../Subscription/SubscriptionGroupHandler":@SubscriptionGroupHandler
 
-	describe "_populateGroupLicenceInvite", ->
+	describe "populateGroupLicenceInvite", ->
 
 		describe "no licence", ->
 			beforeEach (done)->
 				@SubscriptionDomainHandler.getLicenceUserCanJoin.returns()
-				@UserHandler._populateGroupLicenceInvite @user, done
+				@UserHandler.populateGroupLicenceInvite @user, done
 
 			it "should not call NotificationsBuilder", (done)->
 				@NotificationsBuilder.groupPlan.called.should.equal false
@@ -49,7 +49,7 @@ describe "UserHandler", ->
 			beforeEach (done)->
 				@SubscriptionDomainHandler.getLicenceUserCanJoin.returns(@licence)
 				@SubscriptionGroupHandler.isUserPartOfGroup.callsArgWith(2, null, false)
-				@UserHandler._populateGroupLicenceInvite @user, done
+				@UserHandler.populateGroupLicenceInvite @user, done
 
 			it "should create notifcation", (done)->
 				@NotificationsBuilder.groupPlan.calledWith(@user, @licence).should.equal true
@@ -62,7 +62,7 @@ describe "UserHandler", ->
 			beforeEach (done)->
 				@SubscriptionDomainHandler.getLicenceUserCanJoin.returns(@licence)
 				@SubscriptionGroupHandler.isUserPartOfGroup.callsArgWith(2, null, true)
-				@UserHandler._populateGroupLicenceInvite @user, done
+				@UserHandler.populateGroupLicenceInvite @user, done
 
 			it "should create notifcation", (done)->
 				@NotificationsBuilder.groupPlan.called.should.equal false

services/web/test/UnitTests/coffee/infrastructure/LockManager/getLockTests.coffee

@@ -12,6 +12,9 @@ describe 'LockManager - getting the lock', ->
 			"redis-sharelatex":
 				createClient : () =>
 					auth:->
+			"settings-sharelatex":{redis:{}}
+			"./Metrics": inc:->
+
 		@callback = sinon.stub()
 		@doc_id = "doc-id-123"
 	

services/web/test/UnitTests/coffee/infrastructure/LockManager/tryLockTests.coffee

@@ -13,6 +13,8 @@ describe 'LockManager - trying the lock', ->
 				createClient : () =>
 					auth:->
 					set: @set = sinon.stub()
+			"settings-sharelatex":{redis:{}}
+			"./Metrics": inc:->			
 		@callback = sinon.stub()
 		@doc_id = "doc-id-123"