Merge branch 'master' into sk-unlisted-projects

services/web/.gitignore

@@ -61,6 +61,7 @@ public/js/utils/
 
 public/stylesheets/style.css
 public/stylesheets/ol-style.css
+public/stylesheets/*.map
 public/brand/plans.css
 public/minjs/
 

services/web/Gruntfile.coffee

@@ -20,6 +20,8 @@ module.exports = (grunt) ->
 	grunt.loadNpmTasks 'grunt-parallel'
 	grunt.loadNpmTasks 'grunt-exec'
 	grunt.loadNpmTasks 'grunt-postcss'
+	grunt.loadNpmTasks 'grunt-forever'
+	grunt.loadNpmTasks 'grunt-shell'
 	# grunt.loadNpmTasks 'grunt-contrib-imagemin'
 	# grunt.loadNpmTasks 'grunt-sprity'
 
@@ -28,9 +30,15 @@ module.exports = (grunt) ->
 		exec:
 			run:
 				command:"node app.js | ./node_modules/logger-sharelatex/node_modules/bunyan/bin/bunyan --color"
-			cssmin:
-				command:"node_modules/clean-css/bin/cleancss --s0 -o public/stylesheets/style.css public/stylesheets/style.css"
+			cssmin_sl:
+				command:"node_modules/clean-css/bin/cleancss --s0 --source-map -o public/stylesheets/style.css public/stylesheets/style.css"
+			cssmin_ol:
+				command:"node_modules/clean-css/bin/cleancss --s0 --source-map -o public/stylesheets/ol-style.css public/stylesheets/ol-style.css"
 
+		forever:
+			app:
+				options:
+					index: "app.js"
 
 		watch:
 			coffee:
@@ -137,20 +145,31 @@ module.exports = (grunt) ->
 
 		less:
 			app:
+				options:
+					sourceMap: true
+					sourceMapFilename: "public/stylesheets/style.css.map"
+					sourceMapBasepath: "public/stylesheets"
 				files:
 					"public/stylesheets/style.css": "public/stylesheets/style.less"
 			ol:
+				options:
+					sourceMap: true
+					sourceMapFilename: "public/stylesheets/ol-style.css.map"
+					sourceMapBasepath: "public/stylesheets"
 				files:
 					"public/stylesheets/ol-style.css": "public/stylesheets/ol-style.less"
 
 		postcss:
 			options:
-				map: true,
+				map: 
+					prev: "public/stylesheets/"
+					inline: false
+					sourcesContent: true
 				processors: [
 					require('autoprefixer')({browsers: [ 'last 2 versions', 'ie >= 10' ]})
 				]
 			dist:
-				src: 'public/stylesheets/style.css'
+				src: [ "public/stylesheets/style.css", "public/stylesheets/ol-style.css" ]
 
 		env:
 			run:
@@ -244,8 +263,11 @@ module.exports = (grunt) ->
 				pattern: "@@RELEASE@@"
 				replacement: process.env.BUILD_NUMBER || "(unknown build)"
 
-
-			
+		shell:
+			fullAcceptanceTests:
+				command: "bash ./test/acceptance/scripts/full-test.sh"
+			dockerTests:
+				command: 'docker run -v "$(pwd):/app" --env SHARELATEX_ALLOW_PUBLIC_ACCESS=true --rm sharelatex/acceptance-test-runner'
 
 		availabletasks:
 			tasks:
@@ -381,7 +403,7 @@ module.exports = (grunt) ->
 	grunt.registerTask 'compile:server', 'Compile the server side coffee script', ['clean:app', 'coffee:app', 'coffee:app_dir', 'compile:modules:server']
 	grunt.registerTask 'compile:client', 'Compile the client side coffee script', ['coffee:client', 'coffee:sharejs', 'wrap_sharejs', "compile:modules:client", 'compile:modules:inject_clientside_includes']
 	grunt.registerTask 'compile:css', 'Compile the less files to css', ['less', 'postcss:dist']
-	grunt.registerTask 'compile:minify', 'Concat and minify the client side js', ['requirejs', "file_append", "exec:cssmin",]
+	grunt.registerTask 'compile:minify', 'Concat and minify the client side js and css', ['requirejs', "file_append", "exec:cssmin_sl", "exec:cssmin_ol"]
 	grunt.registerTask 'compile:unit_tests', 'Compile the unit tests', ['clean:unit_tests', 'coffee:unit_tests']
 	grunt.registerTask 'compile:acceptance_tests', 'Compile the acceptance tests', ['clean:acceptance_tests', 'coffee:acceptance_tests']
 	grunt.registerTask 'compile:smoke_tests', 'Compile the smoke tests', ['coffee:smoke_tests']
@@ -396,6 +418,18 @@ module.exports = (grunt) ->
 	grunt.registerTask 'test:acceptance', 'Run the acceptance tests (use --grep=<regex> or --feature=<feature> for individual tests)', ['compile:acceptance_tests', 'mochaTest:acceptance']
 	grunt.registerTask 'test:smoke', 'Run the smoke tests', ['compile:smoke_tests', 'mochaTest:smoke']
 	
+	grunt.registerTask(
+		'test:acceptance:full',
+		"Start server and run acceptance tests",
+		['shell:fullAcceptanceTests']
+	)
+
+	grunt.registerTask(
+		'test:acceptance:docker',
+		"Run acceptance tests inside docker container",
+		['compile:acceptance_tests', 'shell:dockerTests']
+	)
+	
 	grunt.registerTask 'test:modules:unit', 'Run the unit tests for the modules', ['compile:modules:server', 'compile:modules:unit_tests'].concat(moduleUnitTestTasks)
 
 	grunt.registerTask 'run:watch', "Compile and run the web-sharelatex server", ['compile', 'env:run', 'parallel']

services/web/Jenkinsfile

@@ -1,11 +1,6 @@
 pipeline {
   
-  agent {
-    docker {
-      image 'node:6.9.5'
-      args "-v /var/lib/jenkins/.npm:/tmp/.npm"
-    }
-  }
+  agent any
   
   environment  {
       HOME = "/tmp"
@@ -18,6 +13,12 @@ pipeline {
   
   stages {
     stage('Set up') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          reuseNode true
+        }
+      }
       steps {
         // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
         // which does not exist in the container's /etc/passwd file, causing the clone to fail.
@@ -40,15 +41,28 @@ pipeline {
         checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/learn-wiki'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@bitbucket.org:sharelatex/learn-wiki-web-module.git']]])
         checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/templates'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/templates-webmodule.git']]])
         checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/track-changes'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/track-changes-web-module.git']]])
+        checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/overleaf-integration'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/overleaf-integration-web-module.git']]])
+        checkout([$class: 'GitSCM', branches: [[name: '*/master']], extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'modules/overleaf-account-merge'], [$class: 'CloneOption', shallow: true]], userRemoteConfigs: [[credentialsId: 'GIT_DEPLOY_KEY', url: 'git@github.com:sharelatex/overleaf-account-merge.git']]])
       }
     }
     
     stage('Install') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          args "-v /var/lib/jenkins/.npm:/tmp/.npm"
+          reuseNode true
+        }
+      }
       steps {
+        sh 'git config --global core.logallrefupdates false'
         sh 'mv app/views/external/robots.txt public/robots.txt'
         sh 'mv app/views/external/googlebdb0f8f7f4a17241.html public/googlebdb0f8f7f4a17241.html'
         sh 'npm install'
         sh 'npm rebuild'
+        // It's too easy to end up shrinkwrapping to an outdated version of translations.
+        // Ensure translations are always latest, regardless of shrinkwrap
+        sh 'npm install git+https://github.com/sharelatex/translations-sharelatex.git#master'
         sh 'npm install --quiet grunt'
         sh 'npm install --quiet grunt-cli'
         sh 'ls -l node_modules/.bin'
@@ -56,29 +70,62 @@ pipeline {
     }
 
     stage('Compile') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          reuseNode true
+        }
+      }
       steps {
         sh 'node_modules/.bin/grunt compile  --verbose'
+        // replace the build number placeholder for sentry
+        sh 'node_modules/.bin/grunt version'
       }
     }
 
     stage('Smoke Test') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          reuseNode true
+        }
+      }
       steps {
         sh 'node_modules/.bin/grunt compile:smoke_tests'
       }
     }
 
     stage('Minify') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          reuseNode true
+        }
+      }
       steps {
         sh 'node_modules/.bin/grunt compile:minify'
       }
     }
     
     stage('Unit Test') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          reuseNode true
+        }
+      }
       steps {
         sh 'env NODE_ENV=development ./node_modules/.bin/grunt test:unit --reporter=tap'
       }
     }
     
+    stage('Acceptance Tests') {
+      steps {
+        sh 'docker pull sharelatex/acceptance-test-runner'
+        sh 'docker run --rm -v $(pwd):/app --env SHARELATEX_ALLOW_PUBLIC_ACCESS=true sharelatex/acceptance-test-runner'
+      }
+    }
+    
     stage('Package') {
       steps {
         sh 'rm -rf ./node_modules/grunt*'
@@ -87,6 +134,7 @@ pipeline {
         sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
       }
     }
+    
     stage('Publish') {
       steps {
         withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
@@ -96,6 +144,18 @@ pipeline {
         }
       }
     }
+    
+    
+    stage('Sync OSS') {
+      when {
+        branch 'master'
+      }
+      steps {
+        sshagent (credentials: ['GIT_DEPLOY_KEY']) {
+          sh 'git push git@github.com:sharelatex/web-sharelatex.git HEAD:master'
+        }
+      }
+    }
   }
   
   post {

services/web/app/coffee/Features/Collaborators/CollaboratorsHandler.coffee

@@ -249,3 +249,55 @@ module.exports = CollaboratorsHandler =
 			if err?
 				return callback(err)
 			callback(null, project?)
+
+	transferProjects: (from_user_id, to_user_id, callback=(err, projects) ->) ->
+		MEMBER_KEYS = ['collaberator_refs', 'readOnly_refs']
+
+		# Find all the projects this user is part of so we can flush them to TPDS
+		query =
+			$or:
+				[{ owner_ref: from_user_id }]
+				.concat(
+					MEMBER_KEYS.map (key) ->
+						q = {}
+						q[key] = from_user_id
+						return q
+				) # [{ collaberator_refs: from_user_id }, ...]
+		Project.find query, { _id: 1 }, (error, projects = []) ->
+			return callback(error) if error?
+
+			project_ids = projects.map (p) -> p._id
+			logger.log {project_ids, from_user_id, to_user_id}, "transferring projects"
+
+			update_jobs = []
+			update_jobs.push (cb) ->
+				Project.update { owner_ref: from_user_id }, { $set: { owner_ref: to_user_id }}, { multi: true }, cb
+			for key in MEMBER_KEYS
+				do (key) ->
+					update_jobs.push (cb) ->
+						query = {}
+						addNewUserUpdate = $addToSet: {}
+						removeOldUserUpdate = $pull: {}
+						query[key] = from_user_id
+						removeOldUserUpdate.$pull[key] = from_user_id
+						addNewUserUpdate.$addToSet[key] = to_user_id
+						# Mongo won't let us pull and addToSet in the same query, so do it in
+						# two. Note we need to add first, since the query is based on the old user.
+						Project.update query, addNewUserUpdate, { multi: true }, (error) ->
+							return cb(error) if error?
+							Project.update query, removeOldUserUpdate, { multi: true }, cb
+
+			# Flush each project to TPDS to add files to new user's Dropbox
+			ProjectEntityHandler = require("../Project/ProjectEntityHandler")
+			flush_jobs = []
+			for project_id in project_ids
+				do (project_id) ->
+					flush_jobs.push (cb) ->
+						ProjectEntityHandler.flushProjectToThirdPartyDataStore project_id, cb
+
+			# Flush in background, no need to block on this
+			async.series flush_jobs, (error) ->
+				if error?
+					logger.err {err: error, project_ids, from_user_id, to_user_id}, "error flushing tranferred projects to TPDS"
+
+			async.series update_jobs, callback

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -29,7 +29,11 @@ module.exports = ClsiManager =
 
 	sendRequestOnce: (project_id, user_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
 		ClsiManager._buildRequest project_id, options, (error, req) ->
-			return callback(error) if error?
+			if error?
+				if error.message is "no main file specified"
+					return callback(null, "validation-problems", null, null, {mainFile:error.message})
+				else
+					return callback(error)
 			logger.log project_id: project_id, "sending compile to CLSI"
 			ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)->
 				if err?
@@ -38,17 +42,17 @@ module.exports = ClsiManager =
 				if validationProblems?
 					logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted"
 					return callback(null, "validation-problems", null, null, validationProblems)
-			ClsiManager._postToClsi project_id, user_id, req, options.compileGroup, (error, response) ->
-				if error?
-					logger.err err:error, project_id:project_id, "error sending request to clsi"
-					return callback(error)
-				logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, compile_status: response?.compile?.status, "received compile response from CLSI"
-				ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
-					if err?
-						logger.err err:err, project_id:project_id, "error getting server id"
-						return callback(err)
-					outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
-					callback(null, response?.compile?.status, outputFiles, clsiServerId)
+				ClsiManager._postToClsi project_id, user_id, req, options.compileGroup, (error, response) ->
+					if error?
+						logger.err err:error, project_id:project_id, "error sending request to clsi"
+						return callback(error)
+					logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, compile_status: response?.compile?.status, "received compile response from CLSI"
+					ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
+						if err?
+							logger.err err:err, project_id:project_id, "error getting server id"
+							return callback(err)
+						outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
+						callback(null, response?.compile?.status, outputFiles, clsiServerId)
 
 	stopCompile: (project_id, user_id, options, callback = (error) ->) ->
 		compilerUrl = @_getCompilerUrl(options?.compileGroup, project_id, user_id, "compile/stop")
@@ -107,6 +111,8 @@ module.exports = ClsiManager =
 				callback null, compile:status:"project-too-large"
 			else if response.statusCode == 409
 				callback null, compile:status:"conflict"
+			else if response.statusCode == 423
+				callback null, compile:status:"compile-in-progress"
 			else
 				error = new Error("CLSI returned non-success code: #{response.statusCode}")
 				logger.error err: error, project_id: project_id, "CLSI returned failure code"
@@ -144,13 +150,8 @@ module.exports = ClsiManager =
 						logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state"
 					# see if we can send an incremental update to the CLSI
 					if docUpdaterDocs? and (options.syncType isnt "full") and not error?
-						# Workaround: for now, always flush project to mongo on compile
-						# until we have automatic periodic flushing on the docupdater
-						# side, to prevent documents staying in redis too long.
-						DocumentUpdaterHandler.flushProjectToMongo project_id, (error) ->
-							return callback(error) if error?
-							Metrics.inc "compile-from-redis"
-							ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback
+						Metrics.inc "compile-from-redis"
+						ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback
 					else
 						Metrics.inc "compile-from-mongo"
 						ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback
@@ -183,7 +184,7 @@ module.exports = ClsiManager =
 				# present in the docupdater. This allows finaliseRequest to
 				# identify the root doc.
 				possibleRootDocIds = [options.rootDoc_id, project.rootDoc_id]
-				for rootDoc_id in possibleRootDocIds when rootDoc_id?
+				for rootDoc_id in possibleRootDocIds when rootDoc_id? and rootDoc_id of docPath
 					path = docPath[rootDoc_id]
 					docs[path] ?= {_id: rootDoc_id, path: path}
 				ClsiManager._finaliseRequest project_id, options, project, docs, [], callback
@@ -209,9 +210,12 @@ module.exports = ClsiManager =
 		resources = []
 		rootResourcePath = null
 		rootResourcePathOverride = null
+		hasMainFile = false
+		numberOfDocsInProject = 0
 
 		for path, doc of docs
 			path = path.replace(/^\//, "") # Remove leading /
+			numberOfDocsInProject++
 			if doc.lines? # add doc to resources unless it is just a stub entry
 				resources.push
 					path:    path
@@ -220,11 +224,20 @@ module.exports = ClsiManager =
 				rootResourcePath = path
 			if options.rootDoc_id? and doc._id.toString() == options.rootDoc_id.toString()
 				rootResourcePathOverride = path
+			if path is "main.tex"
+				hasMainFile = true
 
 		rootResourcePath = rootResourcePathOverride if rootResourcePathOverride?
 		if !rootResourcePath?
-			logger.warn {project_id}, "no root document found, setting to main.tex"
-			rootResourcePath = "main.tex"
+			if hasMainFile
+				logger.warn {project_id}, "no root document found, setting to main.tex"
+				rootResourcePath = "main.tex"
+			else if numberOfDocsInProject is 1 # only one file, must be the main document
+				for path, doc of docs
+					rootResourcePath = path.replace(/^\//, "") # Remove leading /
+				logger.warn {project_id, rootResourcePath: rootResourcePath}, "no root document found, single document in project"
+			else
+				return callback new Error("no main file specified")
 
 		for path, file of files
 			path = path.replace(/^\//, "") # Remove leading /

services/web/app/coffee/Features/Compile/CompileManager.coffee

@@ -18,7 +18,7 @@ module.exports = CompileManager =
 			timer.done()
 			_callback(args...)
 
-		@_checkIfAutoCompileLimitHasBeenHit options.isAutoCompile, (err, canCompile)->
+		@_checkIfAutoCompileLimitHasBeenHit options.isAutoCompile, "everyone", (err, canCompile)->
 			if !canCompile
 				return callback null, "autocompile-backoff", []
 			logger.log project_id: project_id, user_id: user_id, "compiling project"
@@ -34,12 +34,16 @@ module.exports = CompileManager =
 						return callback(error) if error?
 						for key, value of limits
 							options[key] = value
-						# only pass user_id down to clsi if this is a per-user compile
-						compileAsUser = if Settings.disablePerUserCompiles then undefined else user_id
-						ClsiManager.sendRequest project_id, compileAsUser, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
-							return callback(error) if error?
-							logger.log files: outputFiles, "output files"
-							callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
+						# Put a lower limit on autocompiles for free users, based on compileGroup
+						CompileManager._checkCompileGroupAutoCompileLimit options.isAutoCompile, limits.compileGroup, (err, canCompile)->
+							if !canCompile
+								return callback null, "autocompile-backoff", []
+							# only pass user_id down to clsi if this is a per-user compile
+							compileAsUser = if Settings.disablePerUserCompiles then undefined else user_id
+							ClsiManager.sendRequest project_id, compileAsUser, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
+								return callback(error) if error?
+								logger.log files: outputFiles, "output files"
+								callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
 
 
 	stopCompile: (project_id, user_id, callback = (error) ->) ->
@@ -72,18 +76,30 @@ module.exports = CompileManager =
 			else
 				return callback null, true
 
-	_checkIfAutoCompileLimitHasBeenHit: (isAutoCompile, callback = (err, canCompile)->)->
+	_checkCompileGroupAutoCompileLimit: (isAutoCompile, compileGroup, callback = (err, canCompile)->)->
 		if !isAutoCompile
 			return callback(null, true)
-		opts = 
+		if compileGroup is "standard"
+			# apply extra limits to the standard compile group
+			CompileManager._checkIfAutoCompileLimitHasBeenHit isAutoCompile, compileGroup, callback
+		else
+			Metrics.inc "auto-compile-#{compileGroup}"
+			return callback(null, true)	# always allow priority group users to compile
+
+	_checkIfAutoCompileLimitHasBeenHit: (isAutoCompile, compileGroup, callback = (err, canCompile)->)->
+		if !isAutoCompile
+			return callback(null, true)
+		Metrics.inc "auto-compile-#{compileGroup}"
+		opts =
 			endpointName:"auto_compile"
 			timeInterval:20
-			subjectName:"everyone"
-			throttle: 25
+			subjectName:compileGroup
+			throttle: Settings?.rateLimit?.autoCompile?[compileGroup] || 25
 		rateLimiter.addCount opts, (err, canCompile)->
 			if err?
 				canCompile = false
-			logger.log canCompile:canCompile, opts:opts, "checking if auto compile limit has been hit"
+			if !canCompile
+				Metrics.inc "auto-compile-#{compileGroup}-limited"
 			callback err, canCompile
 
 	_ensureRootDocumentIsSet: (project_id, callback = (error) ->) ->

services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee

@@ -128,9 +128,9 @@ module.exports = DocumentUpdaterHandler =
 		# docs from redis via the docupdater. Otherwise we will need to
 		# fall back to getting them from mongo.
 		timer = new metrics.Timer("get-project-docs")
-		url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc?state=#{projectStateHash}"
+		url = "#{settings.apis.documentupdater.url}/project/#{project_id}/get_and_flush_if_old?state=#{projectStateHash}"
 		logger.log project_id:project_id, "getting project docs from document updater"
-		request.get url, (error, res, body)->
+		request.post url, (error, res, body)->
 			timer.done()
 			if error?
 				logger.error err:error, url:url, project_id:project_id, "error getting project docs from doc updater"

services/web/app/coffee/Features/Documents/DocumentController.coffee

@@ -7,7 +7,7 @@ module.exports =
 		doc_id = req.params.doc_id
 		plain = req?.query?.plain == 'true'
 		logger.log doc_id:doc_id, project_id:project_id, "receiving get document request from api (docupdater)"
-		ProjectEntityHandler.getDoc project_id, doc_id, (error, lines, rev, version, ranges) ->
+		ProjectEntityHandler.getDoc project_id, doc_id, {pathname: true}, (error, lines, rev, version, ranges, pathname) ->
 			if error?
 				logger.err err:error, doc_id:doc_id, project_id:project_id, "error finding element for getDocument"
 				return next(error)
@@ -20,6 +20,7 @@ module.exports =
 					lines: lines
 					version: version
 					ranges: ranges
+					pathname: pathname
 				}
 
 	setDocument: (req, res, next = (error) ->) ->
@@ -33,6 +34,3 @@ module.exports =
 				return next(error)
 			logger.log doc_id:doc_id, project_id:project_id, "finished receiving set document request from api (docupdater)"
 			res.sendStatus 200
-
-		
-		

services/web/app/coffee/Features/History/HistoryController.coffee

@@ -4,9 +4,33 @@ settings = require "settings-sharelatex"
 AuthenticationController = require "../Authentication/AuthenticationController"
 
 module.exports = HistoryController =
+	initializeProject: (callback = (error, history_id) ->) ->
+		return callback() if !settings.apis.project_history?.enabled
+		request.post {
+			url: "#{settings.apis.project_history.url}/project"
+		}, (error, res, body)->
+			return callback(error) if error?
+
+			if res.statusCode >= 200 and res.statusCode < 300
+				try
+					project = JSON.parse(body)
+				catch error
+					return callback(error)
+
+				overleaf_id = project?.project?.id
+				if !overleaf_id
+					error = new Error("project-history did not provide an id", project)
+					return callback(error)
+
+				callback null, { overleaf_id }
+			else
+				error = new Error("project-history returned a non-success status code: #{res.statusCode}")
+				callback error
+
 	proxyToHistoryApi: (req, res, next = (error) ->) ->
 		user_id = AuthenticationController.getLoggedInUserId req
-		url = settings.apis.trackchanges.url + req.url
+		url = HistoryController.buildHistoryServiceUrl() + req.url
+
 		logger.log url: url, "proxying to track-changes api"
 		getReq = request(
 			url: url
@@ -18,3 +42,9 @@ module.exports = HistoryController =
 		getReq.on "error", (error) ->
 			logger.error err: error, "track-changes API error"
 			next(error)
+
+	buildHistoryServiceUrl: () ->
+		if settings.apis.project_history?.enabled
+			return settings.apis.project_history.url
+		else
+			return settings.apis.trackchanges.url

services/web/app/coffee/Features/History/HistoryManager.coffee

@@ -1,28 +0,0 @@
-settings = require "settings-sharelatex"
-request = require "request"
-logger = require "logger-sharelatex"
-
-module.exports = HistoryManager =
-	flushProject: (project_id, callback = (error) ->) ->
-		logger.log project_id: project_id, "flushing project in track-changes api"
-		url = "#{settings.apis.trackchanges.url}/project/#{project_id}/flush"
-		request.post url, (error, res, body) ->
-			return callback(error) if error?
-			if 200 <= res.statusCode < 300
-				callback(null)
-			else
-				error = new Error("track-changes api responded with non-success code: #{res.statusCode} #{url}")
-				logger.error err: error, project_id: project_id, "error flushing project in track-changes api"
-				callback(error)
-
-	archiveProject: (project_id, callback = ()->)->
-		logger.log project_id: project_id, "archving project in track-changes api"
-		url = "#{settings.apis.trackchanges.url}/project/#{project_id}/archive"
-		request.post url, (error, res, body) ->
-			return callback(error) if error?
-			if 200 <= res.statusCode < 300
-				callback(null)
-			else
-				error = new Error("track-changes api responded with non-success code: #{res.statusCode} #{url}")
-				logger.error err: error, project_id: project_id, "error archving project in track-changes api"
-				callback(error)

services/web/app/coffee/Features/Project/ProjectController.coffee

@@ -229,6 +229,7 @@ module.exports = ProjectController =
 					else if event?
 						return cb(null, false)
 					else
+						logger.log { user_id, event }, "track changes onboarding not shown yet to this user"
 						return cb(null, true)
 			showPerUserTCNotice: (cb) ->
 				cb = underscore.once(cb)
@@ -247,12 +248,40 @@ module.exports = ProjectController =
 					else if event?
 						return cb(null, false)
 					else
+						logger.log { user_id, event }, "per user track changes notice not shown yet to this user"
 						return cb(null, true)
 			isTokenMember: (cb) ->
 				cb = underscore.once(cb)
 				if !user_id?
 					return cb()
 				CollaboratorsHandler.userIsTokenMember user_id, project_id, cb
+			showAutoCompileOnboarding: (cb) ->
+				cb = underscore.once(cb)
+				if !user_id?
+					return cb()
+
+				# Extract data from user's ObjectId
+				timestamp = parseInt(user_id.toString().substring(0, 8), 16)
+				counter = parseInt(user_id.toString().substring(18, 24), 16)
+
+				rolloutPercentage = 40 # Percentage of users to roll out to
+				if counter % 100 > rolloutPercentage
+					# Don't show if user is not part of roll out
+					return cb(null, { enabled: false, showOnboarding: false })
+				userSignupDate = new Date(timestamp * 1000)
+				if userSignupDate > new Date("2017-10-16")
+					# Don't show for users who registered after it was released
+					return cb(null, { enabled: true, showOnboarding: false })
+				timeout = setTimeout cb, 500
+				AnalyticsManager.getLastOccurance user_id, "shown-autocompile-onboarding", (error, event) ->
+					clearTimeout timeout
+					if error?
+						return cb(null, { enabled: true, showOnboarding: false })
+					else if event?
+						return cb(null, { enabled: true, showOnboarding: false })
+					else
+						logger.log { user_id, event }, "autocompile onboarding not shown yet to this user"
+						return cb(null, { enabled: true, showOnboarding: true })
 		}, (err, results)->
 			if err?
 				logger.err err:err, "error getting details for project page"
@@ -260,9 +289,9 @@ module.exports = ProjectController =
 			project = results.project
 			user = results.user
 			subscription = results.subscription
-			{ showTrackChangesOnboarding, showPerUserTCNotice } = results
+			{ showTrackChangesOnboarding, showPerUserTCNotice, showAutoCompileOnboarding } = results
 
-			daysSinceLastUpdated =  (new Date() - project.lastUpdated) /86400000
+			daysSinceLastUpdated =  (new Date() - project.lastUpdated) / 86400000
 			logger.log project_id:project_id, daysSinceLastUpdated:daysSinceLastUpdated, "got db results for loading editor"
 
 			token = TokenAccessHandler.getRequestToken(req, project_id)
@@ -307,6 +336,8 @@ module.exports = ProjectController =
 					trackChangesState: project.track_changes
 					showTrackChangesOnboarding: !!showTrackChangesOnboarding
 					showPerUserTCNotice: !!showPerUserTCNotice
+					autoCompileEnabled: !!showAutoCompileOnboarding?.enabled
+					showAutoCompileOnboarding: !!showAutoCompileOnboarding?.showOnboarding
 					privilegeLevel: privilegeLevel
 					chatUrl: Settings.apis.chat.url
 					anonymous: anonymous

services/web/app/coffee/Features/Project/ProjectCreationHandler.coffee

@@ -2,11 +2,12 @@ logger = require('logger-sharelatex')
 async = require("async")
 metrics = require('metrics-sharelatex')
 Settings = require('settings-sharelatex')
-ObjectId = require('mongoose').Types.ObjectId	
+ObjectId = require('mongoose').Types.ObjectId
 Project = require('../../models/Project').Project
 Folder = require('../../models/Folder').Folder
 ProjectEntityHandler = require('./ProjectEntityHandler')
 ProjectDetailsHandler = require('./ProjectDetailsHandler')
+HistoryController = require('../History/HistoryController')
 User = require('../../models/User').User
 fs = require('fs')
 Path = require "path"
@@ -14,23 +15,36 @@ _ = require "underscore"
 
 module.exports = ProjectCreationHandler =
 
-	createBlankProject : (owner_id, projectName, callback = (error, project) ->)->
+	createBlankProject : (owner_id, projectName, projectHistoryId, callback = (error, project) ->)->
 		metrics.inc("project-creation")
+		if arguments.length == 3
+			callback = projectHistoryId
+			projectHistoryId = null
+
 		ProjectDetailsHandler.validateProjectName projectName, (error) ->
 			return callback(error) if error?
 			logger.log owner_id:owner_id, projectName:projectName, "creating blank project"
-			rootFolder = new Folder {'name':'rootFolder'}
-			project = new Project
-				 owner_ref  : new ObjectId(owner_id)
-				 name       : projectName
-			if Settings.currentImageName?
-				project.imageName = Settings.currentImageName
-			project.rootFolder[0] = rootFolder
-			User.findById owner_id, "ace.spellCheckLanguage", (err, user)->
-				project.spellCheckLanguage = user.ace.spellCheckLanguage
-				project.save (err)->
-					return callback(err) if err?
-					callback err, project
+			if projectHistoryId?
+				ProjectCreationHandler._createBlankProject owner_id, projectName, projectHistoryId, callback
+			else
+				HistoryController.initializeProject (error, history) ->
+					return callback(error) if error?
+					ProjectCreationHandler._createBlankProject owner_id, projectName, history?.overleaf_id, callback
+
+	_createBlankProject : (owner_id, projectName, projectHistoryId, callback = (error, project) ->)->
+		rootFolder = new Folder {'name':'rootFolder'}
+		project = new Project
+			 owner_ref          : new ObjectId(owner_id)
+			 name               : projectName
+		project.overleaf.history.id = projectHistoryId
+		if Settings.currentImageName?
+			project.imageName = Settings.currentImageName
+		project.rootFolder[0] = rootFolder
+		User.findById owner_id, "ace.spellCheckLanguage", (err, user)->
+			project.spellCheckLanguage = user.ace.spellCheckLanguage
+			project.save (err)->
+				return callback(err) if err?
+				callback err, project
 
 	createBasicProject :  (owner_id, projectName, callback = (error, project) ->)->
 		self = @

services/web/app/coffee/Features/Project/ProjectDetailsHandler.coffee

@@ -10,7 +10,7 @@ ProjectTokenGenerator = require('./ProjectTokenGenerator')
 
 module.exports = ProjectDetailsHandler =
 	getDetails: (project_id, callback)->
-		ProjectGetter.getProject project_id, {name:true, description:true, compiler:true, features:true, owner_ref:true}, (err, project)->
+		ProjectGetter.getProject project_id, {name:true, description:true, compiler:true, features:true, owner_ref:true, overleaf:true}, (err, project)->
 			if err?
 				logger.err err:err, project_id:project_id, "error getting project"
 				return callback(err)
@@ -22,7 +22,11 @@ module.exports = ProjectDetailsHandler =
 					description: project.description
 					compiler: project.compiler
 					features: user.features
-				logger.log project_id:project_id, details:details, "getting project details"
+
+				if project.overleaf?
+					details.overleaf = project.overleaf
+
+				logger.log project_id:project_id, details: details, "getting project details"
 				callback(err, details)
 
 	getProjectDescription: (project_id, callback)->
@@ -54,7 +58,7 @@ module.exports = ProjectDetailsHandler =
 
 	MAX_PROJECT_NAME_LENGTH: 150
 	validateProjectName: (name, callback = (error) ->) ->
-		if name.length == 0
+		if !name? or name.length == 0
 			return callback(new Errors.InvalidNameError("Project name cannot be blank"))
 		else if name.length > @MAX_PROJECT_NAME_LENGTH
 			return callback(new Errors.InvalidNameError("Project name is too long"))
@@ -97,4 +101,3 @@ module.exports = ProjectDetailsHandler =
 				Project.update {_id: project_id}, {$set: {tokens: tokens}}, (err) ->
 					return callback(err) if err?
 					callback(null, tokens)
-

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -131,7 +131,7 @@ module.exports = ProjectEntityHandler =
 	setRootDoc: (project_id, newRootDocID, callback = (error) ->)->
 		logger.log project_id: project_id, rootDocId: newRootDocID, "setting root doc"
 		Project.update {_id:project_id}, {rootDoc_id:newRootDocID}, {}, callback
-	
+
 	unsetRootDoc: (project_id, callback = (error) ->) ->
 		logger.log project_id: project_id, "removing root doc"
 		Project.update {_id:project_id}, {$unset: {rootDoc_id: true}}, {}, callback
@@ -140,8 +140,15 @@ module.exports = ProjectEntityHandler =
 		if typeof(options) == "function"
 			callback = options
 			options = {}
-		DocstoreManager.getDoc project_id, doc_id, options, callback
 
+		if options["pathname"]
+			delete options["pathname"]
+			projectLocator.findElement {project_id: project_id, element_id: doc_id, type: 'doc'}, (error, doc, path) =>
+				return callback(error) if error?
+				DocstoreManager.getDoc project_id, doc_id, options, (error, lines, rev, version, ranges) =>
+					callback(error, lines, rev, version, ranges, path.fileSystem)
+		else
+			DocstoreManager.getDoc project_id, doc_id, options, callback
 
 	addDoc: (project_id, folder_id, docName, docLines, callback = (error, doc, folder_id) ->)=>
 		ProjectGetter.getProjectWithOnlyFolders project_id, (err, project) ->
@@ -158,7 +165,7 @@ module.exports = ProjectEntityHandler =
 			# Put doc in docstore first, so that if it errors, we don't have a doc_id in the project
 			# which hasn't been created in docstore.
 			DocstoreManager.updateDoc project_id.toString(), doc._id.toString(), docLines, 0, {}, (err, modified, rev) ->
-				return callback(err) if err? 
+				return callback(err) if err?
 
 				ProjectEntityHandler._putElement project, folder_id, doc, "doc", (err, result)=>
 					return callback(err) if err?
@@ -207,7 +214,7 @@ module.exports = ProjectEntityHandler =
 	replaceFile: (project_id, file_id, fsPath, callback)->
 		ProjectGetter.getProject project_id, {name:true}, (err, project) ->
 			return callback(err) if err?
-			findOpts = 
+			findOpts =
 				project_id:project._id
 				element_id:file_id
 				type:"file"
@@ -280,7 +287,7 @@ module.exports = ProjectEntityHandler =
 			procesFolder = (previousFolders, folderName, callback)=>
 				previousFolders = previousFolders || []
 				parentFolder = previousFolders[previousFolders.length-1]
-				if parentFolder?  
+				if parentFolder?
 					parentFolder_id = parentFolder._id
 				builtUpPath = "#{builtUpPath}/#{folderName}"
 				projectLocator.findElementByPath project, builtUpPath, (err, foundFolder)=>
@@ -360,7 +367,7 @@ module.exports = ProjectEntityHandler =
 			return callback(err) if err?
 			projectLocator.findElement {project:project, element_id:entity_id, type:entityType}, (err, entity, path)->
 				return callback(err) if err?
-				
+
 				if entityType.match(/folder/)
 					ensureFolderIsNotMovedIntoChild = (callback = (error) ->) ->
 						projectLocator.findElement {project: project, element_id: folder_id, type:"folder"}, (err, destEntity, destPath) ->
@@ -372,7 +379,7 @@ module.exports = ProjectEntityHandler =
 								callback()
 				else
 					ensureFolderIsNotMovedIntoChild = (callback = () ->) -> callback()
-					
+
 				ensureFolderIsNotMovedIntoChild (error) ->
 					return callback(error) if error?
 					self._removeElementFromMongoArray Project, project_id, path.mongo, (err)->
@@ -382,7 +389,7 @@ module.exports = ProjectEntityHandler =
 							return callback(err) if err?
 							ProjectEntityHandler._putElement project, destinationFolder_id, entity, entityType, (err, result)->
 								return callback(err) if err?
-								opts = 
+								opts =
 									project_id:project_id
 									project_name:project.name
 									startPath:path.fileSystem
@@ -506,7 +513,7 @@ module.exports = ProjectEntityHandler =
 
 
 	_countElements : (project, callback)->
-	
+
 		countFolder = (folder, cb = (err, count)->)->
 
 			jobs = _.map folder?.folders, (folder)->

services/web/app/coffee/Features/StaticPages/StaticPagesRouter.coffee

@@ -14,6 +14,7 @@ module.exports =
 		webRouter.get '/privacy_policy', HomeController.externalPage("privacy", "Privacy Policy")
 		webRouter.get '/planned_maintenance', HomeController.externalPage("planned_maintenance", "Planned Maintenance")
 		webRouter.get '/style', HomeController.externalPage("style_guide", "Style Guide")
+		webRouter.get '/ol-style', HomeController.externalPage("ol_style_guide", "Overleaf Style Guide")
 		webRouter.get '/jobs', HomeController.externalPage("jobs", "Jobs")
 
 		webRouter.get '/track-changes-and-comments-in-latex', HomeController.externalPage("review-features-page", "Review features")

services/web/app/coffee/Features/Uploads/ArchiveManager.coffee

@@ -1,56 +1,105 @@
-child   = require "child_process"
 logger  = require "logger-sharelatex"
 metrics = require "metrics-sharelatex"
 fs      = require "fs"
 Path    = require "path"
+fse     = require "fs-extra"
+yauzl   = require "yauzl"
+Settings = require "settings-sharelatex"
 _ = require("underscore")
 
 ONE_MEG = 1024 * 1024
 
 module.exports = ArchiveManager =
 
-
 	_isZipTooLarge: (source, callback = (err, isTooLarge)->)->
 		callback = _.once callback
 
-		unzip = child.spawn("unzip", ["-l", source])
-
-		output = ""
-		unzip.stdout.on "data", (d)->
-			output += d
-
-		error = null
-		unzip.stderr.on "data", (chunk) ->
-			error ||= ""
-			error += chunk
-
-		unzip.on "error", (err) ->
-			logger.error {err, source}, "unzip failed"
-			if err.code == "ENOENT"
-				logger.error "unzip command not found. Please check the unzip command is installed"
-			callback(err)
-
-		unzip.on "close", (exitCode) ->
-			if error?
-				error = new Error(error)
-				logger.warn err:error, source: source, "error checking zip size"
-
-			lines = output.split("\n")
-			lastLine = lines[lines.length - 2]?.trim()
-			totalSizeInBytes = lastLine?.split(" ")?[0]
+		totalSizeInBytes = null
+		yauzl.open source, {lazyEntries: true}, (err, zipfile) ->
+			return callback(err) if err?
+
+			if Settings.maxEntitiesPerProject? and zipfile.entryCount > Settings.maxEntitiesPerProject
+				return callback(null, true) # too many files in zip file
+
+			zipfile.on "error", callback
+
+			# read all the entries
+			zipfile.readEntry()
+			zipfile.on "entry", (entry) ->
+				totalSizeInBytes += entry.uncompressedSize
+				zipfile.readEntry() # get the next entry
+
+			# no more entries to read
+			zipfile.on "end", () ->
+				if !totalSizeInBytes? or isNaN(totalSizeInBytes)
+					logger.err source:source, totalSizeInBytes:totalSizeInBytes, "error getting bytes of zip"
+					return callback(new Error("error getting bytes of zip"))
+				isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
+				callback(null, isTooLarge)
+
+	_checkFilePath: (entry, destination, callback = (err, destFile) ->) ->
+		# check if the entry is a directory
+		endsWithSlash = /\/$/
+		if endsWithSlash.test(entry.fileName)
+			return callback() # don't give a destfile for directory
+		# check that the file does not use a relative path
+		for dir in entry.fileName.split('/')
+			if dir == '..'
+				return callback(new Error("relative path"))
+		# check that the destination file path is normalized
+		dest = "#{destination}/#{entry.fileName}"
+		if dest != Path.normalize(dest)
+			return callback(new Error("unnormalized path"))
+		else
+			return callback(null, dest)
+
+	_writeFileEntry: (zipfile, entry, destFile, callback = (err)->) ->
+		callback = _.once callback
 
-			totalSizeInBytesAsInt = parseInt(totalSizeInBytes)
+		zipfile.openReadStream entry, (err, readStream) ->
+			return callback(err) if err?
+			readStream.on "error", callback
+			readStream.on "end", callback
 
-			if !totalSizeInBytesAsInt? or isNaN(totalSizeInBytesAsInt)
-				logger.err source:source, totalSizeInBytes:totalSizeInBytes, totalSizeInBytesAsInt:totalSizeInBytesAsInt, lastLine:lastLine, exitCode:exitCode, "error getting bytes of zip"
-				return callback(new Error("error getting bytes of zip"))
+			errorHandler = (err) -> # clean up before calling callback
+				readStream.unpipe()
+				readStream.destroy()
+				callback(err)
 
-			isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
+			fse.ensureDir Path.dirname(destFile), (err) ->
+				return errorHandler(err) if err?
+				writeStream = fs.createWriteStream destFile
+				writeStream.on 'error', errorHandler
+				readStream.pipe(writeStream)
 
-			callback(error, isTooLarge)
+	_extractZipFiles: (source, destination, callback = (err) ->) ->
+		callback = _.once callback
 
+		yauzl.open source, {lazyEntries: true}, (err, zipfile) ->
+			return callback(err) if err?
+			zipfile.on "error", callback
+			# read all the entries
+			zipfile.readEntry()
+			zipfile.on "entry", (entry) ->
+				logger.log {source:source, fileName: entry.fileName}, "processing zip file entry"
+				ArchiveManager._checkFilePath entry, destination, (err, destFile) ->
+					if err?
+						logger.warn err:err, source:source, destination:destination, "skipping bad file path"
+						zipfile.readEntry() # bad path, just skip to the next file
+						return
+					if destFile? # only write files
+						ArchiveManager._writeFileEntry zipfile, entry, destFile, (err) ->
+							if err?
+								logger.error err:err, source:source, destFile:destFile, "error unzipping file entry"
+								zipfile.close() # bail out, stop reading file entries
+								return callback(err)
+							else
+								zipfile.readEntry() # continue to the next file
+					else # if it's a directory, continue
+						zipfile.readEntry()
+			# no more entries to read
+			zipfile.on "end", callback
 
-					
 	extractZipArchive: (source, destination, _callback = (err) ->) ->
 		callback = (args...) ->
 			_callback(args...)
@@ -62,36 +111,19 @@ module.exports = ArchiveManager =
 				return callback(err)
 
 			if isTooLarge
-				return callback(new Error("zip_too_large"))		
-
+				return callback(new Error("zip_too_large"))
 
 			timer = new metrics.Timer("unzipDirectory")
 			logger.log source: source, destination: destination, "unzipping file"
 
-			unzip = child.spawn("unzip", [source, "-d", destination])
-
-			# don't remove this line, some zips need
-			# us to listen on this for some unknow reason
-			unzip.stdout.on "data", (d)->
-
-			error = null
-			unzip.stderr.on "data", (chunk) ->
-				error ||= ""
-				error += chunk
-
-			unzip.on "error", (err) ->
-				logger.error {err, source, destination}, "unzip failed"
-				if err.code == "ENOENT"
-					logger.error "unzip command not found. Please check the unzip command is installed"
-				callback(err)
-
-			unzip.on "close", () ->
+			ArchiveManager._extractZipFiles source, destination, (err) ->
 				timer.done()
-				if error?
-					error = new Error(error)
-					logger.error err:error, source: source, destination: destination, "error unzipping file"
-				callback(error)
-	
+				if err?
+					logger.error {err, source, destination}, "unzip failed"
+					callback(err)
+				else
+					callback()
+
 	findTopLevelDirectory: (directory, callback = (error, topLevelDir) ->) ->
 		fs.readdir directory, (error, files) ->
 			return callback(error) if error?

services/web/app/coffee/Features/Uploads/FileTypeManager.coffee

@@ -18,6 +18,8 @@ module.exports = FileTypeManager =
 
 	IGNORE_FILENAMES : [
 		"__MACOSX"
+		".git"
+		".gitignore"
 	]
 	
 	MAX_TEXT_FILE_SIZE: 1 * 1024 * 1024 # 1 MB

services/web/app/coffee/Features/User/UserInfoController.coffee

@@ -4,6 +4,7 @@ UserDeleter = require("./UserDeleter")
 UserUpdater = require("./UserUpdater")
 sanitize = require('sanitizer')
 AuthenticationController = require('../Authentication/AuthenticationController')
+ObjectId = require("mongojs").ObjectId
 
 module.exports = UserController =
 	getLoggedInUsersPersonalInfo: (req, res, next = (error) ->) ->
@@ -19,8 +20,17 @@ module.exports = UserController =
 			UserController.sendFormattedPersonalInfo(user, res, next)
 
 	getPersonalInfo: (req, res, next = (error) ->) ->
-		UserGetter.getUser req.params.user_id, { _id: true, first_name: true, last_name: true, email: true}, (error, user) ->
-			logger.log user_id: req.params.user_id, "reciving request for getting users personal info"
+		{user_id} = req.params
+
+		if user_id.match(/^\d+$/)
+			query = { "overleaf.id": parseInt(user_id, 10) }
+		else if user_id.match(/^[a-f0-9]{24}$/)
+			query = { _id: ObjectId(user_id) }
+		else
+			return res.send(400)
+
+		UserGetter.getUser query, { _id: true, first_name: true, last_name: true, email: true}, (error, user) ->
+			logger.log user_id: req.params.user_id, "receiving request for getting users personal info"
 			return next(error) if error?
 			return res.send(404) if !user?
 			UserController.sendFormattedPersonalInfo(user, res, next)

services/web/app/coffee/models/Project.coffee

@@ -42,6 +42,8 @@ ProjectSchema = new Schema
 		imported_at_ver_id : { type: Number }
 		token           : { type: String }
 		read_token      : { type: String }
+		history         :
+			id            : { type: Number }
 
 ProjectSchema.statics.getProject = (project_or_id, fields, callback)->
 	if project_or_id._id?

services/web/app/views/layout/footer.pug

@@ -1,7 +1,7 @@
 
 
 footer.site-footer
-	.container
+	.site-footer-content
 		.row
 			ul.col-md-9
 				
@@ -38,6 +38,6 @@ footer.site-footer
 				each item in nav.right_footer
 					li
 						if item.url
-							a(href=item.url, class=item.class) !{item.text}
+							a(href=item.url, class=item.class, aria-label=item.label) !{item.text}
 						else
 							| !{item.text}

services/web/app/views/layout/navbar.pug

@@ -4,11 +4,11 @@ nav.navbar.navbar-default
 			button.navbar-toggle(ng-init="navCollapsed = true", ng-click="navCollapsed = !navCollapsed", ng-class="{active: !navCollapsed}")
 				i.fa.fa-bars
 			if settings.nav.custom_logo
-				a(href='/', style='background-image:url("'+settings.nav.custom_logo+'")').navbar-brand
+				a(href='/', aria-label=settings.appName, style='background-image:url("'+settings.nav.custom_logo+'")').navbar-brand
 			else if (nav.title)
-				a(href='/').navbar-title #{nav.title}
+				a(href='/', aria-label=settings.appName).navbar-title #{nav.title}
 			else
-				a(href='/').navbar-brand
+				a(href='/', aria-label=settings.appName).navbar-brand
 
 		.navbar-collapse.collapse(collapse="navCollapsed")
 

services/web/app/views/project/editor.pug

@@ -124,6 +124,8 @@ block requirejs
 		window.trackChangesState = data.trackChangesState;
 		window.showTrackChangesOnboarding = #{!!showTrackChangesOnboarding};
 		window.showPerUserTCNotice = #{!!showPerUserTCNotice};
+		window.autoCompileEnabled = #{!!autoCompileEnabled};
+		window.showAutoCompileOnboarding = #{!!showAutoCompileOnboarding}
 		window.wikiEnabled = #{!!(settings.apis.wiki && settings.apis.wiki.url)};
 		window.enableTokenAccessUI = #{enableTokenAccessUI}
 		window.requirejs = {

services/web/app/views/project/editor/editor.pug

@@ -94,4 +94,17 @@ div.full-size(
 	ng-show="ui.view == 'pdf'"
 )
 	include ./pdf
-	
+
+#onboarding-autocompile.onboarding-autocompile.popover(
+	ng-controller="AutoCompileOnboardingController"
+	ng-if="onboarding.autoCompile == 'show'"
+	ng-class="placement"
+)
+	.popover-inner
+		h3.popover-title #{translate("auto_compile")}
+		.popover-content
+			p #{translate("try_out_auto_compile_setting")}
+			img(src="/img/onboarding/autocompile/setting-dropdown.png" width="100%")
+			p #{translate("auto_compile_onboarding_description")}
+			button.btn.btn-default.btn-block(ng-click="dismiss()")
+				| #{translate("got_it")}

services/web/app/views/project/editor/feature-onboarding.pug

@@ -33,7 +33,7 @@
 					loop
 				)
 					source(ng-src="{{ '/img/onboarding/review-panel/open-review.mp4' }}", type="video/mp4")
-					img(src="/img/onboarding/review-panel/open-review.gif")
+					img(ng-src="{{ '/img/onboarding/review-panel/open-review.gif' }}", alt="Open review panel demo")
 			div(ng-show="onboarding.innerStep === 2;")
 				video.feat-onboard-video(
 					video-play-state="onboarding.innerStep === 2;"
@@ -41,7 +41,7 @@
 					loop
 				)
 					source(ng-src="{{ '/img/onboarding/review-panel/commenting.mp4' }}", type="video/mp4")
-					img(src="/img/onboarding/review-panel/commenting.gif")
+					img(ng-src="{{ '/img/onboarding/review-panel/commenting.gif' }}", alt="Commenting demo")
 			div(ng-show="onboarding.innerStep === 3;")
 				video.feat-onboard-video(
 					video-play-state="onboarding.innerStep === 3;"
@@ -49,7 +49,7 @@
 					loop
 				)
 					source(ng-src="{{ '/img/onboarding/review-panel/add-changes.mp4' }}", type="video/mp4")
-					img(src="/img/onboarding/review-panel/add-changes.gif")
+					img(ng-src="{{ '/img/onboarding/review-panel/add-changes.gif' }}", alt="Add changes demo")
 			div(ng-show="onboarding.innerStep === 4;")
 				video.feat-onboard-video(
 					video-play-state="onboarding.innerStep === 4;"
@@ -57,7 +57,7 @@
 					loop
 				)
 					source(ng-src="{{ '/img/onboarding/review-panel/accept-changes.mp4' }}", type="video/mp4")
-					img(src="/img/onboarding/review-panel/accept-changes.gif")
+					img(ng-src="{{ '/img/onboarding/review-panel/accept-changes.gif' }}", alt="Accept changes demo")
 			button.btn.btn-primary.feat-onboard-nav-btn(
 				ng-click="gotoNextStep();"
 				ng-disabled="onboarding.innerStep === onboarding.nSteps;")

services/web/app/views/project/editor/pdf.pug

@@ -1,6 +1,6 @@
 div.full-size.pdf(ng-controller="PdfController")
 	.toolbar.toolbar-tall
-		.btn-group(
+		.btn-group#recompile(
 			dropdown,
 			tooltip-html="'"+translate('recompile_pdf')+" <span class=\"keyboard-shortcut\">({{modifierKey}} + Enter)</span>'"
 			tooltip-class="keyboard-tooltip"
@@ -26,6 +26,17 @@ div.full-size.pdf(ng-controller="PdfController")
 			)
 				span.caret
 			ul.dropdown-menu.dropdown-menu-left
+				// Only show if on beta program or part of rollout
+				if user.betaProgram || autoCompileEnabled
+					li.dropdown-header #{translate("auto_compile")}
+					li
+						a(href, ng-click="autocompile_enabled = true")
+							i.fa.fa-fw(ng-class="{'fa-check': autocompile_enabled}")
+							| &nbsp;#{translate('on')}
+					li
+						a(href, ng-click="autocompile_enabled = false")
+							i.fa.fa-fw(ng-class="{'fa-check': !autocompile_enabled}")
+							| &nbsp;#{translate('off')}
 				li.dropdown-header #{translate("compile_mode")}
 				li
 					a(href, ng-click="draft = false")
@@ -306,6 +317,9 @@ div.full-size.pdf(ng-controller="PdfController")
 				div
 					li(ng-repeat="entry in pdf.validation.conflictedPaths") {{ '/'+entry['path'] }}
 
+			.alert.alert-danger(ng-show="pdf.validation.mainFile")
+				strong #{translate("main_file_not_found")} 
+				span #{translate("please_set_main_file")}
 
 		.pdf-errors(ng-switch-when="errors")
 	
@@ -333,6 +347,10 @@ div.full-size.pdf(ng-controller="PdfController")
 				strong #{translate("pdf_compile_rate_limit_hit")} 
 				span #{translate("project_flagged_too_many_compiles")}
 
+			.alert.alert-danger(ng-show="pdf.compileInProgress")
+				strong #{translate("pdf_compile_in_progress_error")}. 
+				span #{translate("pdf_compile_try_again")}
+
 			.alert.alert-danger(ng-show="pdf.timedout")
 				p
 					strong #{translate("timedout")}. 
@@ -376,6 +394,12 @@ div.full-size.pdf(ng-controller="PdfController")
 						ng-click="startFreeTrial('compile-timeout')"
 					) #{translate("start_free_trial")}
 
+
+			.alert.alert-danger(ng-show="pdf.autoCompileDisabled")
+				p
+					strong #{translate("autocompile_disabled")}. 
+					span #{translate("autocompile_disabled_reason")}
+
 			.alert.alert-danger(ng-show="pdf.projectTooLarge")
 				strong #{translate("project_too_large")} 
 				span #{translate("project_too_large_please_reduce")}

services/web/app/views/project/list.pug

@@ -15,58 +15,58 @@ block content
 			}
 		};
 
-	.content.content-alt.project-list-page(ng-controller="ProjectPageController")
-		.container
-			.announcements(
-				ng-controller="AnnouncementsController"
-				ng-class="{ 'announcements-open': ui.isOpen }"
-				ng-cloak
-			)
-				.announcements-backdrop(
-					ng-if="ui.isOpen"
-					ng-click="toggleAnnouncementsUI();"
-				)
-				a.announcements-btn(
-					href
-					ng-if="announcements.length"
-					ng-click="toggleAnnouncementsUI();"
-					ng-class="{ 'announcements-btn-open': ui.isOpen, 'announcements-btn-has-new': ui.newItems }"
+	.announcements(
+		ng-controller="AnnouncementsController"
+		ng-class="{ 'announcements-open': ui.isOpen }"
+		ng-cloak
+	)
+		.announcements-backdrop(
+			ng-if="ui.isOpen"
+			ng-click="toggleAnnouncementsUI();"
+		)
+		a.announcements-btn(
+			href
+			ng-if="announcements.length"
+			ng-click="toggleAnnouncementsUI();"
+			ng-class="{ 'announcements-btn-open': ui.isOpen, 'announcements-btn-has-new': ui.newItems }"
+		)
+			span.announcements-badge(ng-if="ui.newItems") {{ ui.newItems }}
+		.announcements-body(
+			ng-if="ui.isOpen"
+		)
+			.announcements-scroller
+				.announcement(
+					ng-repeat="announcement in announcements | filter:(ui.newItems ? { read: false } : '') track by announcement.id"
 				)
-					span.announcements-badge(ng-if="ui.newItems") {{ ui.newItems }}
-				.announcements-body(
-					ng-if="ui.isOpen"
+					h2.announcement-header {{ announcement.title }}
+					p.announcement-description(ng-bind-html="announcement.excerpt")
+					.announcement-meta
+						p.announcement-date {{ announcement.date | date:"longDate" }}
+						a.announcement-link(
+							ng-href="{{ announcement.url }}"
+							ng-click="logAnnouncementClick()",
+							target="_blank"
+						) Read more
+				div.text-center(
+					ng-if="ui.newItems > 0 && ui.newItems < announcements.length"
 				)
-					.announcements-scroller
-						.announcement(
-							ng-repeat="announcement in announcements | filter:(ui.newItems ? { read: false } : '') track by announcement.id"
-						)
-							h2.announcement-header {{ announcement.title }}
-							p.announcement-description(ng-bind-html="announcement.excerpt")
-							.announcement-meta
-								p.announcement-date {{ announcement.date | date:"longDate" }}
-								a.announcement-link(
-									ng-href="{{ announcement.url }}"
-									ng-click="logAnnouncementClick()",
-									target="_blank"
-								) Read more
-						div.text-center(
-							ng-if="ui.newItems > 0 && ui.newItems < announcements.length"
-						)
-							a.btn.btn-default.btn-sm(
-								href
-								ng-click="showAll();"
-							) Show all
+					a.btn.btn-default.btn-sm(
+						href
+						ng-click="showAll();"
+					) Show all
+	.content.content-alt.project-list-page(ng-controller="ProjectPageController")
+		.project-list-content
 
-			.row(ng-cloak)
-				span(ng-if="projects.length > 0")
-					aside.col-md-2.col-xs-3
+			.row.project-list-row(ng-cloak)
+				.project-list-container(ng-if="projects.length > 0")
+					aside.project-list-sidebar.col-md-2.col-xs-3
 						include ./list/side-bar
 
-					.col-md-10.col-xs-9
+					.project-list-main.col-md-10.col-xs-9
 						include ./list/notifications
 						include ./list/project-list
 				
-				span(ng-if="projects.length === 0")
+				.project-list-empty(ng-if="projects.length === 0")
 					.col-md-offset-2.col-md-8.col-md-offset-2.col-xs-8
 						include ./list/empty-project-list
 

services/web/app/views/project/list/project-list.pug

@@ -7,6 +7,7 @@
 			.form-group.has-feedback.has-feedback-left.col-md-7.col-xs-12
 				input.form-control.col-md-7.col-xs-12(
 					placeholder=translate('search_projects')+"…",
+					aria-label=translate('search_projects')+"…",
 					autofocus='autofocus',
 					ng-model="searchText.value",
 					focus-on='search:clear',
@@ -127,6 +128,7 @@
 							input.select-all(
 								select-all,
 								type="checkbox"
+								aria-label=translate('select_all_projects')
 							)
 							span.header.clickable(ng-click="changePredicate('name')") #{translate("title")}
 								i.tablesort.fa(ng-class="getSortIconClass('name')")
@@ -147,6 +149,7 @@
 								type="checkbox",
 								ng-model="project.selected"
 								stop-propagation="click"
+								aria-label=translate('select_project') + " '{{ project.name }}'"
 							)
 							span
 								a.projectName(

services/web/app/views/project/list/side-bar.pug

@@ -1,5 +1,5 @@
 .dropdown(dropdown)
-	a.btn.btn-primary.dropdown-toggle(
+	a.btn.btn-primary.sidebar-new-proj-btn.dropdown-toggle(
 		href="#",
 		data-toggle="dropdown",
 		dropdown-toggle
@@ -45,7 +45,7 @@
 			a(href) #{translate("shared_with_you")}
 		li(ng-class="{active: (filter == 'archived')}", ng-click="filterProjects('archived')")
 			a(href) #{translate("deleted_projects")}
-		li
+		li.separator
 			h2 #{translate("folders")}
 		li.tag(
 			ng-repeat="tag in tags | orderBy:'name'",

services/web/cleanup.js

@@ -1,11 +0,0 @@
-var keys = require('./app/js/infrastructure/Keys');
-var settings = require('settings-sharelatex');
-var queueName = process.argv[2];
-var projectQueueName = process.argv[3];
-var queue = require('fairy').connect(settings.redis.web).queue(queueName);
-console.log("cleaning up queue "+ queueName + " " + projectQueueName);
-queue._requeue_group(projectQueueName);
-
-//fairy should kill the process but just in case
-thirtySeconds = 30 * 1000
-setTimeout(process.exit, thirtySeconds)

services/web/config/settings.defaults.coffee

@@ -109,6 +109,9 @@ module.exports = settings =
 			url : "http://localhost:3005"
 		trackchanges:
 			url : "http://localhost:3015"
+		project_history:
+			enabled: false
+			url : "http://localhost:3054"
 		docstore:
 			url : "http://localhost:3016"
 			pubUrl: "http://localhost:3016"
@@ -442,3 +445,8 @@ module.exports = settings =
 	#	name : "all projects",
 	#	url: "/templates/all"
 	#}]
+
+	rateLimits:
+		autoCompile:
+			everyone: 100
+			standard: 25

services/web/package.json

@@ -25,11 +25,12 @@
     "dateformat": "1.0.4-1.2.3",
     "express": "4.13.0",
     "express-session": "^1.14.2",
+    "fs-extra": "^4.0.2",
     "heapdump": "^0.3.7",
     "helmet": "^3.8.1",
     "http-proxy": "^1.8.1",
-    "ioredis": "^2.4.0",
     "jade": "~1.3.1",
+    "jsonwebtoken": "^8.0.1",
     "ldapjs": "^0.7.1",
     "lodash": "^4.13.1",
     "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#master",
@@ -54,8 +55,7 @@
     "passport-oauth2-refresh": "^1.0.0",
     "passport-saml": "^0.15.0",
     "pug": "^2.0.0-beta6",
-    "redis": "0.10.1",
-    "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2",
+    "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
     "request": "^2.69.0",
     "requests": "^0.1.7",
     "rimraf": "2.2.6",
@@ -68,16 +68,17 @@
     "underscore": "1.6.0",
     "uuid": "^3.0.1",
     "v8-profiler": "^5.2.3",
-    "xml2js": "0.2.0"
+    "xml2js": "0.2.0",
+    "yauzl": "^2.8.0"
   },
   "devDependencies": {
     "autoprefixer": "^6.6.1",
     "bunyan": "0.22.1",
     "chai": "3.5.0",
     "chai-spies": "",
-    "grunt": "0.4.5",
     "clean-css": "^3.4.18",
     "es6-promise": "^4.0.5",
+    "grunt": "0.4.5",
     "grunt-available-tasks": "0.4.1",
     "grunt-bunyan": "0.5.0",
     "grunt-contrib-clean": "0.5.0",
@@ -89,12 +90,14 @@
     "grunt-exec": "^0.4.7",
     "grunt-execute": "^0.2.2",
     "grunt-file-append": "0.0.6",
+    "grunt-forever": "^0.4.7",
     "grunt-git-rev-parse": "^0.1.4",
     "grunt-mocha-test": "0.9.0",
     "grunt-newer": "^1.2.0",
     "grunt-parallel": "^0.5.1",
     "grunt-postcss": "^0.8.0",
     "grunt-sed": "^0.1.1",
+    "grunt-shell": "^2.1.0",
     "sandboxed-module": "0.2.0",
     "sinon": "^1.17.0",
     "timekeeper": "",

services/web/public/coffee/ide.coffee

@@ -12,7 +12,8 @@ define [
 	"ide/labels/LabelsManager"
 	"ide/review-panel/ReviewPanelManager"
 	"ide/SafariScrollPatcher"
-	"ide/FeatureOnboardingController"
+	"ide/FeatureOnboardingController",
+	"ide/AutoCompileOnboardingController",
 	"ide/settings/index"
 	"ide/share/index"
 	"ide/chat/index"
@@ -71,12 +72,17 @@ define [
 			view: "editor"
 			chatOpen: false
 			pdfLayout: 'sideBySide'
-			reviewPanelOpen: localStorage("ui.reviewPanelOpen.#{window.project_id}")
-			miniReviewPanelVisible: false
+			pdfHidden: false,
+			pdfWidth: 0,
+			reviewPanelOpen: localStorage("ui.reviewPanelOpen.#{window.project_id}"),
+			miniReviewPanelVisible: false,
+		}
+		$scope.onboarding = {
+			autoCompile: if window.showAutoCompileOnboarding then 'unseen' else 'dismissed'
 		}
 		$scope.user = window.user
 		$scope.__enableTokenAccessUI = window.enableTokenAccessUI == true
-		
+
 		$scope.$watch "project.features.trackChangesVisible", (visible) ->
 			return if !visible?
 			$scope.ui.showCollabFeaturesOnboarding = window.showTrackChangesOnboarding and visible
@@ -101,6 +107,10 @@ define [
 			if value?
 				localStorage "ui.reviewPanelOpen.#{window.project_id}", value
 
+		$scope.$on "layout:pdf:resize", (_, layoutState) ->
+			$scope.ui.pdfHidden = layoutState.east.initClosed
+			$scope.ui.pdfWidth = layoutState.east.size
+
 		# Tracking code.
 		$scope.$watch "ui.view", (newView, oldView) ->
 			if newView? and newView != "editor" and newView != "pdf"
@@ -183,6 +193,20 @@ define [
 		if ide.browserIsSafari
 			ide.safariScrollPatcher = new SafariScrollPatcher($scope)
 
+		# Fix Chrome 61 and 62 text-shadow rendering
+		browserIsChrome61or62 = false
+		try
+			chromeVersion = parseFloat(navigator.userAgent.split(" Chrome/")[1]) || null;
+			browserIsChrome61or62 = (
+				chromeVersion? &&
+				(chromeVersion == 61 || chromeVersion == 62)
+			)
+			if browserIsChrome61or62
+				document.styleSheets[0].insertRule(".ace_editor.ace_autocomplete .ace_completion-highlight { text-shadow: none !important; }", 1)
+		catch err
+			console.error err
+
+
 		# User can append ?ft=somefeature to url to activate a feature toggle
 		ide.featureToggle = location?.search?.match(/^\?ft=(\w+)$/)?[1]
 

services/web/public/coffee/ide/AutoCompileOnboardingController.coffee

@@ -0,0 +1,26 @@
+define [
+	"base"
+], (App) ->
+	App.controller "AutoCompileOnboardingController", ($scope, event_tracking) ->
+		recompileBtn = angular.element('#recompile')
+		popover = angular.element('#onboarding-autocompile')
+		{ top, left } = recompileBtn.offset()
+
+		# If pdf panel smaller than recompile button + popover, show to left.
+		# Otherwise show to right
+		if $scope.ui.pdfWidth < 475
+			$scope.placement = 'left'
+			popover.offset({
+				top: top,
+				left: left - popover.width()
+			})
+		else
+			$scope.placement = 'right'
+			popover.offset({
+				top: top,
+				left: left + recompileBtn.width()
+			})
+
+		$scope.dismiss = () ->
+			$scope.onboarding.autoCompile = 'dismissed'
+			event_tracking.sendMB "shown-autocompile-onboarding"

services/web/public/coffee/ide/editor/Document.coffee

@@ -252,21 +252,34 @@ define [
 
 		_joinDoc: (callback = (error) ->) ->
 			if @doc?
-				@ide.socket.emit 'joinDoc', @doc_id, @doc.getVersion(), (error, docLines, version, updates, ranges) =>
+				@ide.socket.emit 'joinDoc', @doc_id, @doc.getVersion(), { encodeRanges: true }, (error, docLines, version, updates, ranges) =>
 					return callback(error) if error?
 					@joined = true
 					@doc.catchUp( updates )
-					@_catchUpRanges( ranges?.changes, ranges?.comments )
+					@_decodeRanges(ranges)
+					@_catchUpRanges(ranges?.changes, ranges?.comments)
 					callback()
 			else
-				@ide.socket.emit 'joinDoc', @doc_id, (error, docLines, version, updates, ranges) =>
+				@ide.socket.emit 'joinDoc', @doc_id, { encodeRanges: true }, (error, docLines, version, updates, ranges) =>
 					return callback(error) if error?
 					@joined = true
 					@doc = new ShareJsDoc @doc_id, docLines, version, @ide.socket
+					@_decodeRanges(ranges)
 					@ranges = new RangesTracker(ranges?.changes, ranges?.comments)
 					@_bindToShareJsDocEvents()
 					callback()
 
+		_decodeRanges: (ranges) ->
+			decodeFromWebsockets = (text) -> decodeURIComponent(escape(text))
+			try
+				for change in ranges.changes or []
+					change.op.i = decodeFromWebsockets(change.op.i) if change.op.i?
+					change.op.d = decodeFromWebsockets(change.op.d) if change.op.d?
+				for comment in ranges.comments or []
+					comment.op.c = decodeFromWebsockets(comment.op.c) if comment.op.c?
+			catch err
+				console.log(err)
+
 		_leaveDoc: (callback = (error) ->) ->
 			sl_console.log '[_leaveDoc] Sending leaveDoc request'
 			@ide.socket.emit 'leaveDoc', @doc_id, (error) =>
@@ -309,6 +322,9 @@ define [
 				@ide.pushEvent "op:acknowledged",
 					doc_id: @doc_id
 					op: op
+				@ide.$scope.$emit "ide:opAcknowledged",
+					doc_id: @doc_id
+					op: op
 				@trigger "op:acknowledged"
 			@doc.on "op:timeout", (op) =>
 				@ide.pushEvent "op:timeout",

services/web/public/coffee/ide/editor/directives/aceEditor.coffee

@@ -374,6 +374,21 @@ define [
 					if scope.eventsBridge?
 						session.on "changeScrollTop", onScroll
 
+					$rootScope.hasLintingError = false
+					session.on('changeAnnotation', () ->
+						# Both linter errors and compile logs are set as error annotations,
+						# however when the user types something, the compile logs are
+						# replaced with linter errors. When we check for lint errors before
+						# autocompile we are guaranteed to get linter errors
+						hasErrors = session
+							.getAnnotations()
+							.filter((annotation) -> annotation.type != 'info')
+							.length > 0
+
+						if ($rootScope.hasLintingError != hasErrors)
+							$rootScope.hasLintingError = hasErrors
+					)
+
 					setTimeout () ->
 						# Let any listeners init themselves
 						onScroll(editor.renderer.getScrollTop())

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/AutoCompleteManager.coffee

@@ -1,21 +1,13 @@
 define [
 	"ide/editor/directives/aceEditor/auto-complete/CommandManager"
 	"ide/editor/directives/aceEditor/auto-complete/EnvironmentManager"
+	"ide/editor/directives/aceEditor/auto-complete/Helpers"
 	"ace/ace"
 	"ace/ext-language_tools"
-], (CommandManager, EnvironmentManager) ->
+], (CommandManager, EnvironmentManager, Helpers) ->
 	Range = ace.require("ace/range").Range
 	aceSnippetManager = ace.require('ace/snippets').snippetManager
 
-	getLastCommandFragment = (lineUpToCursor) ->
-		if m = lineUpToCursor.match(/(\\[^\\]+)$/)
-			return m[1]
-		else
-			return null
-
-	getCommandNameFromFragment = (commandFragment) ->
-		commandFragment?.match(/\\(\w+)\{/)?[1]
-
 	class AutoCompleteManager
 		constructor: (@$scope, @editor, @element, @labelsManager, @graphics, @preamble) ->
 			@suggestionManager = new CommandManager()
@@ -48,15 +40,11 @@ define [
 			Preamble = @preamble
 			GraphicsCompleter =
 				getCompletions: (editor, session, pos, prefix, callback) ->
-					upToCursorRange = new Range(pos.row, 0, pos.row, pos.column)
-					lineUpToCursor = editor.getSession().getTextRange(upToCursorRange)
-					commandFragment = getLastCommandFragment(lineUpToCursor)
+					context = Helpers.getContext(editor, pos)
+					{lineUpToCursor, commandFragment, lineBeyondCursor, needsClosingBrace} = context
 					if commandFragment
 						match = commandFragment.match(/^~?\\(includegraphics(?:\[.*])?){([^}]*, *)?(\w*)/)
 						if match
-							beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999)
-							lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange)
-							needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/)
 							commandName = match[1]
 							currentArg = match[3]
 							graphicsPaths = Preamble.getGraphicsPaths()
@@ -78,15 +66,11 @@ define [
 			labelsManager = @labelsManager
 			LabelsCompleter =
 				getCompletions: (editor, session, pos, prefix, callback) ->
-					upToCursorRange = new Range(pos.row, 0, pos.row, pos.column)
-					lineUpToCursor = editor.getSession().getTextRange(upToCursorRange)
-					commandFragment = getLastCommandFragment(lineUpToCursor)
+					context = Helpers.getContext(editor, pos)
+					{lineUpToCursor, commandFragment, lineBeyondCursor, needsClosingBrace} = context
 					if commandFragment
 						refMatch = commandFragment.match(/^~?\\([a-z]*ref){([^}]*, *)?(\w*)/)
 						if refMatch
-							beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999)
-							lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange)
-							needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/)
 							commandName = refMatch[1]
 							currentArg = refMatch[2]
 							result = []
@@ -108,15 +92,13 @@ define [
 			references = @$scope.$root._references
 			ReferencesCompleter =
 				getCompletions: (editor, session, pos, prefix, callback) ->
-					upToCursorRange = new Range(pos.row, 0, pos.row, pos.column)
-					lineUpToCursor = editor.getSession().getTextRange(upToCursorRange)
-					commandFragment = getLastCommandFragment(lineUpToCursor)
+					context = Helpers.getContext(editor, pos)
+					{lineUpToCursor, commandFragment, lineBeyondCursor, needsClosingBrace} = context
 					if commandFragment
-						citeMatch = commandFragment.match(/^~?\\([a-z]*cite[a-z]*(?:\[.*])?){([^}]*, *)?(\w*)/)
+						citeMatch = commandFragment.match(
+							/^~?\\([a-z]*cite[a-z]*(?:\[.*])?){([^}]*, *)?(\w*)/
+						)
 						if citeMatch
-							beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999)
-							lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange)
-							needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/)
 							commandName = citeMatch[1]
 							previousArgs = citeMatch[2]
 							currentArg = citeMatch[3]
@@ -160,8 +142,8 @@ define [
 		onChange: (change) ->
 			cursorPosition = @editor.getCursorPosition()
 			end = change.end
-			range = new Range(end.row, 0, end.row, end.column)
-			lineUpToCursor = @editor.getSession().getTextRange(range)
+			context = Helpers.getContext(@editor, end)
+			{lineUpToCursor, commandFragment, commandName, lineBeyondCursor, needsClosingBrace} = context
 			if lineUpToCursor.match(/.*%.*/)
 				return
 			lastCharIsBackslash = lineUpToCursor.slice(-1) == "\\"
@@ -170,8 +152,6 @@ define [
 			if lastTwoChars.match(/^\\[^a-z]$/)
 				@editor?.completer?.detach?()
 				return
-			commandFragment = getLastCommandFragment(lineUpToCursor)
-			commandName = getCommandNameFromFragment(commandFragment)
 			if commandName in ['begin', 'end']
 				return
 			# Check that this change was made by us, not a collaborator
@@ -233,10 +213,10 @@ define [
 											range.start.column,
 										)
 									)
-									# Delete back to last backslash, as appropriate
-									lastBackslashIndex = lineUpToCursor.lastIndexOf('\\')
-									if lastBackslashIndex != -1
-										leftRange.start.column = lastBackslashIndex
+									# Delete back to command start, as appropriate
+									commandStartIndex = Helpers.getLastCommandFragmentIndex(lineUpToCursor)
+									if commandStartIndex != -1
+										leftRange.start.column = commandStartIndex
 									else
 										leftRange.start.column -= completions.filterText.length
 									editor.session.remove(leftRange)
@@ -310,5 +290,5 @@ define [
 						currentLineOffset = i + 1
 						break
 				currentLine = text.slice(currentLineOffset, pos)
-				fragment = getLastCommandFragment(currentLine) or ""
+				fragment = Helpers.getLastCommandFragment(currentLine) or ""
 				return fragment

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/CommandManager.coffee

@@ -1,10 +1,15 @@
 define [], () ->
 	noArgumentCommands = [
 		'item', 'hline', 'lipsum', 'centering', 'noindent', 'textwidth', 'draw',
-		'maketitle', 'newpage', 'verb', 'bibliography', 'fi', 'hfill', 'par',
-		'in', 'sum', 'cdot', 'alpha', 'ldots', 'else', 'linewidth', 'left',
-		'right', 'today', 'clearpage', 'newline', 'endinput', 'mu',
-		'tableofcontents', 'vfill', 'bigskip', 'fill', 'cleardoublepage'
+		'maketitle', 'newpage', 'verb', 'bibliography', 'hfill', 'par',
+		'in', 'sum', 'cdot', 'ldots', 'linewidth', 'left', 'right', 'today',
+		'clearpage', 'newline', 'endinput', 'tableofcontents', 'vfill',
+		'bigskip', 'fill', 'cleardoublepage', 'infty', 'leq', 'geq', 'times',
+		'alpha', 'beta', 'gamma', 'delta', 'epsilon', 'varepsilon', 'zeta',
+		'eta', 'theta', 'vartheta', 'iota', 'kappa', 'lambda', 'mu', 'nu', 'xi',
+		'pi', 'varpi', 'rho', 'varrho', 'sigma', 'varsigma', 'tau', 'upsilon',
+		'phi', 'varphi', 'chi', 'psi', 'omega', 'Gamma', 'Delta', 'Theta',
+		'Lambda', 'Xi', 'Pi', 'Sigma', 'Upsilon', 'Phi', 'Psi', 'Omega'
 	]
 	singleArgumentCommands = [
 		'chapter', 'usepackage', 'section', 'label', 'textbf', 'subsection',
@@ -13,11 +18,12 @@ define [], () ->
 		'hspace', 'bibitem', 'url', 'large', 'subsubsection', 'textsc', 'date',
 		'footnote', 'small', 'thanks', 'underline', 'graphicspath', 'pageref',
 		'section*', 'subsection*', 'subsubsection*', 'sqrt', 'text',
-		'normalsize', 'Large', 'paragraph', 'pagestyle', 'thispagestyle',
-		'bibliographystyle'
+		'normalsize', 'footnotesize', 'Large', 'paragraph', 'pagestyle',
+		'thispagestyle', 'bibliographystyle', 'hat'
 	]
 	doubleArgumentCommands = [
-		'newcommand', 'frac', 'renewcommand', 'setlength', 'href', 'newtheorem'
+		'newcommand', 'frac', 'dfrac', 'renewcommand', 'setlength', 'href',
+		'newtheorem'
 	]
 	tripleArgumentCommands = [
 		'addcontentsline', 'newacronym', 'multicolumn'
@@ -25,12 +31,12 @@ define [], () ->
 	special = ['LaTeX', 'TeX']
 
 	rawCommands = [].concat(
-						noArgumentCommands,
-						singleArgumentCommands,
-						doubleArgumentCommands,
-						tripleArgumentCommands,
-						special
-					)
+		noArgumentCommands,
+		singleArgumentCommands,
+		doubleArgumentCommands,
+		tripleArgumentCommands,
+		special
+	)
 
 	noArgumentCommands = for cmd in noArgumentCommands
 		{
@@ -79,7 +85,7 @@ define [], () ->
 			# hacky solution: limit iterations
 			limit = null
 			if window?._ide?.browserIsSafari
-				limit = 100
+				limit = 5000
 
 			# fully formed commands
 			realCommands = []

services/web/public/coffee/ide/editor/directives/aceEditor/auto-complete/Helpers.coffee

@@ -0,0 +1,45 @@
+define [
+	"ace/ace"
+	"ace/ext-language_tools"
+], () ->
+	Range = ace.require("ace/range").Range
+
+	Helpers =
+		getLastCommandFragment: (lineUpToCursor) ->
+			if (index = Helpers.getLastCommandFragmentIndex(lineUpToCursor)) > -1
+				return lineUpToCursor.slice(index)
+			else
+				return null
+
+		getLastCommandFragmentIndex: (lineUpToCursor) ->
+			# This is hack to let us skip over commands in arguments, and
+			# go to the command on the same 'level' as us. E.g.
+			#    \includegraphics[width=\textwidth]{..
+			# should not match the \textwidth.
+			blankArguments = lineUpToCursor.replace /\[([^\]]*)\]/g, (args) ->
+				Array(args.length+1).join('.')
+			if m = blankArguments.match(/(\\[^\\]+)$/)
+				return m.index
+			else
+				return -1
+
+		getCommandNameFromFragment: (commandFragment) ->
+			commandFragment?.match(/\\(\w+)\{/)?[1]
+
+		getContext: (editor, pos) ->
+			upToCursorRange = new Range(pos.row, 0, pos.row, pos.column)
+			lineUpToCursor = editor.getSession().getTextRange(upToCursorRange)
+			commandFragment = Helpers.getLastCommandFragment(lineUpToCursor)
+			commandName = Helpers.getCommandNameFromFragment(commandFragment)
+			beyondCursorRange = new Range(pos.row, pos.column, pos.row, 99999)
+			lineBeyondCursor = editor.getSession().getTextRange(beyondCursorRange)
+			needsClosingBrace = !lineBeyondCursor.match(/^[^{]*}/)
+			return {
+				lineUpToCursor,
+				commandFragment,
+				commandName,
+				lineBeyondCursor,
+				needsClosingBrace
+			}
+
+	return Helpers

services/web/public/coffee/ide/history/controllers/HistoryListController.coffee

@@ -115,7 +115,10 @@ define [
 			$scope.resetHoverState()
 
 		$scope.displayName = (user) ->
-			full_name = "#{user.first_name} #{user.last_name}"
+			if user.name?
+				full_name = user.name
+			else
+				full_name = "#{user.first_name} #{user.last_name}"
 			fallback_name = "Unknown"
 			if !user?
 				fallback_name

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -5,8 +5,10 @@ define [
 	"libs/bib-log-parser"
 	"services/log-hints-feedback"
 ], (App, Ace, HumanReadableLogs, BibLogParser) ->
-	App.controller "PdfController", ($scope, $http, ide, $modal, synctex, event_tracking, logHintsFeedback, localStorage) ->
+	AUTO_COMPILE_TIMEOUT = 5000
+	OP_ACKNOWLEDGEMENT_TIMEOUT = 1100
 
+	App.controller "PdfController", ($scope, $http, ide, $modal, synctex, event_tracking, logHintsFeedback, localStorage) ->
 		# enable per-user containers by default
 		perUserCompile = true
 		autoCompile = true
@@ -66,13 +68,50 @@ define [
 		$scope.$on "project:joined", () ->
 			return if !autoCompile
 			autoCompile = false
-			$scope.recompile(isAutoCompile: true)
+			$scope.recompile(isAutoCompileOnLoad: true)
 			$scope.hasPremiumCompile = $scope.project.features.compileGroup == "priority"
 
 		$scope.$on "pdf:error:display", () ->
 			$scope.pdf.view = 'errors'
 			$scope.pdf.renderingError = true
 
+		autoCompileTimeout = null
+		triggerAutoCompile = () ->
+			return if autoCompileTimeout or $scope.ui.pdfHidden
+
+			timeSinceLastCompile = Date.now() - $scope.recompiledAt
+			# If time is non-monotonic, assume that the user's system clock has been
+			# changed and continue with recompile
+			isTimeNonMonotonic = timeSinceLastCompile < 0
+
+			if isTimeNonMonotonic || timeSinceLastCompile >= AUTO_COMPILE_TIMEOUT
+				if (!ide.$scope.hasLintingError)
+					$scope.recompile(isAutoCompileOnChange: true)
+			else
+				# Extend remainder of timeout
+				autoCompileTimeout = setTimeout () ->
+					autoCompileTimeout = null
+					triggerAutoCompile()
+				, AUTO_COMPILE_TIMEOUT - timeSinceLastCompile
+
+		autoCompileListener = null
+		toggleAutoCompile = (enabling) ->
+			if enabling
+				autoCompileListener = ide.$scope.$on "ide:opAcknowledged", _.debounce(triggerAutoCompile, OP_ACKNOWLEDGEMENT_TIMEOUT)
+			else
+				autoCompileListener() if autoCompileListener
+				autoCompileListener = null
+
+		$scope.autocompile_enabled = localStorage("autocompile_enabled:#{$scope.project_id}") or false
+		$scope.$watch "autocompile_enabled", (newValue, oldValue) ->
+			if newValue? and oldValue != newValue
+				localStorage("autocompile_enabled:#{$scope.project_id}", newValue)
+				toggleAutoCompile(newValue)
+				event_tracking.sendMB "autocompile-setting-changed", { value: newValue }
+
+		if (window.user?.betaProgram or window.showAutoCompileOnboarding) and $scope.autocompile_enabled
+			toggleAutoCompile(true)
+
 		# abort compile if syntax checks fail
 		$scope.stop_on_validation_error = localStorage("stop_on_validation_error:#{$scope.project_id}")
 		$scope.stop_on_validation_error ?= true # turn on for all users by default
@@ -88,7 +127,7 @@ define [
 		sendCompileRequest = (options = {}) ->
 			url = "/project/#{$scope.project_id}/compile"
 			params = {}
-			if options.isAutoCompile
+			if options.isAutoCompileOnLoad or options.isAutoCompileOnChange
 				params["auto_compile"]=true
 			# if the previous run was a check, clear the error logs
 			$scope.pdf.logEntries = [] if $scope.check
@@ -105,9 +144,9 @@ define [
 				rootDoc_id: options.rootDocOverride_id or null
 				draft: $scope.draft
 				check: checkType
-				# use incremental compile for beta users but revert to a full
+				# use incremental compile for all users but revert to a full
 				# compile if there is a server error
-				incrementalCompilesEnabled: window.user?.betaProgram and not $scope.pdf.error
+				incrementalCompilesEnabled: not $scope.pdf.error
 				_csrf: window.csrfToken
 			}, {params: params}
 
@@ -128,6 +167,8 @@ define [
 			$scope.pdf.compileTerminated = false
 			$scope.pdf.compileExited = false
 			$scope.pdf.failedCheck = false
+			$scope.pdf.compileInProgress = false
+			$scope.pdf.autoCompileDisabled = false
 
 			# make a cache to look up files by name
 			fileByPath = {}
@@ -166,7 +207,13 @@ define [
 				$scope.shouldShowLogs = true
 				fetchLogs(fileByPath)
 			else if response.status == "autocompile-backoff"
-				$scope.pdf.view = 'uncompiled'
+				if $scope.pdf.isAutoCompileOnLoad # initial autocompile
+					$scope.pdf.view = 'uncompiled'
+				else # background autocompile from typing
+					$scope.pdf.view = 'errors'
+					$scope.pdf.autoCompileDisabled = true
+					$scope.autocompile_enabled = false # disable any further autocompiles
+					event_tracking.sendMB "autocompile-rate-limited", {hasPremiumCompile: $scope.hasPremiumCompile}
 			else if response.status == "project-too-large"
 				$scope.pdf.view = 'errors'
 				$scope.pdf.projectTooLarge = true
@@ -184,6 +231,10 @@ define [
 			else if response.status == "validation-problems"
 				$scope.pdf.view = "validation-problems"
 				$scope.pdf.validation = response.validationProblems
+				$scope.shouldShowLogs = false
+			else if response.status == "compile-in-progress"
+				$scope.pdf.view = 'errors'
+				$scope.pdf.compileInProgress = true
 			else if response.status == "success"
 				$scope.pdf.view = 'pdf'
 				$scope.shouldShowLogs = false
@@ -369,9 +420,13 @@ define [
 		$scope.recompile = (options = {}) ->
 			return if $scope.pdf.compiling
 
+			if !options.isAutoCompileOnLoad and $scope.onboarding.autoCompile == 'unseen'
+				$scope.onboarding.autoCompile = 'show'
+
 			event_tracking.sendMBSampled "editor-recompile-sampled", options
 
 			$scope.pdf.compiling = true
+			$scope.pdf.isAutoCompileOnLoad = options?.isAutoCompileOnLoad # initial autocompile
 
 			if options?.force
 				# for forced compile, turn off validation check and ignore errors
@@ -401,6 +456,8 @@ define [
 					$scope.pdf.renderingError = false
 					$scope.pdf.error = true
 					$scope.pdf.view = 'errors'
+				.finally () ->
+					$scope.recompiledAt = Date.now()
 
 		# This needs to be public.
 		ide.$scope.recompile = $scope.recompile

services/web/public/coffee/ide/settings/controllers/SettingsController.coffee

@@ -55,7 +55,11 @@ define [
 
 		$scope.$watch "project.rootDoc_id", (rootDoc_id, oldRootDoc_id) =>
 			return if @ignoreUpdates
-			if oldRootDoc_id? and rootDoc_id != oldRootDoc_id
+			# don't save on initialisation, Angular passes oldRootDoc_id as
+			# undefined in this case.
+			return if typeof oldRootDoc_id is "undefined"
+			# otherwise only save changes, null values are allowed
+			if (rootDoc_id != oldRootDoc_id)
 				settings.saveProjectSettings({rootDocId: rootDoc_id})
 
 

services/web/public/coffee/main/project-list/project-list.coffee

@@ -26,22 +26,12 @@ define [
 		storedUIOpts = JSON.parse(localStorage("project_list"))
 
 		recalculateProjectListHeight = () ->
-			topOffset = $(".project-list-card")?.offset()?.top
-			bottomOffset = $("footer").outerHeight() + 25
-			sideBarHeight = $("aside").height() - 56
-			# When footer is visible and page doesn't need to scroll we just make it
-			# span between header and footer
-			height = $window.innerHeight - topOffset - bottomOffset
-			
-			# When page is small enough that this pushes the project list smaller than
-			# the side bar, then the window going to have to scroll to take into account the
-			# footer. So we now start to track to the bottom of the window, with a 25px padding
-			# since the footer is hidden below the fold. Don't ever get bigger than the sidebar
-			# though since that's what triggered this happening in the first place.
-			if height < sideBarHeight
-				height = Math.min(sideBarHeight, $window.innerHeight - topOffset - 25)
+			$projListCard = $(".project-list-card")
+			topOffset = $projListCard.offset()?.top
+			cardPadding = $projListCard.outerHeight() - $projListCard.height()
+			bottomOffset = $("footer").outerHeight()
+			height = $window.innerHeight - topOffset - bottomOffset - cardPadding
 			$scope.projectListHeight = height
-		
 
 		angular.element($window).bind "resize", () ->
 			recalculateProjectListHeight()

services/web/public/stylesheets/app/editor/feature-onboarding.less

@@ -98,4 +98,48 @@ a.feat-onboard-dismiss {
 		color: #FFF;
 		opacity: 1;
 	}
-}
+}
+
+.onboarding-autocompile {
+	display: block;
+	top: 10px;
+
+	img {
+		margin-bottom: 10px;
+		border: 1px solid @gray-lighter;
+	}
+
+	&::before, &::after {
+		content: '';
+		border-width: 11px;
+		border-style: solid;
+		border-color: transparent;
+		top: 7px;
+		display: block;
+		position: absolute;
+	}
+
+	&.right::before {
+		border-left-width: 0;
+		border-right-color: rgba(0, 0, 0, .3);
+		left: -11px;
+	}
+
+	&.right::after {
+		border-left-width: 0;
+		border-right-color: #f7f7f7;
+		left: -9.5px;
+	}
+
+	&.left::before {
+		border-right-width: 0;
+		border-left-color: rgba(0, 0, 0, .3);
+		right: -11px
+	}
+
+	&.left::after {
+		border-right-width: 0;
+		border-left-color: #f7f7f7;
+		right: -9.5px;
+	}
+}

services/web/public/stylesheets/app/editor/review-panel.less

@@ -947,6 +947,7 @@
 	justify-content: center;
 	border-radius: 3px;
 	box-shadow: 0 0 20px 10px rgba(0, 0, 0, .3);
+	z-index: 1;
 
 	&::before {
 		content: '';

services/web/public/stylesheets/app/ol-style-guide.less

@@ -0,0 +1,66 @@
+.renderColorSwatchClasses(@colorName) {
+	@colorVal: @@colorName;
+	@colorValRed: red(@colorVal);
+	@colorValGreen: green(@colorVal);
+	@colorValBlue: blue(@colorVal);
+	@colorValAsRGB: 'rgb(@{colorValRed}, @{colorValGreen}, @{colorValBlue})';
+
+	&.@{colorName} {
+		.color-swatch {
+			background-color: @colorVal;
+		}
+		.color-less-var::before {
+			content: '@@{colorName}';
+		}
+		.color-hex-val::before {
+			content: '@{colorVal}';
+		}
+		.color-rgb-val::before {
+			font-size: 10px;
+			content: '@{colorValAsRGB}';
+		}
+	}
+}
+
+.color-row {
+	display: flex;
+	justify-content: space-between;
+}
+
+.color-box {
+	background: white;
+	margin: 10px 4px;
+	border-radius: 4px;
+	width: 16.666%;
+	.renderColorSwatchClasses(ol-blue-gray-1);
+	.renderColorSwatchClasses(ol-blue-gray-2);
+	.renderColorSwatchClasses(ol-blue-gray-3);
+	.renderColorSwatchClasses(ol-blue-gray-4);
+	.renderColorSwatchClasses(ol-blue-gray-5);
+	.renderColorSwatchClasses(ol-blue-gray-6);
+	.renderColorSwatchClasses(ol-green);
+	.renderColorSwatchClasses(ol-dark-green);
+	.renderColorSwatchClasses(ol-blue);
+	.renderColorSwatchClasses(ol-dark-blue);
+	.renderColorSwatchClasses(ol-red);
+	.renderColorSwatchClasses(ol-dark-red);
+}
+
+.color-swatch {
+	height: 100px;
+	width: 100px;
+	margin: 10px auto;
+	border-radius: 4px;
+}
+
+.color-label {
+	display: flex;
+	flex-direction: column;
+	margin: 0 3px 10px;
+}
+
+.color-label pre {
+	font-size: 12px;
+	line-height: 1.8em;
+	margin: 0 auto;
+}

services/web/public/stylesheets/app/project-list.less

@@ -18,9 +18,56 @@
 }
 
 .project-list-page {
-	position: relative;
+	position: absolute;
+	top: @header-height;
+	bottom: @footer-height;
+	padding-top: 0;
+	padding-bottom: 0;
+	width: 100%;
+	overflow-x: hidden;
+	overflow-y: auto;
 }
 
+.project-list-content when (@is-overleaf) {
+	.container-fluid;
+	margin: 0;
+	height: 100%;
+}
+
+.project-list-content when (@is-overleaf = false) {
+	.container;
+}
+
+.sidebar-new-proj-btn when (@is-overleaf) {
+	.btn-block;
+}
+
+	.project-list-row when (@is-overleaf) {
+		height: 100%;
+	}
+		.project-list-container when (@is-overleaf) {
+			height: 100%;
+		}
+		.project-list-sidebar {
+			background-color: @sidebar-bg;
+			padding-top: @content-margin-vertical;
+			padding-bottom: @content-margin-vertical;
+		} 
+
+		.project-list-sidebar when (@is-overleaf) {
+			height: 100%;
+			overflow-x: hidden;
+			overflow-y: auto;
+			-ms-overflow-style: -ms-autohiding-scrollbar;
+		}
+
+		.project-list-main {
+			padding-top: @content-margin-vertical;
+			padding-bottom: @content-margin-vertical;
+			height: 100%;
+			overflow: hidden;
+		}
+
 .project-header {
 	.btn-group > .btn {
 		padding-left: @line-height-base / 2;
@@ -48,6 +95,9 @@
 	}
 	p {
 		margin-bottom: @line-height-computed / 4;
+		&.small {
+			color: @sidebar-color;
+		}
 	}
 }
 
@@ -76,27 +126,38 @@
 }
 
 ul.folders-menu {
-	margin: 0;
+	margin: @folders-menu-margin;
 	.subdued {
 		color: @gray-light;
 	}
 	> li {
 		cursor: pointer;
-		line-height: 1.8;
 		position: relative;
 		> a {
+			display: block;
+			color: @sidebar-link-color;
+			padding: @folders-menu-item-v-padding @folders-menu-item-h-padding;
+			border-bottom: solid 1px transparent;
+			&:hover {
+				background-color: @sidebar-hover-bg;
+				text-decoration: @sidebar-hover-text-decoration;
+			}
+			&:focus {
+				text-decoration: none;
+			}
+		}
+		> a when (@is-overleaf = false) {
 			font-size: 0.9rem;
-			color: #333;
-			padding: (@line-height-computed / 4);
-			display: inline-block;
-			line-height: 1.2;
+		}
+		&.separator {
+			padding: @folders-menu-item-v-padding @folders-menu-item-h-padding;
+			cursor: auto;
 		}
 	}
 	> li.active {
-		//border-right: 4px solid @red;
-		background-color: @link-color;
-		border-radius: @border-radius-small;
+		border-radius: @sidebar-active-border-radius;
 		> a {
+			background-color: @sidebar-active-bg;
 			font-weight: 700;
 			color: white;
 			.subdued {
@@ -108,10 +169,13 @@ ul.folders-menu {
 		color: @gray;
 	}
 	h2 {
-		margin-top: @line-height-computed / 2;
-		margin-bottom: @line-height-computed / 4;
-		font-size: @font-size-base;
-		font-weight: 500;
+		margin-top: @folders-title-margin-top;
+		margin-bottom: @folders-title-margin-bottom;
+		font-size: @folders-title-font-size;
+		color: @folders-title-color;
+		text-transform: @folders-title-text-transform;
+		padding: @folders-title-padding;
+		font-weight: @folders-title-font-weight;
 		font-family: @font-family-sans-serif;
 	}
 	> li.tag {
@@ -120,7 +184,7 @@ ul.folders-menu {
 				color: white;
 				border-color: white;
 				&:hover {
-					background-color: darken(@brand-primary, 10%);
+					background-color: @folders-tag-menu-active-hover;
 				}
 			}
 		}
@@ -128,7 +192,7 @@ ul.folders-menu {
 			font-style: italic;
 			margin-bottom: @line-height-computed / 4;
 			a {
-				line-height: 1.7;
+				line-height: @folders-untagged-line-height;
 				&:hover,
 				&:focus {
 					text-decoration: none;
@@ -140,7 +204,7 @@ ul.folders-menu {
 		}
 		&:hover {
 			&:not(.active) {
-				background-color: darken(@gray-lightest, 2%);
+				background-color: @folders-tag-hover;
 			}
 			.tag-menu {
 				display: block
@@ -148,30 +212,23 @@ ul.folders-menu {
 		}
 		&:not(.active) {
 			.tag-menu > a:hover {
-				background-color: @gray-light;
+				background-color: @folders-tag-menu-hover;
 			}
 		}
 		a.tag-name {
-			padding: 2px (@line-height-computed / 4);
-			margin-right: 18px;
-			display: inline-block;
+			padding: @folders-tag-padding;
+			display: @folders-tag-display;
 			position: relative;
-			i {
-				position: absolute;
-				top: 5px;
-				left: 6px;
-			}
 			span.name {
-				display: inline-block;
-				padding-left: 22px;
-				line-height: 1.4;
+				padding-left: 0.5em;
+				line-height: @folders-tag-line-height;
 			}
 		}
 		.tag-menu {
 			> a {
-				border: 1px solid @gray;
+				border: 1px solid @folders-tag-border-color;
 				border-radius: @border-radius-small;
-				color: @text-color;
+				color: @folders-tag-menu-color;
 				display: block;
 				width: 16px;
 				height: 16px;
@@ -184,7 +241,8 @@ ul.folders-menu {
 			}
 			display: none;
 			position: absolute;
-			top: 6px;
+			top: 50%;
+			margin-top: -8px; // Half the element height.
 			right: 4px;
 			&.open {
 				display: block;
@@ -204,32 +262,34 @@ ul.structured-list {
 	margin: 0;
 	overflow: hidden;
 	overflow-y: auto;
+	-ms-overflow-style: -ms-autohiding-scrollbar;
 	li {
-		border-bottom: 1px solid @gray-lightest;
+		border-bottom: 1px solid @structured-list-border-color;
 		padding: (@line-height-computed / 4) 0;
-		&:first-child {
-			.header {
-				font-size: 1rem;
-			}
-		}
+		
 		&:last-child {
 			border-bottom: 0 none;
 		}
 		&:hover {
-			background-color: @gray-lightest;
+			background-color: @structured-list-hover-color;
 		}
-		&:first-child:hover {
-			background-color: white;
+		&:first-child {
+			border-bottom-color: @structured-header-border-color;
+			&:hover {
+				background-color: transparent;
+			}
 		}
 		a {
-			color: darken(@blue, 10%);
+			color: @structured-list-link-color;
 		}
-		.header {
-			text-transform: uppercase;
+		.header when (@is-overleaf = true) {
+			font-weight: 600;
 		}
-		.select-item, .select-all {
-			display: inline-block;
+
+		.header when (@is-overleaf = false) {
+			text-transform: uppercase;
 		}
+		
 		.select-item, .select-all {
 			position: absolute;
 			left: @line-height-computed;
@@ -245,17 +305,25 @@ ul.structured-list {
 	}
 }
 
+.project-list-card when (@is-overleaf) {
+	padding: 0 (@line-height-computed / 4);
+}
+
 ul.project-list {
 	li {
-		.last-modified, .owner {
+		.last-modified when (@is-overleaf = false) {
+			font-size: .8rem;
+		}
+		.owner when (@is-overleaf = false) {
 			font-size: .8rem;
 		}
-		.owner {
+		.owner when (@is-overleaf = false) {
 			margin-right: 0;
 		}
 		.projectName {
 			margin-right: @line-height-computed / 4;
 		}
+
 		.tag-label {
 			margin-left: @line-height-computed / 4;
 			position: relative;
@@ -267,6 +335,12 @@ ul.project-list {
 				display: inline-block;
 				padding-top: 0.3em;
 				color: #FFF;
+				border-radius: @tag-border-radius;
+				background-color: @tag-bg-color;
+				&:hover,
+				&:focus {
+					background-color: @tag-bg-hover-color;
+				}
 			}
 			.tag-label-name {
 				padding-right: 0.3em;
@@ -347,7 +421,7 @@ ul.project-list {
 
 .announcements {
 	position: absolute;
-	bottom: 0;
+	bottom: @footer-height;
 	right: 0;
 	height: 150px;
 	width: 100%;
@@ -429,7 +503,7 @@ ul.project-list {
 	margin-right: 95px;
 	bottom: 30px;
 	width: 700px;
-	max-height: 52%;
+	max-height: 40%;
 	min-height: 100px;
 	background: #FFF;
 	z-index: 1;

services/web/public/stylesheets/components/buttons.less

@@ -14,10 +14,10 @@
   vertical-align: middle;
   cursor: pointer;
   background-image: none; // Reset unusual Firefox-on-Android default style; see https://github.com/necolas/normalize.css/issues/214
-  border: 1px solid transparent;
-  border-bottom: 2px solid transparent;
+  border: @btn-border-width solid transparent;
+  border-bottom: @btn-border-bottom-width solid transparent;
   white-space: nowrap;
-  .button-size(@padding-base-vertical; @padding-base-horizontal; @font-size-base; @line-height-base; @border-radius-base);
+  .button-size(@padding-base-vertical; @padding-base-horizontal; @font-size-base; @line-height-base; @btn-border-radius-base);
   .user-select(none);
 
   &,
@@ -124,14 +124,14 @@
 
 .btn-lg {
   // line-height: ensure even-numbered height of button next to large input
-  .button-size(@padding-large-vertical; @padding-large-horizontal; @font-size-large; @line-height-large; @border-radius-large);
+  .button-size(@padding-large-vertical; @padding-large-horizontal; @font-size-large; @line-height-large; @btn-border-radius-large);
 }
 .btn-sm {
   // line-height: ensure proper height of button next to small input
-  .button-size(@padding-small-vertical; @padding-small-horizontal; @font-size-small; @line-height-small; @border-radius-small);
+  .button-size(@padding-small-vertical; @padding-small-horizontal; @font-size-small; @line-height-small; @btn-border-radius-small);
 }
 .btn-xs {
-  .button-size(@padding-xs-vertical; @padding-xs-horizontal; @font-size-small; @line-height-small; @border-radius-small);
+  .button-size(@padding-xs-vertical; @padding-xs-horizontal; @font-size-small; @line-height-small; @btn-border-radius-small);
 }
 
 

services/web/public/stylesheets/components/card.less

@@ -1,8 +1,8 @@
 .card {
   background-color: white;
   border-radius: @border-radius-base;
-  -webkit-box-shadow: 0 2px 4px rgba(0,0,0,0.1);
-  box-shadow: 0 2px 4px rgba(0,0,0,0.15);
+  -webkit-box-shadow: @card-box-shadow;
+  box-shadow: @card-box-shadow;
   padding: @line-height-computed;
   .page-header {
   	margin: 0 0 1.5625rem;

services/web/public/stylesheets/components/footer.less

@@ -1,7 +1,13 @@
 footer.site-footer {
+	background-color: @footer-bg-color;
 	border-top: 1px solid @gray-lighter;
-	padding: 2em;
 	font-size: 0.9rem;
+	position: absolute;
+	bottom: 0;
+	width: 100%;
+	height: @footer-height;
+	line-height: @footer-height - 1; // Hack — in Chrome, using the full @footer-height would generate vertical scrolling
+		
 	ul {
 		list-style: none;
 		margin: 0px;
@@ -22,6 +28,21 @@ footer.site-footer {
 			vertical-align: text-bottom;
 		}
 	}
+
+	a {
+		color: @footer-link-color;
+		&:hover,
+		&:focus {
+			color: @footer-link-hover-color;
+		}
+	}
+}
+
+.site-footer-content when (@is-overleaf = true) {
+	.container-fluid;
+}
+.site-footer-content when (@is-overleaf = false) {
+	.container;
 }
 
 .sprite-icon-lang {

services/web/public/stylesheets/components/navbar.less

@@ -174,9 +174,9 @@
     font-size: 20px;
     display: inline-block;
     margin-top: 2px;
-    color: #666;
+    color: @navbar-title-color;
     &:hover, &:active, &:focus {
-        color: #333;
+        color: @navbar-title-color-hover;
         text-decoration: none;
     }
 }
@@ -378,13 +378,17 @@
 .navbar-default {
   background-color: @navbar-default-bg;
   border-color: @navbar-default-border;
-  padding: 1rem 2rem;
-
+  padding: @navbar-default-padding;
+  position: absolute;
+  top: 0;
+  width: 100%;
+  height: @header-height;
+  
   .navbar-brand {
     position: absolute;
     top: 5px;
     bottom: 5px;
-    width: 180px;
+    width: @navbar-brand-width;
     padding: 0;
     background-image: @navbar-brand-image-url;
     background-size: contain;
@@ -400,11 +404,11 @@
     > li > a {
       color: @navbar-default-link-color;
       border: 2px solid transparent;
-      border-radius: @border-radius-base;
-      font-size: @font-size-base * .8;
-      font-weight: 700;
-      line-height: 1;
-      padding: 10px 10px 11px;
+      border-radius: @navbar-btn-border-radius;
+      font-size: @navbar-btn-font-size;
+      font-weight: @navbar-btn-font-weight;
+      line-height: @navbar-btn-line-height;
+      padding: @navbar-btn-padding;
 
       &:hover,
       &:focus {
@@ -432,15 +436,15 @@
 
     > li.subdued > a {
       border: 0;
-      color: @gray;
-      padding: 12px 12px 13px;
+      color: @navbar-subdued-color;
+      padding: @navbar-subdued-padding;
       margin-left: 0;
       &:hover {
-        color: @gray-dark;
-        background-color: @gray-lightest;
+        color: @navbar-subdued-hover-color;
+        background-color: @navbar-subdued-hover-bg;
       }
       &:focus {
-          color: @gray;
+          color: @navbar-subdued-color;
           background-color: transparent;
       }
     }
@@ -485,8 +489,8 @@
       &,
       &:hover,
       &:focus {
-        color: @gray-dark;
-        background-color: @gray-lightest;
+        color: @navbar-subdued-hover-color;
+        background-color: @navbar-subdued-hover-bg;
       }
     }
 

services/web/public/stylesheets/components/progress-bars.less

@@ -29,8 +29,8 @@
   height: @line-height-computed;
   margin-bottom: @line-height-computed;
   background-color: @progress-bg;
-  border-radius: @border-radius-base;
-  border: 1px solid @progress-border-color;
+  border-radius: @progress-border-radius;
+  border: @progress-border-width solid @progress-border-color;
   .box-shadow(inset 0 1px 2px rgba(0,0,0,.1));
 }
 
@@ -44,7 +44,7 @@
   color: @progress-bar-color;
   text-align: center;
   background-color: @progress-bar-bg;
-  .box-shadow(inset 0 -1px 0 rgba(0,0,0,.15));
+  .box-shadow(@progress-bar-shadow);
   .transition(width .6s ease);
 }
 

services/web/public/stylesheets/core/_common-variables.less

@@ -85,7 +85,6 @@
 @border-radius-base:        3px;
 @border-radius-large:       5px;
 @border-radius-small:       2px;
-
 //** Global color for active items (e.g., navs or dropdowns).
 @component-active-color:    #fff;
 //** Global background color for active items (e.g., navs or dropdowns).
@@ -789,10 +788,7 @@
 //** Horizontal offset for forms and lists.
 @component-offset-horizontal: 180px;
 
-@content-margin-top: @line-height-computed;
-@content-margin-top: @line-height-computed;
-
-// Custom
+@content-margin-vertical: @line-height-computed;
 
 @left-menu-width: 260px;
 @left-menu-animation-duration: 0.35s;
@@ -803,3 +799,94 @@
 @editor-dark-background-color: #333;
 @editor-dark-toolbar-border-color: #222;
 @editor-dark-highlight-color: #FFA03A;
+
+
+// Custom
+@is-overleaf   : false;
+@header-height : 68px;
+@footer-height : 50px;
+
+// Backgrounds
+@content-alt-bg-color: lighten(@gray-lightest, 2.5%);
+
+// Typography
+@text-small-color: @gray;
+
+// Navbar
+@navbar-title-color         : #666;
+@navbar-title-color-hover   : #333;
+@navbar-default-padding     : 1rem 2rem;
+@navbar-brand-width         : 180px;
+@navbar-btn-font-size       : @font-size-base * 0.8;
+@navbar-btn-border-radius   : @border-radius-base;
+@navbar-btn-font-weight     : 700;
+@navbar-btn-padding         : 10px 10px 11px;
+@navbar-btn-line-height     : 1;
+@navbar-subdued-padding     : 12px 12px 13px;
+@navbar-subdued-color       : @gray;
+@navbar-subdued-hover-bg    : @gray-lightest;
+@navbar-subdued-hover-color : @gray-dark;
+
+// Button colors and sizing
+@btn-border-radius-large : @border-radius-large;
+@btn-border-radius-base  : @border-radius-base;
+@btn-border-radius-small : @border-radius-small;
+@btn-border-width        : 1px;
+@btn-border-bottom-width : 2px; 
+
+// Cards
+@card-box-shadow: 0 2px 4px rgba(0,0,0,0.15);
+
+// Project table
+@structured-list-link-color     : darken(@blue, 10%);
+@structured-header-border-color : @gray-lightest;
+@structured-list-border-color   : @gray-lightest;
+@structured-list-hover-color    : @gray-lightest;
+@structured-list-line-height    : @line-height-base;
+
+// Sidebar
+@sidebar-bg                    : transparent;
+@sidebar-color                 : @gray;
+@sidebar-link-color            : #333;
+@sidebar-active-border-radius  : @border-radius-small;
+@sidebar-active-bg             : @link-color;
+@sidebar-hover-bg              : transparent;
+@sidebar-hover-text-decoration : underline;
+
+@folders-menu-margin           : 0;
+@folders-menu-line-height      : 1.2;
+@folders-menu-item-v-padding   : (@line-height-computed / 4);
+@folders-menu-item-h-padding   : (@line-height-computed / 4);
+@folders-title-padding         : 0;
+@folders-title-margin-top      : (@line-height-computed / 2);
+@folders-title-margin-bottom   : (@line-height-computed / 4);
+@folders-title-font-size       : @font-size-base;
+@folders-title-font-weight     : 500;
+@folders-title-line-height     : @headings-line-height;
+@folders-title-color           : inherit;
+@folders-title-text-transform  : none;
+@folders-tag-padding           : 2px 20px 2px @folders-menu-item-h-padding;
+@folders-tag-line-height       : 1.8;
+@folders-tag-display           : block;
+@folders-tag-menu-color        : @gray;
+@folders-tag-hover             : darken(@gray-lightest, 2%);
+@folders-tag-border-color      : @text-color;
+@folders-tag-menu-active-hover : darken(@brand-primary, 10%);
+@folders-tag-menu-hover        : @gray-light;
+@folders-untagged-line-height  : 1.7;
+
+// Progress bars
+@progress-border-radius : @border-radius-base;
+@progress-border-width  : 1px;
+@progress-bar-shadow    : inset 0 -1px 0 rgba(0,0,0,.15);
+
+// Footer
+@footer-link-color       : @link-color;
+@footer-link-hover-color : @link-hover-color;
+@footer-bg-color         : transparent;
+@footer-padding          : 2em;
+
+// Tags
+@tag-border-radius  : 0.25em;
+@tag-bg-color       : @label-default-bg;
+@tag-bg-hover-color : darken(@label-default-bg, 10%);

services/web/public/stylesheets/core/ol-variables.less

@@ -1,11 +1,164 @@
-@ol-green:      #4A9F48;
-@ol-dark-green: #1C5B26;
+@import "./_common-variables.less";
+
+@is-overleaf: true;
+@header-height: 68px;
+@footer-height: 50px;
+
+// Styleguide colors
+@ol-blue-gray-1 : #E4E8EE;
+@ol-blue-gray-2 : #9DA7B7;
+@ol-blue-gray-3 : #5D6879;
+@ol-blue-gray-4 : #485973;
+@ol-blue-gray-5 : #2C3645;
+@ol-blue-gray-6 : #1E2530;
+
+@ol-green       : #4F9C45;
+@ol-dark-green  : #1C5B26;
+@ol-blue        : #4B7FD1;
+@ol-dark-blue   : #2857A1;
+@ol-red         : #C9453E;
+@ol-dark-red    : #A6312B;
+
+@ol-type-color  : @ol-blue-gray-3;
+
+// Navbar customization
+@navbar-title-color       : @ol-blue-gray-1;
+@navbar-title-color-hover : @ol-blue-gray-2;
+@navbar-brand-width       : 130px;
+@navbar-default-color     : #FFF;
+@navbar-default-bg        : @ol-blue-gray-6;
+@navbar-default-border    : transparent;
+@navbar-brand-image-url   : url(/img/ol-brand/overleaf-white.svg);
+
+// Backgrounds
+@body-bg                : #FFF;
+@content-alt-bg-color   : @ol-blue-gray-1;
+
+// Typography
+@text-small-color       : @ol-type-color;
+@text-color             : @ol-type-color;
+@link-color             : @ol-blue;
+@link-hover-color       : @ol-dark-blue;
+
+// Button colors and sizing
+@btn-border-width        : 0;
+@btn-border-bottom-width : 0;
+
+@btn-border-radius-large : 9999px;
+@btn-border-radius-base  : 9999px;
+@btn-border-radius-small : 9999px;
+
+@btn-default-color       : #FFF;
+@btn-default-bg          : @ol-blue-gray-4;
+@btn-default-border      : transparent;
+
+@btn-primary-color       : #FFF;
+@btn-primary-bg          : @ol-green;
+@btn-primary-border      : transparent;
+
+@btn-success-color       : #FFF;
+@btn-success-bg          : @ol-green;
+@btn-success-border      : transparent;
+
+@btn-info-color          : #FFF;
+@btn-info-bg             : @ol-blue;
+@btn-info-border         : transparent;
+
+// Tags
+@tag-border-radius       : 9999px;
+@tag-bg-color            : @ol-green;
+@tag-bg-hover-color      : @ol-dark-green;
+
+// Navbar
+@navbar-default-padding           : (@grid-gutter-width / 2) 0;
+@navbar-default-link-color        : #FFF;
+@navbar-default-link-hover-bg     : @ol-green;
+@navbar-default-link-active-bg    : @ol-green;
+@navbar-default-link-hover-color  : @ol-green;
+@navbar-btn-font-size             : @font-size-base;
+@navbar-btn-border-radius         : @btn-border-radius-base;
+@navbar-btn-font-weight           : 400;
+@navbar-btn-padding               : (@padding-base-vertical - 1) @padding-base-horizontal @padding-base-vertical;
+@navbar-btn-line-height           : @line-height-base;
+
+@navbar-subdued-color             : #FFF;
+@navbar-subdued-padding           : (@padding-base-vertical + 1) (@padding-base-horizontal + 1) (@padding-base-vertical + 2);
+@navbar-subdued-hover-bg          : #FFF;
+@navbar-subdued-hover-color       : @ol-green;
+
+// Forms
+@input-color                      : @ol-blue-gray-3;
+@input-border-radius              : unit(@line-height-base, em);
+@input-height-base                : @line-height-computed + (@padding-base-vertical * 2) - 1;
+// TODO Warning color-orange?
+@btn-warning-color                : #FFF;
+@btn-warning-bg                   : @ol-red;
+@btn-warning-border               : transparent;
+
+@btn-danger-color                 : #FFF;
+@btn-danger-bg                    : @ol-red;
+@btn-danger-border                : transparent;
+
+// Cards
+@card-box-shadow                  : none;
+
+// Sidebar
+@sidebar-bg                     : @ol-blue-gray-5;
+@sidebar-color                  : @ol-blue-gray-2;
+@sidebar-link-color             : #FFF;
+@sidebar-active-border-radius   : 0;
+@sidebar-active-bg              : @ol-blue-gray-6;
+@sidebar-hover-bg               : @ol-blue-gray-4;
+@sidebar-hover-text-decoration  : none;
+
+@folders-menu-margin            : 0 -(@grid-gutter-width / 2);
+@folders-menu-line-height       : @structured-list-line-height;
+@folders-menu-item-v-padding    : (@line-height-computed / 4);
+@folders-menu-item-h-padding    : (@grid-gutter-width / 2);
+@folders-title-padding          : @folders-menu-item-v-padding 0;
+@folders-title-margin-top       : 0;
+@folders-title-margin-bottom    : 0;
+@folders-title-font-weight      : normal;
+@folders-title-font-size        : @font-size-small;
+@folders-title-color            : @ol-blue-gray-2;
+@folders-title-text-transform   : uppercase;
+@folders-tag-display            : block;
+@folders-tag-line-height        : 1.4;
+@folders-tag-padding            : @folders-menu-item-v-padding 20px @folders-menu-item-v-padding @folders-menu-item-h-padding;
+@folders-tag-menu-color         : #FFF;
+@folders-tag-hover              : @sidebar-hover-bg;
+@folders-tag-border-color       : @folders-tag-menu-color;
+@folders-tag-menu-hover         : rgba(0, 0, 0, .1);
+@folders-tag-menu-active-hover  : rgba(0, 0, 0, .1);
+@folders-untagged-line-height   : @folders-menu-line-height;
+
+// Project table
+@structured-list-line-height    : 2.5;
+@structured-list-link-color     : @ol-blue;
+@structured-header-border-color : shade(@ol-blue-gray-1, 5%);
+@structured-list-border-color   : @ol-blue-gray-1;
+@structured-list-hover-color    : lighten(@ol-blue-gray-1, 5%);
+
+
+// Progress bars
+@progress-border-radius         : @line-height-computed;
+@progress-border-width          : 0;
+@progress-bar-bg                : @ol-blue-gray-4;
+@progress-bar-success-bg        : @ol-green;
+@progress-bar-warning-bg        : @brand-warning;
+@progress-bar-danger-bg         : @ol-red;
+@progress-bar-info-bg           : @ol-blue;
+@progress-bar-shadow            : none;
+
+// Footer
+@footer-bg-color                : #FFF;
+@footer-link-color              : @ol-green;
+@footer-link-hover-color        : @ol-dark-green;
+@footer-padding                 : 2em 0;
 
 //== Colors
 //
 //## Gray and brand colors for use across Bootstrap.
-
-
 @gray-darker:           #252525;
 @gray-dark:             #505050;
 @gray:                  #7a7a7a;
@@ -28,10 +181,6 @@
 @brand-warning:         @orange;
 @brand-danger:          #E03A06;
 
-@navbar-brand-image-url: url(/img/ol-brand/logo-horizontal.png);
-
 @editor-loading-logo-padding-top: 115.44%;
 @editor-loading-logo-background-url: url(/img/ol-brand/overleaf-o-grey.svg);
-@editor-loading-logo-foreground-url: url(/img/ol-brand/overleaf-o.svg);
-
-@import "./_common-variables.less";
+@editor-loading-logo-foreground-url: url(/img/ol-brand/overleaf-o.svg);

services/web/public/stylesheets/core/scaffolding.less

@@ -21,6 +21,7 @@
 html {
   //font-size: 62.5%;
   -webkit-tap-highlight-color: rgba(0,0,0,0);
+  height: 100%;
 }
 
 body {
@@ -29,6 +30,13 @@ body {
   line-height: @line-height-base;
   color: @text-color;
   background-color: @body-bg;
+  min-height: 100%;
+  position: relative;
+  padding-top: @header-height;
+  padding-bottom: @footer-height;
+  & > .content {
+    min-height: calc(~"100vh -" (@header-height + @footer-height));
+  }
 }
 
 // Reset fonts for relevant elements
@@ -137,12 +145,12 @@ hr {
 }
 
 .content {
-  padding-top: @content-margin-top;
-  padding-bottom: @content-margin-top;
+  padding-top: @content-margin-vertical;
+  padding-bottom: @content-margin-vertical;
 }
 
 .content-alt {
-  background-color: lighten(@gray-lightest, 2.5%);
+  background-color: @content-alt-bg-color;
 }
 
 .row-spaced {

services/web/public/stylesheets/core/type.less

@@ -81,7 +81,7 @@ p {
 
 // Ex: 14px base font * 85% = about 12px
 small,
-.small  { font-size: 90%; color: @gray }
+.small  { font-size: 90%; color: @text-small-color; }
 
 // Undo browser default styling
 cite    { font-style: normal; }

services/web/public/stylesheets/ol-style.less

@@ -1,3 +1,4 @@
 // Core variables and mixins
 @import "core/ol-variables.less";
+@import "app/ol-style-guide.less";
 @import "_style_includes.less";

services/web/test/UnitTests/coffee/Collaborators/CollaboratorsHandlerTests.coffee

@@ -431,3 +431,103 @@ describe "CollaboratorsHandler", ->
 				expect(err).to.not.exist
 				expect(isTokenMember).to.equal false
 				done()
+
+	describe 'transferProjects', ->
+		beforeEach ->
+			@from_user_id = "from-user-id"
+			@to_user_id = "to-user-id"
+			@projects = [{
+				_id: "project-id-1"
+			}, {
+				_id: "project-id-2"
+			}]
+			@Project.find = sinon.stub().yields(null, @projects)
+			@Project.update = sinon.stub().yields()
+			@ProjectEntityHandler.flushProjectToThirdPartyDataStore = sinon.stub().yields()
+
+		describe "successfully", ->
+			beforeEach ->
+				@CollaboratorHandler.transferProjects @from_user_id, @to_user_id, @callback
+
+			it "should look up the affected projects", ->
+				@Project.find
+					.calledWith({
+						$or : [
+							{ owner_ref: @from_user_id }
+							{ collaberator_refs: @from_user_id }
+							{ readOnly_refs: @from_user_id }
+						]
+					})
+					.should.equal true
+
+			it "should transfer owned projects", ->
+				@Project.update
+					.calledWith({
+						owner_ref: @from_user_id	
+					}, {
+						$set: { owner_ref: @to_user_id }
+					}, {
+						multi: true
+					})
+					.should.equal true
+
+			it "should transfer collaborator projects", ->
+				@Project.update
+					.calledWith({
+						collaberator_refs: @from_user_id	
+					}, {
+						$addToSet: { collaberator_refs: @to_user_id }
+					}, {
+						multi: true
+					})
+					.should.equal true
+				@Project.update
+					.calledWith({
+						collaberator_refs: @from_user_id	
+					}, {
+						$pull: { collaberator_refs: @from_user_id }
+					}, {
+						multi: true
+					})
+					.should.equal true
+
+			it "should transfer read only collaborator projects", ->
+				@Project.update
+					.calledWith({
+						readOnly_refs: @from_user_id	
+					}, {
+						$addToSet: { readOnly_refs: @to_user_id }
+					}, {
+						multi: true
+					})
+					.should.equal true
+				@Project.update
+					.calledWith({
+						readOnly_refs: @from_user_id	
+					}, {
+						$pull: { readOnly_refs: @from_user_id }
+					}, {
+						multi: true
+					})
+					.should.equal true
+
+			it "should flush each project to the TPDS", ->
+				for project in @projects
+					@ProjectEntityHandler.flushProjectToThirdPartyDataStore
+						.calledWith(project._id)
+						.should.equal true
+
+			it "should call the callback", ->
+				@callback.called.should.equal true
+
+		describe "when flushing to TPDS fails", ->
+			beforeEach ->
+				@ProjectEntityHandler.flushProjectToThirdPartyDataStore = sinon.stub().yields(new Error('oops'))
+				@CollaboratorHandler.transferProjects @from_user_id, @to_user_id, @callback
+
+			it "should log an error", ->
+				@logger.err.called.should.equal true
+
+			it "should not return an error since it happens in the background", ->
+				@callback.called.should.equal true
+				@callback.calledWith(new Error('oops')).should.equal false

services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee

@@ -33,7 +33,7 @@ describe "ClsiManager", ->
 				getProjectDocsIfMatch: sinon.stub().callsArgWith(2,null,null)
 			"./ClsiCookieManager": @ClsiCookieManager
 			"./ClsiStateManager": @ClsiStateManager
-			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), err: sinon.stub(), warn: sinon.stub() }
 			"request": @request = sinon.stub()
 			"./ClsiFormatChecker": @ClsiFormatChecker
 			"metrics-sharelatex": @Metrics =
@@ -122,6 +122,21 @@ describe "ClsiManager", ->
 			it "should call the callback with a success status", ->
 				@callback.calledWith(null, @status, ).should.equal true
 
+		describe "when the resources fail the precompile check", ->
+			beforeEach ->
+				@ClsiFormatChecker.checkRecoursesForProblems = sinon.stub().callsArgWith(1, new Error("failed"))
+				@ClsiManager._postToClsi = sinon.stub().callsArgWith(4, null, {
+					compile:
+						status: @status = "failure"
+				})
+				@ClsiManager.sendRequest @project_id, @user_id, {}, @callback
+
+			it "should call the callback only once", ->
+				@callback.calledOnce.should.equal true
+
+			it "should call the callback with an error", ->
+				@callback.calledWithExactly(new Error("failed")).should.equal true
+
 	describe "deleteAuxFiles", ->
 		beforeEach ->
 			@ClsiManager._makeRequest = sinon.stub().callsArg(2)
@@ -247,12 +262,12 @@ describe "ClsiManager", ->
 					.calledWith(@project_id, {compiler:1, rootDoc_id: 1, imageName: 1, rootFolder: 1})
 					.should.equal true
 
-			it "should flush the project to the database", ->
+			it "should not explicitly flush the project to the database", ->
 				@DocumentUpdaterHandler.flushProjectToMongo
 					.calledWith(@project_id)
-					.should.equal true
+					.should.equal false
 
-			it "should get only the live docs from the docupdater", ->
+			it "should get only the live docs from the docupdater with a background flush in docupdater", ->
 				@DocumentUpdaterHandler.getProjectDocsIfMatch
 					.calledWith(@project_id)
 					.should.equal true
@@ -331,7 +346,49 @@ describe "ClsiManager", ->
 
 			it "should set to main.tex", ->
 				@request.compile.rootResourcePath.should.equal "main.tex"
-		
+
+		describe "when there is no valid root document and no main.tex document", ->
+			beforeEach () ->
+				@project.rootDoc_id = "not-valid"
+				@docs = {
+					"/other.tex": @doc_1 = {
+						name: "other.tex"
+						_id: "mock-doc-id-1"
+						lines: ["Hello", "world"]
+					},
+					"/chapters/chapter1.tex": @doc_2 = {
+						name: "chapter1.tex"
+						_id: "mock-doc-id-2"
+						lines: [
+							"Chapter 1"
+						]
+					}
+				}
+				@ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs)
+				@ClsiManager._buildRequest @project, null, @callback
+
+			it "should report an error", ->
+				@callback.calledWith(new Error("no main file specified")).should.equal true
+
+
+		describe "when there is no valid root document and a single document which is not main.tex", ->
+			beforeEach (done) ->
+				@project.rootDoc_id = "not-valid"
+				@docs = {
+					"/other.tex": @doc_1 = {
+						name: "other.tex"
+						_id: "mock-doc-id-1"
+						lines: ["Hello", "world"]
+					}
+				}
+				@ProjectEntityHandler.getAllDocs = sinon.stub().callsArgWith(1, null, @docs)
+				@ClsiManager._buildRequest @project, null, (@error, @request) =>
+					done()
+
+			it "should set io to the only file", ->
+				@request.compile.rootResourcePath.should.equal "other.tex"
+
+
 		describe "with the draft option", ->
 			it "should add the draft option into the request", (done) ->
 				@ClsiManager._buildRequest @project_id, {timeout:100, draft: true}, (error, request) =>

services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee

@@ -44,7 +44,7 @@ describe "CompileManager", ->
 
 		describe "succesfully", ->
 			beforeEach ->
-				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (_, cb)-> cb(null, true)
+				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (isAutoCompile, compileGroup, cb)-> cb(null, true)
 				@CompileManager.compile @project_id, @user_id, {}, @callback
 
 			it "should check the project has not been recently compiled", ->
@@ -84,7 +84,7 @@ describe "CompileManager", ->
 				
 		describe "when the project has been recently compiled", ->
 			it "should return", (done)->
-				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (_, cb)-> cb(null, true)
+				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (isAutoCompile, compileGroup, cb)-> cb(null, true)
 				@CompileManager._checkIfRecentlyCompiled = sinon.stub().callsArgWith(2, null, true)
 				@CompileManager.compile @project_id, @user_id, {}, (err, status)->
 					status.should.equal "too-recently-compiled"
@@ -92,7 +92,7 @@ describe "CompileManager", ->
 
 		describe "should check the rate limit", ->
 			it "should return", (done)->
-				@CompileManager._checkIfAutoCompileLimitHasBeenHit = sinon.stub().callsArgWith(1, null, false)
+				@CompileManager._checkIfAutoCompileLimitHasBeenHit = sinon.stub().callsArgWith(2, null, false)
 				@CompileManager.compile @project_id, @user_id, {}, (err, status)->
 					status.should.equal "autocompile-backoff"
 					done()
@@ -222,14 +222,14 @@ describe "CompileManager", ->
 	describe "_checkIfAutoCompileLimitHasBeenHit", ->
 
 		it "should be able to compile if it is not an autocompile", (done)->
-			@ratelimiter.addCount.callsArgWith(1, null, true)
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit false, (err, canCompile)=>
+			@ratelimiter.addCount.callsArgWith(2, null, true)
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit false, "everyone", (err, canCompile)=>
 				canCompile.should.equal true
 				done()
 
 		it "should be able to compile if rate limit has remianing", (done)->
 			@ratelimiter.addCount.callsArgWith(1, null, true)
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=>
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
 				args = @ratelimiter.addCount.args[0][0]
 				args.throttle.should.equal 25
 				args.subjectName.should.equal "everyone"
@@ -240,13 +240,13 @@ describe "CompileManager", ->
 
 		it "should be not able to compile if rate limit has no remianing", (done)->
 			@ratelimiter.addCount.callsArgWith(1, null, false)
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=>
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
 				canCompile.should.equal false
 				done()
 
 		it "should return false if there is an error in the rate limit", (done)->
 			@ratelimiter.addCount.callsArgWith(1, "error")
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=>
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
 				canCompile.should.equal false
 				done()
 

services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee

@@ -265,19 +265,19 @@ describe 'DocumentUpdaterHandler', ->
 					v: @version
 				@docs = [ @doc0, @doc0, @doc0 ]
 				@body = JSON.stringify @docs
-				@request.get = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body)
+				@request.post = sinon.stub().callsArgWith(1, null, {statusCode: 200}, @body)
 				@handler.getProjectDocsIfMatch @project_id, @project_state_hash, @callback
 
 			it 'should get the documenst from the document updater', ->
-				url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/doc?state=#{@project_state_hash}"
-				@request.get.calledWith(url).should.equal true
+				url = "#{@settings.apis.documentupdater.url}/project/#{@project_id}/get_and_flush_if_old?state=#{@project_state_hash}"
+				@request.post.calledWith(url).should.equal true
 
 			it "should call the callback with the documents", ->
 				@callback.calledWithExactly(null, @docs).should.equal true
 
 		describe "when the document updater API returns an error", ->
 			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
+				@request.post = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
 				@handler.getProjectDocsIfMatch @project_id, @project_state_hash, @callback
 
 			it "should return an error to the callback", ->
@@ -285,7 +285,7 @@ describe 'DocumentUpdaterHandler', ->
 
 		describe "when the document updater returns a conflict error code", ->
 			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, { statusCode: 409 }, "Conflict")
+				@request.post = sinon.stub().callsArgWith(1, null, { statusCode: 409 }, "Conflict")
 				@handler.getProjectDocsIfMatch @project_id, @project_state_hash, @callback
 
 			it "should return the callback with no documents", ->
@@ -312,7 +312,7 @@ describe 'DocumentUpdaterHandler', ->
 
 		describe "when the document updater API returns an error", ->
 			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
+				@request.post = sinon.stub().callsArgWith(1, @error = new Error("something went wrong"), null, null)
 				@handler.getProjectDocsIfMatch @project_id, @project_state_hash, @callback
 
 			it "should return an error to the callback", ->
@@ -320,7 +320,7 @@ describe 'DocumentUpdaterHandler', ->
 
 		describe "when the document updater returns a conflict error code", ->
 			beforeEach ->
-				@request.get = sinon.stub().callsArgWith(1, null, { statusCode: 409 }, "Conflict")
+				@request.post = sinon.stub().callsArgWith(1, null, { statusCode: 409 }, "Conflict")
 				@handler.getProjectDocsIfMatch @project_id, @project_state_hash, @callback
 
 			it "should return the callback with no documents", ->

services/web/test/UnitTests/coffee/Documents/DocumentControllerTests.coffee

@@ -24,6 +24,7 @@ describe "DocumentController", ->
 		@doc_lines = ["one", "two", "three"]
 		@version = 42
 		@ranges = {"mock": "ranges"}
+		@pathname = '/a/b/c/file.tex'
 		@rev = 5
 
 	describe "getDocument", ->
@@ -34,12 +35,12 @@ describe "DocumentController", ->
 
 		describe "when the document exists", ->
 			beforeEach ->
-				@ProjectEntityHandler.getDoc = sinon.stub().callsArgWith(2, null, @doc_lines, @rev, @version, @ranges)
+				@ProjectEntityHandler.getDoc = sinon.stub().callsArgWith(3, null, @doc_lines, @rev, @version, @ranges, @pathname)
 				@DocumentController.getDocument(@req, @res, @next)
 
 			it "should get the document from Mongo", ->
 				@ProjectEntityHandler.getDoc
-					.calledWith(@project_id, @doc_id)
+					.calledWith(@project_id, @doc_id, pathname: true)
 					.should.equal true
 
 			it "should return the document data to the client as JSON", ->
@@ -48,10 +49,11 @@ describe "DocumentController", ->
 					lines: @doc_lines
 					version: @version
 					ranges: @ranges
+					pathname: @pathname
 
 		describe "when the document doesn't exist", ->
 			beforeEach ->
-				@ProjectEntityHandler.getDoc = sinon.stub().callsArgWith(2, new Errors.NotFoundError("not found"), null)
+				@ProjectEntityHandler.getDoc = sinon.stub().callsArgWith(3, new Errors.NotFoundError("not found"), null)
 				@DocumentController.getDocument(@req, @res, @next)
 
 			it "should call next with the NotFoundError", ->

services/web/test/UnitTests/coffee/History/HistoryControllerTests.coffee

@@ -6,6 +6,7 @@ SandboxedModule = require('sandboxed-module')
 
 describe "HistoryController", ->
 	beforeEach ->
+		@callback = sinon.stub()
 		@user_id = "user-id-123"
 		@AuthenticationController =
 			getLoggedInUserId: sinon.stub().returns(@user_id)
@@ -14,46 +15,134 @@ describe "HistoryController", ->
 			"settings-sharelatex": @settings = {}
 			"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub()}
 			"../Authentication/AuthenticationController": @AuthenticationController
+		@settings.apis =
+			trackchanges:
+				enabled: false
+				url: "http://trackchanges.example.com"
+			project_history:
+				url: "http://project_history.example.com"
 
 	describe "proxyToHistoryApi", ->
 		beforeEach ->
 			@req = { url: "/mock/url", method: "POST" }
 			@res = "mock-res"
 			@next = sinon.stub()
-			@settings.apis =
-				trackchanges:
-					url: "http://trackchanges.example.com"
 			@proxy =
 				events: {}
 				pipe: sinon.stub()
 				on: (event, handler) -> @events[event] = handler
 			@request.returns @proxy
-			@HistoryController.proxyToHistoryApi @req, @res, @next
 
 		describe "successfully", ->
-			it "should get the user id", ->
-				@AuthenticationController.getLoggedInUserId
-					.calledWith(@req)
-					.should.equal true
-
-			it "should call the track changes api", ->
-				@request
-					.calledWith({
-						url: "#{@settings.apis.trackchanges.url}#{@req.url}"
-						method: @req.method
-						headers:
-							"X-User-Id": @user_id
-					})
-					.should.equal true
-
-			it "should pipe the response to the client", ->
-				@proxy.pipe
-					.calledWith(@res)
-					.should.equal true
+			describe "with project history enabled", ->
+				beforeEach ->
+					@settings.apis.project_history.enabled = true
+					@HistoryController.proxyToHistoryApi @req, @res, @next
+
+				it "should get the user id", ->
+					@AuthenticationController.getLoggedInUserId
+						.calledWith(@req)
+						.should.equal true
+
+				it "should call the project history api", ->
+					@request
+						.calledWith({
+							url: "#{@settings.apis.project_history.url}#{@req.url}"
+							method: @req.method
+							headers:
+								"X-User-Id": @user_id
+						})
+						.should.equal true
+
+				it "should pipe the response to the client", ->
+					@proxy.pipe
+						.calledWith(@res)
+						.should.equal true
+
+			describe "with project history disabled", ->
+				beforeEach ->
+					@settings.apis.project_history.enabled = false
+					@HistoryController.proxyToHistoryApi @req, @res, @next
+
+				it "should call the track changes api", ->
+					@request
+						.calledWith({
+							url: "#{@settings.apis.trackchanges.url}#{@req.url}"
+							method: @req.method
+							headers:
+								"X-User-Id": @user_id
+						})
+						.should.equal true
 
 		describe "with an error", ->
 			beforeEach ->
+				@HistoryController.proxyToHistoryApi @req, @res, @next
 				@proxy.events["error"].call(@proxy, @error = new Error("oops"))
 
 			it "should pass the error up the call chain", ->
 				@next.calledWith(@error).should.equal true
+
+	describe "initializeProject", ->
+		describe "with project history enabled", ->
+			beforeEach ->
+				@settings.apis.project_history.enabled = true
+
+			describe "project history returns a successful response", ->
+				beforeEach ->
+					@overleaf_id = 1234
+					@res = statusCode: 200
+					@body = JSON.stringify(project: id: @overleaf_id)
+					@request.post = sinon.stub().callsArgWith(1, null, @res, @body)
+
+					@HistoryController.initializeProject @callback
+
+				it "should call the project history api", ->
+					@request.post.calledWith(
+						url: "#{@settings.apis.project_history.url}/project"
+					).should.equal true
+
+				it "should return the callback with the overleaf id", ->
+					@callback.calledWithExactly(null, { @overleaf_id }).should.equal true
+
+			describe "project history returns a response without the project id", ->
+				beforeEach ->
+					@res = statusCode: 200
+					@body = JSON.stringify(project: {})
+					@request.post = sinon.stub().callsArgWith(1, null, @res, @body)
+
+					@HistoryController.initializeProject @callback
+
+				it "should return the callback with an error", ->
+					@callback
+						.calledWith(sinon.match.has("message", "project-history did not provide an id"))
+						.should.equal true
+
+			describe "project history returns a unsuccessful response", ->
+				beforeEach ->
+					@res = statusCode: 404
+					@request.post = sinon.stub().callsArgWith(1, null, @res)
+
+					@HistoryController.initializeProject @callback
+
+				it "should return the callback with an error", ->
+					@callback
+						.calledWith(sinon.match.has("message", "project-history returned a non-success status code: 404"))
+						.should.equal true
+
+			describe "project history errors", ->
+				beforeEach ->
+					@error = sinon.stub()
+					@request.post = sinon.stub().callsArgWith(1, @error)
+
+					@HistoryController.initializeProject @callback
+
+				it "should return the callback with the error", ->
+					@callback.calledWithExactly(@error).should.equal true
+
+		describe "with project history disabled", ->
+			beforeEach ->
+				@settings.apis.project_history.enabled = false
+				@HistoryController.initializeProject @callback
+
+			it "should return the callback", ->
+				@callback.calledWithExactly().should.equal true

services/web/test/UnitTests/coffee/History/HistoryManagerTests.coffee

@@ -1,63 +0,0 @@
-chai = require('chai')
-expect = chai.expect
-chai.should()
-sinon = require("sinon")
-modulePath = "../../../../app/js/Features/History/HistoryManager"
-SandboxedModule = require('sandboxed-module')
-
-describe "HistoryManager", ->
-	beforeEach ->
-		@HistoryManager = SandboxedModule.require modulePath, requires:
-			"request" : @request = sinon.stub()
-			"settings-sharelatex": @settings =
-				apis:
-					trackchanges:
-						url: "trackchanges.sharelatex.com"
-			"logger-sharelatex": @logger = {log: sinon.stub(), error: sinon.stub()}
-		@project_id = "project-id-123"
-		@callback = sinon.stub()
-		@request.post = sinon.stub()
-
-	describe "flushProject", ->
-		describe "with a successful response code", ->
-			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, statusCode: 204, "")
-				@HistoryManager.flushProject @project_id, @callback
-
-			it "should flush the project in the track changes api", ->
-				@request.post
-					.calledWith("#{@settings.apis.trackchanges.url}/project/#{@project_id}/flush")
-					.should.equal true
-
-			it "should call the callback without an error", ->
-				@callback.calledWith(null).should.equal true
-
-		describe "with a failed response code", ->
-			beforeEach ->
-				@request.post = sinon.stub().callsArgWith(1, null, statusCode: 500, "")
-				@HistoryManager.flushProject @project_id, @callback
-
-			it "should call the callback with an error", ->
-				@callback.calledWith(new Error("track-changes api responded with a non-success code: 500")).should.equal true
-
-			it "should log the error", ->
-				@logger.error
-					.calledWith({
-						err: new Error("track-changes api responded with a non-success code: 500")
-						project_id: @project_id
-					}, "error flushing project in track-changes api")
-					.should.equal true
-
-	describe "ArchiveProject", ->
-
-		it "should call the post endpoint", (done)->
-			@request.post.callsArgWith(1, null, {})
-			@HistoryManager.archiveProject @project_id, (err)=>
-				@request.post.calledWith("#{@settings.apis.trackchanges.url}/project/#{@project_id}/archive")
-				done()
-
-		it "should return an error on a non success", (done)->
-			@request.post.callsArgWith(1, null, {statusCode:500})
-			@HistoryManager.archiveProject @project_id, (err)=>
-				expect(err).to.exist
-				done()

services/web/test/UnitTests/coffee/Project/ProjectCreationHandlerTests.coffee

@@ -22,6 +22,8 @@ describe 'ProjectCreationHandler', ->
 				@._id = project_id
 				@owner_ref = options.owner_ref
 				@name = options.name
+				@overleaf =
+					history: {}
 			save: sinon.stub().callsArg(0)
 			rootFolder:[{
 				_id: rootFolderId
@@ -36,11 +38,13 @@ describe 'ProjectCreationHandler', ->
 			setRootDoc: sinon.stub().callsArg(2)
 		@ProjectDetailsHandler =
 			validateProjectName: sinon.stub().yields()
+		@HistoryController =
+			initializeProject: sinon.stub().callsArg(0)
 
-		@user = 
+		@user =
 			first_name:"first name here"
 			last_name:"last name here"
-			ace: 
+			ace:
 				spellCheckLanguage:"de"
 
 		@User = findById:sinon.stub().callsArgWith(2, null, @user)
@@ -49,6 +53,7 @@ describe 'ProjectCreationHandler', ->
 			'../../models/User': User:@User
 			'../../models/Project':{Project:@ProjectModel}
 			'../../models/Folder':{Folder:@FolderModel}
+			'../History/HistoryController': @HistoryController
 			'./ProjectEntityHandler':@ProjectEntityHandler
 			"./ProjectDetailsHandler":@ProjectDetailsHandler
 			"settings-sharelatex": @Settings = {}
@@ -60,32 +65,48 @@ describe 'ProjectCreationHandler', ->
 
 	describe 'Creating a Blank project', ->
 		beforeEach ->
+			@overleaf_id = 1234
+			@HistoryController.initializeProject = sinon.stub().callsArgWith(0, null, { @overleaf_id })
 			@ProjectModel::save = sinon.stub().callsArg(0)
 
 		describe "successfully", ->
-
 			it "should save the project", (done)->
 				@handler.createBlankProject ownerId, projectName, =>
 					@ProjectModel::save.called.should.equal true
 					done()
-				
+
 			it "should return the project in the callback", (done)->
 				@handler.createBlankProject ownerId, projectName, (err, project)->
 					project.name.should.equal projectName
 					(project.owner_ref + "").should.equal ownerId
 					done()
 
+			it "should initialize the project overleaf if history id not provided", (done)->
+				@handler.createBlankProject ownerId, projectName, done
+				@HistoryController.initializeProject.calledWith().should.equal true
+
+			it "should set the overleaf id if overleaf id not provided", (done)->
+				@handler.createBlankProject ownerId, projectName, (err, project)=>
+					project.overleaf.history.id.should.equal @overleaf_id
+					done()
+
+			it "should set the overleaf id if overleaf id provided", (done)->
+				overleaf_id = 2345
+				@handler.createBlankProject ownerId, projectName, overleaf_id, (err, project)->
+					project.overleaf.history.id.should.equal overleaf_id
+					done()
+
 			it "should set the language from the user", (done)->
 				@handler.createBlankProject ownerId, projectName, (err, project)->
 					project.spellCheckLanguage.should.equal "de"
 					done()
-			
+
 			it "should set the imageName to currentImageName if set", (done) ->
 				@Settings.currentImageName = "mock-image-name"
 				@handler.createBlankProject ownerId, projectName, (err, project)=>
 					project.imageName.should.equal @Settings.currentImageName
 					done()
-			
+
 			it "should not set the imageName if no currentImageName", (done) ->
 				@Settings.currentImageName = null
 				@handler.createBlankProject ownerId, projectName, (err, project)=>
@@ -96,21 +117,21 @@ describe 'ProjectCreationHandler', ->
 			beforeEach ->
 				@ProjectModel::save = sinon.stub().callsArgWith(0, new Error("something went wrong"))
 				@handler.createBlankProject ownerId, projectName, @callback
-			
+
 			it 'should return the error to the callback', ->
 				should.exist @callback.args[0][0]
-		
+
 		describe "with an invalid name", ->
 			beforeEach ->
 				@ProjectDetailsHandler.validateProjectName = sinon.stub().yields(new Error("bad name"))
 				@handler.createBlankProject ownerId, projectName, @callback
-			
+
 			it 'should return the error to the callback', ->
 				should.exist @callback.args[0][0]
-			
+
 			it 'should not try to create the project', ->
 				@ProjectModel::save.called.should.equal false
-				
+
 
 	describe 'Creating a basic project', ->
 		beforeEach ->

services/web/test/UnitTests/coffee/Project/ProjectDetailsHandlerTests.coffee

@@ -12,7 +12,7 @@ describe 'ProjectDetailsHandler', ->
 	beforeEach ->
 		@project_id = "321l3j1kjkjl"
 		@user_id = "user-id-123"
-		@project = 
+		@project =
 			name: "project"
 			description: "this is a great project"
 			something:"should not exist"
@@ -20,7 +20,7 @@ describe 'ProjectDetailsHandler', ->
 			owner_ref: @user_id
 		@user =
 			features: "mock-features"
-		@ProjectGetter = 
+		@ProjectGetter =
 			getProjectWithoutDocLines: sinon.stub().callsArgWith(1, null, @project)
 			getProject: sinon.stub().callsArgWith(2, null, @project)
 		@ProjectModel =
@@ -43,7 +43,7 @@ describe 'ProjectDetailsHandler', ->
 	describe "getDetails", ->
 
 		it "should find the project and owner", (done)->
-			@handler.getDetails @project_id, (err, details)=>				
+			@handler.getDetails @project_id, (err, details)=>
 				details.name.should.equal @project.name
 				details.description.should.equal @project.description
 				details.compiler.should.equal @project.compiler
@@ -51,6 +51,13 @@ describe 'ProjectDetailsHandler', ->
 				assert.equal(details.something, undefined)
 				done()
 
+		it "should find overleaf metadata if it exists", (done)->
+			@project.overleaf = { id: 'id' }
+			@handler.getDetails @project_id, (err, details)=>
+				details.overleaf.should.equal @project.overleaf
+				assert.equal(details.something, undefined)
+				done()
+
 		it "should return an error for a non-existent project", (done)->
 			@ProjectGetter.getProject.callsArg(2, null, null)
 			err = new Errors.NotFoundError("project not found")
@@ -80,7 +87,7 @@ describe 'ProjectDetailsHandler', ->
 			@handler.getProjectDescription @project_id, (returnedErr, returnedDescription)=>
 				err.should.equal returnedErr
 				description.should.equal returnedDescription
-				done()	
+				done()
 
 	describe "setProjectDescription", ->
 
@@ -111,7 +118,7 @@ describe 'ProjectDetailsHandler', ->
 			@handler.renameProject @project_id, @newName, =>
 				@tpdsUpdateSender.moveEntity.calledWith({project_id:@project_id, project_name:@project.name, newProjectName:@newName}).should.equal true
 				done()
-		
+
 		it "should not do anything with an invalid name", (done) ->
 			@handler.validateProjectName = sinon.stub().yields(new Error("invalid name"))
 			@handler.renameProject @project_id, @newName, =>
@@ -120,6 +127,12 @@ describe 'ProjectDetailsHandler', ->
 				done()
 
 	describe "validateProjectName", ->
+
+		it "should reject undefined names", (done) ->
+			@handler.validateProjectName undefined, (error) ->
+				expect(error).to.exist
+				done()
+
 		it "should reject empty names", (done) ->
 			@handler.validateProjectName "", (error) ->
 				expect(error).to.exist

services/web/test/UnitTests/coffee/Project/ProjectEntityHandlerTests.coffee

@@ -14,17 +14,17 @@ describe 'ProjectEntityHandler', ->
 	doc_id = '4eecb1c1bffa66588e0000a2'
 	folder_id = "4eecaffcbffa66588e000008"
 	rootFolderId = "4eecaffcbffa66588e000007"
-	
+
 	beforeEach ->
-		@FileStoreHandler =  
+		@FileStoreHandler =
 			uploadFileFromDisk:(project_id, fileRef,  localImagePath, callback)->callback()
 			copyFile: sinon.stub().callsArgWith(4, null)
 		@tpdsUpdateSender =
 			addDoc:sinon.stub().callsArg(1)
 			addFile:sinon.stub().callsArg(1)
 			addFolder:sinon.stub().callsArg(1)
-		@rootFolder = 
-			_id:rootFolderId, 
+		@rootFolder =
+			_id:rootFolderId,
 			folders:[
 				{name:"level1", folders:[]}
 			]
@@ -46,7 +46,7 @@ describe 'ProjectEntityHandler', ->
 		@FileModel =  class File
 			constructor:(options)->
 				{@name} = options
-				@._id = "file_id" 
+				@._id = "file_id"
 				@rev = 0
 		@FolderModel = class Folder
 			constructor:(options)->
@@ -57,12 +57,12 @@ describe 'ProjectEntityHandler', ->
 
 		@ProjectModel.findById = (project_id, callback)=> callback(null, @project)
 		@ProjectModel.getProject = (project_id, fields, callback)=> callback(null, @project)
-		@ProjectGetter = 
+		@ProjectGetter =
 			getProjectWithOnlyFolders : (project_id, callback)=> callback(null, @project)
 			getProjectWithoutDocLines : (project_id, callback)=> callback(null, @project)
 			getProject:sinon.stub()
 		@projectUpdater = markAsUpdated:sinon.stub()
-		@projectLocator = 
+		@projectLocator =
 			findElement : sinon.stub()
 		@settings =
 			maxEntitiesPerProject:200
@@ -97,8 +97,8 @@ describe 'ProjectEntityHandler', ->
 				else
 					cb null, @parentFolder
 			@ProjectEntityHandler.addFolder = (project_id, parentFolder_id, folderName, callback)=>
-				callback null, {name:folderName}, @parentFolder_id 
-		
+				callback null, {name:folderName}, @parentFolder_id
+
 		it 'should return the root folder if the path is just a slash', (done)->
 			path = "/"
 			@ProjectEntityHandler.mkdirp project_id, path, (err, folders, lastFolder)=>
@@ -239,7 +239,7 @@ describe 'ProjectEntityHandler', ->
 			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, path: @pathAfterMove)
 			@ProjectGetter.getProject.callsArgWith(2, null, @project)
 			@tpdsUpdateSender.moveEntity = sinon.stub().callsArg(1)
-			
+
 		describe "moving a doc", ->
 			beforeEach (done) ->
 				@docId = "4eecaffcbffa66588e000009"
@@ -257,10 +257,10 @@ describe 'ProjectEntityHandler', ->
 			it 'should remove the element from its current position', ->
 				@ProjectEntityHandler._removeElementFromMongoArray
 					.calledWith(@ProjectModel, project_id, @path.mongo ).should.equal true
-					
+
 			it "should put the element back in the new folder", ->
 				@ProjectEntityHandler._putElement.calledWith(@project, folder_id, @doc, "docs").should.equal true
-					
+
 			it 'should tell the third party data store', ->
 				@tpdsUpdateSender.moveEntity
 					.calledWith({
@@ -271,7 +271,7 @@ describe 'ProjectEntityHandler', ->
 						rev: @doc.rev
 					})
 					.should.equal true
-					
+
 		describe "moving a folder", ->
 			beforeEach ->
 				@folder_id = "folder-to-move"
@@ -294,7 +294,7 @@ describe 'ProjectEntityHandler', ->
 					else
 						console.log "UNKNOWN ID", options
 				sinon.spy @projectLocator, "findElement"
-						
+
 			describe "when the destination folder is outside the moving folder", ->
 				beforeEach (done) ->
 					@path.fileSystem = "/one/directory"
@@ -318,7 +318,7 @@ describe 'ProjectEntityHandler', ->
 							@path.mongo
 						)
 						.should.equal true
-						
+
 				it "should put the element back in the new folder", ->
 					@ProjectEntityHandler._putElement
 						.calledWith(
@@ -328,7 +328,7 @@ describe 'ProjectEntityHandler', ->
 							"folder"
 						)
 						.should.equal true
-						
+
 				it 'should tell the third party data store', ->
 					@tpdsUpdateSender.moveEntity
 						.calledWith({
@@ -339,7 +339,7 @@ describe 'ProjectEntityHandler', ->
 							rev: @folder.rev
 						})
 						.should.equal true
-						
+
 			describe "when the destination folder is inside the moving folder", ->
 				beforeEach ->
 					@path.fileSystem = "/one/two"
@@ -355,7 +355,7 @@ describe 'ProjectEntityHandler', ->
 							project: @project
 						})
 						.should.equal true
-						
+
 				it "should return an error", ->
 					@callback
 						.calledWith(new Error("destination folder is a child folder of me"))
@@ -385,16 +385,41 @@ describe 'ProjectEntityHandler', ->
 			@rev = 5
 			@version = 42
 			@ranges = {"mock": "ranges"}
+
 			@DocstoreManager.getDoc = sinon.stub().callsArgWith(3, null, @lines, @rev, @version, @ranges)
-			@ProjectEntityHandler.getDoc project_id, doc_id, @callback
 
-		it "should call the docstore", ->
-			@DocstoreManager.getDoc
-				.calledWith(project_id, doc_id)
-				.should.equal true
+		describe 'without pathname option', ->
+			beforeEach ->
+				@ProjectEntityHandler.getDoc project_id, doc_id, @callback
+
+			it "should call the docstore", ->
+				@DocstoreManager.getDoc
+					.calledWith(project_id, doc_id)
+					.should.equal true
+
+			it "should call the callback with the lines, version and rev", ->
+				@callback.calledWith(null, @lines, @rev, @version, @ranges).should.equal true
+
+		describe 'with pathname option', ->
+			beforeEach ->
+				@project = 'a project'
+				@path = mongo: "mongo.path", fileSystem: "/file/system/path"
+				@projectLocator.findElement = sinon.stub().callsArgWith(1, null, {}, @path)
+				@ProjectEntityHandler.getDoc project_id, doc_id, {pathname: true}, @callback
+
+			it "should call the project locator", ->
+				@projectLocator.findElement
+					.calledWith({project_id: project_id, element_id: doc_id, type: 'doc'})
+					.should.equal true
+
+			it "should call the docstore", ->
+				@DocstoreManager.getDoc
+					.calledWith(project_id, doc_id)
+					.should.equal true
+
+			it "should return the pathname if option given", ->
+				@callback.calledWith(null, @lines, @rev, @version, @ranges, @path.fileSystem).should.equal true
 
-		it "should call the callback with the lines, version and rev", ->
-			@callback.calledWith(null, @lines, @rev, @version, @ranges).should.equal true
 
 	describe 'addDoc', ->
 		beforeEach ->
@@ -876,7 +901,7 @@ describe 'ProjectEntityHandler', ->
 						path: path
 					})
 					.should.equal true
-	
+
 	describe "setRootDoc", ->
 		it "should call Project.update", ->
 			@project_id = "project-id-123234adfs"
@@ -907,22 +932,22 @@ describe 'ProjectEntityHandler', ->
 
 		it 'should copy the file in FileStoreHandler', (done)->
 			@ProjectEntityHandler._putElement = sinon.stub().callsArgWith(4, null, {path:{fileSystem:"somehintg"}})
-			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)=>     
+			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)=>
 				@FileStoreHandler.copyFile.calledWith(oldProject_id, oldFileRef._id, project_id, fileRef._id).should.equal true
 				done()
 
 		it 'should put file into folder by calling put element', (done)->
-			@ProjectEntityHandler._putElement = (passedProject, passedFolder_id, passedFileRef, passedType, callback)-> 
+			@ProjectEntityHandler._putElement = (passedProject, passedFolder_id, passedFileRef, passedType, callback)->
 				passedProject._id.should.equal project_id
 				passedFolder_id.should.equal folder_id
 				passedFileRef.name.should.equal fileName
 				passedType.should.equal 'file'
 				done()
 
-			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)->     
+			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)->
 
 		it 'should return doc and parent folder', (done)->
-			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)->     
+			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)->
 				parentFolder.should.equal folder_id
 				fileRef.name.should.equal fileName
 				done()
@@ -942,7 +967,7 @@ describe 'ProjectEntityHandler', ->
 				options.rev.should.equal 0
 				done()
 
-			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)->     
+			@ProjectEntityHandler.copyFileFromExistingProject project_id, folder_id, oldProject_id, oldFileRef, (err, fileRef, parentFolder)->
 
 
 	describe "renameEntity", ->
@@ -1054,7 +1079,7 @@ describe 'ProjectEntityHandler', ->
 			@folder =
 				_id: ObjectId()
 				name: "someFolder"
-			@doc = 
+			@doc =
 				_id: ObjectId()
 				name: "new.tex"
 			@path = mongo: "mongo.path", fileSystem: "/file/system/old.tex"
@@ -1064,7 +1089,7 @@ describe 'ProjectEntityHandler', ->
 
 
 		describe "updating the project", ->
-			
+
 
 			it "should use the correct mongo path", (done)->
 				@ProjectEntityHandler._putElement @project, @folder._id, @doc, "docs", (err)=>
@@ -1089,12 +1114,12 @@ describe 'ProjectEntityHandler', ->
 					done()
 
 			it "should error if the element has no _id", (done)->
-				doc = 
+				doc =
 					name:"something"
 				@ProjectEntityHandler._putElement @project, @folder._id, doc, "doc", (err)=>
 					@ProjectModel.update.called.should.equal false
 					done()
-			
+
 
 
 		describe "_countElements", ->
@@ -1109,7 +1134,7 @@ describe 'ProjectEntityHandler', ->
 						fileRefs:{}
 						folders: [
 							{
-								docs:[_id:1234], 
+								docs:[_id:1234],
 								fileRefs:[{_id:23123}, {_id:123213}, {_id:2312}]
 								folders:[
 									{
@@ -1131,7 +1156,7 @@ describe 'ProjectEntityHandler', ->
 							}
 						]
 					}
-				]				
+				]
 
 			it "should return the correct number", (done)->
 				@ProjectEntityHandler._countElements @project, (err, count)->
@@ -1142,19 +1167,19 @@ describe 'ProjectEntityHandler', ->
 				@project.rootFolder[0].folders[0].folders = undefined
 				@ProjectEntityHandler._countElements @project, (err, count)->
 					count.should.equal 17
-					done()				
+					done()
 
 			it "should deal with null docs", (done)->
 				@project.rootFolder[0].folders[0].docs = undefined
 				@ProjectEntityHandler._countElements @project, (err, count)->
 					count.should.equal 23
-					done()				
+					done()
 
 			it "should deal with null fileRefs", (done)->
 				@project.rootFolder[0].folders[0].folders[0].fileRefs = undefined
 				@ProjectEntityHandler._countElements @project, (err, count)->
 					count.should.equal 23
-					done()	
+					done()
 
 
 

services/web/test/UnitTests/coffee/Uploads/ArchiveManagerTests.coffee

@@ -10,24 +10,22 @@ describe "ArchiveManager", ->
 	beforeEach ->
 		@logger =
 			error: sinon.stub()
+			warn: sinon.stub()
 			err:->
 			log: sinon.stub()
-		@process = new events.EventEmitter
-		@process.stdout = new events.EventEmitter
-		@process.stderr = new events.EventEmitter
-
-		@child =
-			spawn: sinon.stub().returns(@process)
-
-
 		@metrics =
 			Timer: class Timer
 				done: sinon.stub()
+		@zipfile = new events.EventEmitter
+		@zipfile.readEntry = sinon.stub()
+		@zipfile.close = sinon.stub()
+
 		@ArchiveManager = SandboxedModule.require modulePath, requires:
-			"child_process": @child
+			"yauzl": @yauzl = {open: sinon.stub().callsArgWith(2, null, @zipfile)}
 			"logger-sharelatex": @logger
 			"metrics-sharelatex": @metrics
 			"fs": @fs = {}
+			"fs-extra": @fse = {}
 	
 	describe "extractZipArchive", ->
 		beforeEach ->
@@ -39,10 +37,10 @@ describe "ArchiveManager", ->
 		describe "successfully", ->
 			beforeEach (done) ->
 				@ArchiveManager.extractZipArchive @source, @destination, done
-				@process.emit "close"
+				@zipfile.emit "end"
 
-			it "should run unzip", ->
-				@child.spawn.calledWithExactly("unzip", [@source, "-d", @destination]).should.equal true
+			it "should run yauzl", ->
+				@yauzl.open.calledWith(@source).should.equal true
 
 			it "should time the unzip", ->
 				@metrics.Timer::done.called.should.equal true
@@ -50,13 +48,12 @@ describe "ArchiveManager", ->
 			it "should log the unzip", ->
 				@logger.log.calledWith(sinon.match.any, "unzipping file").should.equal true
 
-		describe "with an error on stderr", ->
+		describe "with an error in the zip file header", ->
 			beforeEach (done) ->
+				@yauzl.open = sinon.stub().callsArgWith(2, new Error("Something went wrong"))
 				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
 					@callback(error)
 					done()
-				@process.stderr.emit "data", "Something went wrong"
-				@process.emit "close"
 
 			it "should return the callback with an error", ->
 				@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
@@ -74,60 +71,177 @@ describe "ArchiveManager", ->
 			it "should return the callback with an error", ->
 				@callback.calledWithExactly(new Error("zip_too_large")).should.equal true
 
-			it "should not call spawn", ->
-				@child.spawn.called.should.equal false
+			it "should not call yauzl.open", ->
+				@yauzl.open.called.should.equal false
 
-		describe "with an error on the process", ->
+		describe "with an error in the extracted files", ->
 			beforeEach (done) ->
 				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
 					@callback(error)
 					done()
-				@process.emit "error", new Error("Something went wrong")
+				@zipfile.emit "error", new Error("Something went wrong")
 
 			it "should return the callback with an error", ->
 				@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
 
 			it "should log out the error", ->
 				@logger.error.called.should.equal true
-	
+
+		describe "with a relative extracted file path", ->
+			beforeEach (done) ->
+				@zipfile.openReadStream = sinon.stub()
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+				@zipfile.emit "entry", {fileName: "../testfile.txt"}
+				@zipfile.emit "end"
+
+			it "should not write try to read the file entry", ->
+				@zipfile.openReadStream.called.should.equal false
+
+			it "should log out a warning", ->
+				@logger.warn.called.should.equal true
+
+		describe "with an unnormalized extracted file path", ->
+			beforeEach (done) ->
+				@zipfile.openReadStream = sinon.stub()
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+				@zipfile.emit "entry", {fileName: "foo/./testfile.txt"}
+				@zipfile.emit "end"
+
+			it "should not write try to read the file entry", ->
+				@zipfile.openReadStream.called.should.equal false
+
+			it "should log out a warning", ->
+				@logger.warn.called.should.equal true
+
+		describe "with a directory entry", ->
+			beforeEach (done) ->
+				@zipfile.openReadStream = sinon.stub()
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+				@zipfile.emit "entry", {fileName: "testdir/"}
+				@zipfile.emit "end"
+
+			it "should not write try to read the entry", ->
+				@zipfile.openReadStream.called.should.equal false
+
+			it "should not log out a warning", ->
+				@logger.warn.called.should.equal false
+
+		describe "with an error opening the file read stream", ->
+			beforeEach (done) ->
+				@zipfile.openReadStream = sinon.stub().callsArgWith(1, new Error("Something went wrong"))
+				@writeStream = new events.EventEmitter
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+				@zipfile.emit "entry", {fileName: "testfile.txt"}
+				@zipfile.emit "end"
+
+			it "should return the callback with an error", ->
+				@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
+
+			it "should log out the error", ->
+				@logger.error.called.should.equal true
+
+			it "should close the zipfile", ->
+				@zipfile.close.called.should.equal true
+
+		describe "with an error in the file read stream", ->
+			beforeEach (done) ->
+				@readStream = new events.EventEmitter
+				@readStream.pipe = sinon.stub()
+				@zipfile.openReadStream = sinon.stub().callsArgWith(1, null, @readStream)
+				@writeStream = new events.EventEmitter
+				@fs.createWriteStream = sinon.stub().returns @writeStream
+				@fse.ensureDir = sinon.stub().callsArg(1)
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+				@zipfile.emit "entry", {fileName: "testfile.txt"}
+				@readStream.emit "error", new Error("Something went wrong")
+				@zipfile.emit "end"
+
+			it "should return the callback with an error", ->
+				@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
+
+			it "should log out the error", ->
+				@logger.error.called.should.equal true
+
+			it "should close the zipfile", ->
+				@zipfile.close.called.should.equal true
+
+		describe "with an error in the file write stream", ->
+			beforeEach (done) ->
+				@readStream = new events.EventEmitter
+				@readStream.pipe = sinon.stub()
+				@readStream.unpipe = sinon.stub()
+				@readStream.destroy = sinon.stub()
+				@zipfile.openReadStream = sinon.stub().callsArgWith(1, null, @readStream)
+				@writeStream = new events.EventEmitter
+				@fs.createWriteStream = sinon.stub().returns @writeStream
+				@fse.ensureDir = sinon.stub().callsArg(1)
+				@ArchiveManager.extractZipArchive @source, @destination, (error) =>
+					@callback(error)
+					done()
+				@zipfile.emit "entry", {fileName: "testfile.txt"}
+				@writeStream.emit "error", new Error("Something went wrong")
+				@zipfile.emit "end"
+
+			it "should return the callback with an error", ->
+				@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
+
+			it "should log out the error", ->
+				@logger.error.called.should.equal true
+
+			it "should unpipe from the readstream", ->
+				@readStream.unpipe.called.should.equal true
+
+			it "should destroy the readstream", ->
+				@readStream.destroy.called.should.equal true
+
+			it "should close the zipfile", ->
+				@zipfile.close.called.should.equal true
+
 	describe "_isZipTooLarge", ->
-		beforeEach ->
-			@output = (totalSize)->"  Length     Date   Time    Name \n--------    ----   ----    ---- \n241  03-12-16 12:20   main.tex \n108801  03-12-16 12:20   ddd/x1J5kHh.jpg \n--------                   ------- \n#{totalSize}                   2 files\n"
 
 		it "should return false with small output", (done)->
 			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
 				isTooLarge.should.equal false
 				done()
-			@process.stdout.emit "data", @output("109042")
-			@process.emit "close"
+			@zipfile.emit "entry", {uncompressedSize: 109042}
+			@zipfile.emit "end"
 
 		it "should return true with large bytes", (done)->
 			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
 				isTooLarge.should.equal true
 				done()
-			@process.stdout.emit "data", @output("1090000000000000042")
-			@process.emit "close"
+			@zipfile.emit "entry", {uncompressedSize: 1090000000000000042}
+			@zipfile.emit "end"
 
 		it "should return error on no data", (done)->
 			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
 				expect(error).to.exist
 				done()
-			@process.stdout.emit "data", ""
-			@process.emit "close"
+			@zipfile.emit "entry", {}
+			@zipfile.emit "end"
 
 		it "should return error if it didn't get a number", (done)->
 			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
 				expect(error).to.exist
 				done()
-			@process.stdout.emit "data", @output("total_size_string")
-			@process.emit "close"
+			@zipfile.emit "entry", {uncompressedSize:"random-error"}
+			@zipfile.emit "end"
 
-		it "should return error if the is only a bit of data", (done)->
+		it "should return error if there is no data", (done)->
 			@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
 				expect(error).to.exist
 				done()
-			@process.stdout.emit "data", "  Length     Date   Time    Name \n--------"
-			@process.emit "close"
+			@zipfile.emit "end"
 
 	describe "findTopLevelDirectory", ->
 		beforeEach ->

services/web/test/UnitTests/coffee/User/UserInfoControllerTests.coffee

@@ -12,9 +12,9 @@ ObjectId = require("mongojs").ObjectId
 
 describe "UserInfoController", ->
 	beforeEach ->
-		@UserDeleter = 
+		@UserDeleter =
 			deleteUser: sinon.stub().callsArgWith(1)
-		@UserUpdater = 
+		@UserUpdater =
 			updatePersonalInfo: sinon.stub()
 		@sanitizer = escape:(v)->v
 		sinon.spy @sanitizer, "escape"
@@ -50,23 +50,47 @@ describe "UserInfoController", ->
 				.should.equal true
 
 	describe "getPersonalInfo", ->
-		beforeEach ->
-			@user_id = ObjectId().toString()
-			@user =
-				_id: ObjectId(@user_id)
-			@req.params = user_id: @user_id
-
-		describe "when the user exists", ->
+		describe "when the user exists with sharelatex id", ->
 			beforeEach ->
+				@user_id = ObjectId().toString()
+				@user =
+					_id: ObjectId(@user_id)
+				@req.params = user_id: @user_id
 				@UserGetter.getUser = sinon.stub().callsArgWith(2, null, @user)
 				@UserInfoController.sendFormattedPersonalInfo = sinon.stub()
 				@UserInfoController.getPersonalInfo(@req, @res, @next)
 
 			it "should look up the user in the database", ->
 				@UserGetter.getUser
-					.calledWith(@user_id, { _id: true, first_name: true, last_name: true, email: true })
+					.calledWith(
+						{ _id: ObjectId(@user_id) },
+						{ _id: true, first_name: true, last_name: true, email: true }
+					).should.equal true
+
+			it "should send the formatted details back to the client", ->
+				@UserInfoController.sendFormattedPersonalInfo
+					.calledWith(@user, @res, @next)
 					.should.equal true
-				
+
+		describe "when the user exists with overleaf id", ->
+			beforeEach ->
+				@user_id = 12345
+				@user =
+					_id: ObjectId()
+					overleaf:
+						id: @user_id
+				@req.params = user_id: @user_id.toString()
+				@UserGetter.getUser = sinon.stub().callsArgWith(2, null, @user)
+				@UserInfoController.sendFormattedPersonalInfo = sinon.stub()
+				@UserInfoController.getPersonalInfo(@req, @res, @next)
+
+			it "should look up the user in the database", ->
+				@UserGetter.getUser
+					.calledWith(
+						{ "overleaf.id": @user_id },
+						{ _id: true, first_name: true, last_name: true, email: true }
+					).should.equal true
+
 			it "should send the formatted details back to the client", ->
 				@UserInfoController.sendFormattedPersonalInfo
 					.calledWith(@user, @res, @next)
@@ -74,13 +98,24 @@ describe "UserInfoController", ->
 
 		describe "when the user does not exist", ->
 			beforeEach ->
+				@user_id = ObjectId().toString()
+				@req.params = user_id: @user_id
 				@UserGetter.getUser = sinon.stub().callsArgWith(2, null, null)
-				@UserInfoController.sendFormattedPersonalInfo = sinon.stub()
 				@UserInfoController.getPersonalInfo(@req, @res, @next)
 
 			it "should return 404 to the client", ->
 				@res.statusCode.should.equal 404
 
+		describe "when the user id is invalid", ->
+			beforeEach ->
+				@user_id = "invalid"
+				@req.params = user_id: @user_id
+				@UserGetter.getUser = sinon.stub().callsArgWith(2, null, null)
+				@UserInfoController.getPersonalInfo(@req, @res, @next)
+
+			it "should return 400 to the client", ->
+				@res.statusCode.should.equal 400
+
 	describe "sendFormattedPersonalInfo", ->
 		beforeEach ->
 			@user =

services/web/test/acceptance/coffee/AuthorizationTests.coffee

@@ -4,6 +4,9 @@ User = require "./helpers/User"
 request = require "./helpers/request"
 settings = require "settings-sharelatex"
 
+MockDocstoreApi = require './helpers/MockDocstoreApi'
+MockDocUpdaterApi = require './helpers/MockDocUpdaterApi'
+
 try_read_access = (user, project_id, test, callback) ->
 	async.series [
 		(cb) ->

services/web/test/acceptance/coffee/helpers/MockDocUpdaterApi.coffee

@@ -0,0 +1,15 @@
+express = require("express")
+app = express()
+
+module.exports = MockDocUpdaterApi =
+	run: () ->
+		app.post "/project/:project_id/flush", (req, res, next) =>
+			res.sendStatus 200
+
+		app.listen 3003, (error) ->
+			throw error if error?
+		.on "error", (error) ->
+			console.error "error starting MockDocUpdaterApi:", error.message
+			process.exit(1)
+
+MockDocUpdaterApi.run()

services/web/test/acceptance/coffee/helpers/MockDocstoreApi.coffee

@@ -0,0 +1,33 @@
+express = require("express")
+bodyParser = require "body-parser"
+app = express()
+
+module.exports = MockDocStoreApi =
+	docs: {}
+
+	run: () ->
+		app.post "/project/:project_id/doc/:doc_id", bodyParser.json(), (req, res, next) =>
+			{project_id, doc_id} = req.params
+			{lines, version, ranges} = req.body
+			@docs[project_id] ?= {}
+			@docs[project_id][doc_id] = {lines, version, ranges}
+			@docs[project_id][doc_id].rev ?= 0
+			@docs[project_id][doc_id].rev += 1
+			res.json {
+				modified: true
+				rev: @docs[project_id][doc_id].rev
+			}
+		
+		app.get "/project/:project_id/doc", (req, res, next) =>
+			docs = (doc for doc_id, doc of @docs[req.params.project_id])
+			res.send JSON.stringify docs
+
+		app.listen 3016, (error) ->
+			throw error if error?
+		.on "error", (error) ->
+			console.error "error starting MockDocStoreApi:", error.message
+			process.exit(1)
+
+
+MockDocStoreApi.run()
+

services/web/test/acceptance/scripts/full-test.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+# If you're running on OS X, you probably need to manually
+# 'rm -r node_modules/bcrypt; npm install bcrypt' inside
+# the docker container, before it will start.
+# npm rebuild bcrypt
+
+echo ">> Starting server..."
+
+grunt --no-color forever:app:start
+
+echo ">> Server started"
+
+sleep 5
+
+echo ">> Running acceptance tests..."
+grunt --no-color mochaTest:acceptance
+_test_exit_code=$?
+
+echo ">> Killing server"
+
+grunt --no-color forever:app:stop
+
+echo ">> Done"
+
+exit $_test_exit_code