simplify incremental request to docupdater

if project state hasn't changed, get the docs from the docupdater -- we
check/set the hash and return the docs in a single request.  Otherwise
do a full request from mongo.

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -123,35 +123,40 @@ module.exports = ClsiManager =
 			if project.compiler not in ClsiManager.VALID_COMPILERS
 				project.compiler = "pdflatex"
 
-			ClsiStateManager.checkProjectStateMatch project_id, project, (error, projectStateUnchanged, projectStateHash) ->
+			ClsiManager.getContentFromDocUpdaterIfMatch project_id, project, (error, projectStateHash, docUpdaterDocs) ->
 				return callback(error) if error?
-				logger.log project_id: project_id, projectStateUnchanged: projectStateUnchanged, "checked project state"
+				logger.log project_id: project_id, projectStateHash: projectStateHash, docs: docUpdaterDocs?, "checked project state"
 				# see if we can send an incremental update to the CLSI
-				if projectStateUnchanged and options.syncType isnt "full"
-					ClsiManager._getContentFromDocUpdater project_id, (error, docUpdaterDocs) ->
-						return callback(error) if error?
-						ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) ->
-							return callback(error) if error?
-							docs = {}
-							for doc in docUpdaterDocs or []
-								path = docPath[doc._id]
-								docs[path] = doc
-							options.syncType = "incremental"
-							options.syncState = projectStateHash
-							# send new docs but not files as those are already on the clsi
-							ClsiManager._finaliseRequest project_id, options, project, docs, [], callback
+				if docUpdaterDocs? and options.syncType isnt "full"
+					ClsiManager._buildRequestFromDocupdater project_id, options, project, projectStateHash, docUpdaterDocs, callback
 				else
-					ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
-						return callback(error) if error?
-						# FIXME want to store state after project has been sent to
-						# clsi, but need to do it here.
-						ClsiStateManager.setProjectState project_id, project, (error, projectStateHash) ->
-							if error?
-								logger.err err:error, project_id:project_id, "error storing state in redis"
-								#return callback(error)
-							options.syncType = "full"
-							options.syncState = projectStateHash
-							ClsiManager._finaliseRequest project_id, options, project, docs, files, callback
+					ClsiManager._buildRequestFromMongo project_id, options, project, projectStateHash, callback
+
+	getContentFromDocUpdaterIfMatch: (project_id, project, callback = (error, projectStateHash, docs) ->) ->
+		ClsiStateManager.computeHash project, (error, projectStateHash) ->
+			return callback(error) if error?
+			DocumentUpdaterHandler.getProjectDocsIfMatch project_id, projectStateHash, (error, docs) ->
+				return callback(error) if error?
+				callback(null, projectStateHash, docs)
+
+	_buildRequestFromDocupdater: (project_id, options, project, projectStateHash, docUpdaterDocs, callback = (error, request) ->) ->
+		ProjectEntityHandler.getAllDocPathsFromProject project, (error, docPath) ->
+				return callback(error) if error?
+				docs = {}
+				for doc in docUpdaterDocs or []
+					path = docPath[doc._id]
+					docs[path] = doc
+				# send new docs but not files as those are already on the clsi
+				options.syncType = "incremental"
+				options.syncState = projectStateHash
+				ClsiManager._finaliseRequest project_id, options, project, docs, [], callback
+
+	_buildRequestFromMongo: (project_id, options, project, projectStateHash, callback = (error, request) ->) ->
+		ClsiManager._getContentFromMongo project_id, (error, docs, files) ->
+			return callback(error) if error?
+			options.syncType = "full"
+			options.syncState = projectStateHash
+			ClsiManager._finaliseRequest project_id, options, project, docs, files, callback
 
 	_getContentFromDocUpdater: (project_id, callback = (error, docs) ->) ->
 		DocumentUpdaterHandler.getProjectDocs project_id, (error, docs) ->

services/web/app/coffee/Features/Compile/ClsiStateManager.coffee

@@ -1,12 +1,7 @@
 Settings = require "settings-sharelatex"
-RedisWrapper = require("../../infrastructure/RedisWrapper")
-rclient = RedisWrapper.client("clsi_state")
 logger = require "logger-sharelatex"
 crypto = require "crypto"
 
-buildKey = (project_id)->
-	return "clsistate:#{project_id}"  # FIXME: should we cluster these on project??
-
 # The "state" of a project is a hash of the relevant attributes in the
 # project object in this case we only need the rootFolder.
 #
@@ -18,32 +13,15 @@ buildKey = (project_id)->
 # be updated.  If it doesn't change then we can overwrite changed docs
 # in place on the clsi, getting them from the docupdater.
 #
-# The docupdater is also responsible for unsetting the key in redis if
-# it removes any documents from the doc updater.
+# The docupdater is responsible for setting the key in redis, and
+# unsetting it if it removes any documents from the doc updater.
 
 buildState = (project) ->
 	json = JSON.stringify(project.rootFolder)
 	return crypto.createHash('sha1').update(json, 'utf8').digest('hex')
 
-clsiStateEnabled = Settings.clsiState
-
-OneHour = 3600 * 1000
-
 module.exports = ClsiStateManager =
 
-	checkProjectStateMatch: (project_id, project, callback = (err, ok) ->) ->
-		newState = buildState(project)
-		rclient.get buildKey(project_id), (err, oldState) ->
-			return callback(err) if err?
-			logger.log project_id: project_id, new_state: newState, old_state: oldState, "got project state from redis"
-			if newState is oldState
-				callback(null,true,oldState)
-			else
-				callback(null,false)
-
-	setProjectState: (project_id, project, callback = (err)->)->
-		projectState = buildState(project)
-		logger.log project_id: project_id, projectState: projectState, "setting project state in redis"
-		rclient.set buildKey(project_id), projectState, "PX", OneHour, (err) ->
-			return callback(err) if err?
-			callback(null,projectState)
+	computeHash: (project, callback = (err, hash) ->) ->
+		hash = buildState(project)
+		callback(null, hash)

services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee

@@ -123,16 +123,18 @@ module.exports = DocumentUpdaterHandler =
 				logger.error project_id:project_id, doc_id:doc_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}"
 				callback new Error("doc updater returned a non-success status code: #{res.statusCode}")
 
-	getProjectDocs: (project_id, callback = (error, docs) ->) ->
+	getProjectDocsIfMatch: (project_id, projectStateHash, callback = (error, docs) ->) ->
 		timer = new metrics.Timer("get-project-docs")
-		url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc"
+		url = "#{settings.apis.documentupdater.url}/project/#{project_id}/doc?state=#{projectStateHash}"
 		logger.log project_id:project_id, "getting project docs from document updater"
 		request.get url, (error, res, body)->
 			timer.done()
 			if error?
 				logger.error err:error, url:url, project_id:project_id, "error getting project docs from doc updater"
 				return callback(error)
-			if res.statusCode >= 200 and res.statusCode < 300
+			if res.statusCode is 409
+				return callback()
+			else if res.statusCode >= 200 and res.statusCode < 300
 				logger.log project_id:project_id, "got project docs from document document updater"
 				try
 					docs = JSON.parse(body)
@@ -140,7 +142,6 @@ module.exports = DocumentUpdaterHandler =
 						doc.lines = JSON.parse(doc.lines)
 				catch error
 					return callback(error)
-				logger.log project_id: project_id, docs: docs, "RESULT"
 				callback null, docs
 			else
 				logger.error project_id:project_id, url: url, "doc updater returned a non-success status code: #{res.statusCode}"

services/web/app/coffee/Features/Project/ProjectEntityHandler.coffee

@@ -84,7 +84,6 @@ module.exports = ProjectEntityHandler =
 			docPath = {}
 			for folderPath, folder of folders
 				for doc in (folder.docs or [])
-					console.log "PATH", path.join(folderPath, doc.name), doc._id, doc.name
 					docPath[doc._id] = path.join(folderPath, doc.name)
 			logger.log count:_.keys(docPath).length, project_id:project._id, "returning docPaths for project"
 			callback null, docPath