Flush changes to mongo before sending request to references service

services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee

@@ -12,7 +12,7 @@ module.exports = ReferencesSearchController =
 		if (!docIds or (!(docIds instanceof Array) and (docIds != 'ALL')))
 			logger.err {projectId, docIds}, "docIds is not valid, should be either Array or String 'ALL'"
 			return res.send 400
-		logger.log {projectId, docIds}, "index references for project"
+		logger.log {projectId, docIds: docIds}, "index references for project"
 		ReferencesSearchHandler.index projectId, docIds, (err, data) ->
 			if err
 				logger.err {err, projectId}, "error indexing references"

services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee

@@ -2,6 +2,7 @@ logger = require("logger-sharelatex")
 request = require("request")
 settings = require("settings-sharelatex")
 Project = require("../../models/Project").Project
+DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
 U = require('underscore')
 Async = require('async')
 
@@ -46,22 +47,32 @@ module.exports = ReferencesSearchHandler =
 				if err
 					logger.err {err, projectId}, "error checking whether to do full index"
 					return callback(err)
-				bibDocUrls = docIds.map (docId) ->
-					ReferencesSearchHandler._buildDocUrl projectId, docId
-				logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service"
-				request.post {
-					url: "#{settings.apis.references.url}/project/#{projectId}/index"
-					json:
-						docUrls: bibDocUrls
-						fullIndex: isFullIndex
-				}, (err, res, data) ->
-					if err
-						logger.err {err, projectId}, "error communicating with references api"
-						return callback(err)
-					if 200 <= res.statusCode < 300
-						logger.log {projectId}, "got keys from references api"
-						return callback(null, data)
-					else
-						err = new Error("references api responded with non-success code: #{res.statusCode}")
-						logger.log {err, projectId}, "error updating references"
-						return callback(err)
+				# TODO: flush documents to mongo
+				logger.log {projectId, docIds}, 'flushing docs to mongo before calling references service'
+				Async.series(
+					docIds.map((docId) -> (cb) -> DocumentUpdaterHandler.flushDocToMongo(projectId, docId, cb)),
+					(err) ->
+						# continue
+						if err
+							logger.err {err, projectId, docIds}, "error flushing docs to mongo"
+							return callback(err)
+						bibDocUrls = docIds.map (docId) ->
+							ReferencesSearchHandler._buildDocUrl projectId, docId
+						logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service"
+						request.post {
+							url: "#{settings.apis.references.url}/project/#{projectId}/index"
+							json:
+								docUrls: bibDocUrls
+								fullIndex: isFullIndex
+						}, (err, res, data) ->
+							if err
+								logger.err {err, projectId}, "error communicating with references api"
+								return callback(err)
+							if 200 <= res.statusCode < 300
+								logger.log {projectId}, "got keys from references api"
+								return callback(null, data)
+							else
+								err = new Error("references api responded with non-success code: #{res.statusCode}")
+								logger.log {err, projectId}, "error updating references"
+								return callback(err)
+				)

services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee

@@ -11,13 +11,16 @@ define [
 					if entity?.name?.match /.*\.bib$/
 						@indexReferences([doc.doc_id], true)
 
+			# When we join the project:
+			#   index all references files
+			#   and don't broadcast to all clients
 			@$scope.$on 'project:joined', (e) =>
 				@indexReferences("ALL", false)
 
 			setTimeout(
 				(self) ->
 					self.ide.socket.on 'references:keys:updated', (keys) ->
-						console.log '>> got keys from socket'
+						# console.log '>> got keys from socket'
 						self._storeReferencesKeys(keys)
 				, 100
 				, this
@@ -26,8 +29,11 @@ define [
 		_storeReferencesKeys: (newKeys) ->
 			if window._ENABLE_REFERENCES_AUTOCOMPLETE != true
 				return
-			console.log '>> storing references keys'
-			@$scope.$root._references.keys = newKeys
+			# console.log '>> storing references keys'
+			oldKeys = @$scope.$root._references.keys
+			console.log "#{oldKeys.length} + #{newKeys.length}"
+			@$scope.$root._references.keys = _.union(oldKeys, newKeys)
+			console.log "end>> #{@$scope.$root._references.keys.length}"
 
 		# docIds: List[String]|String('ALL'), shouldBroadcast: Bool
 		indexReferences: (docIds, shouldBroadcast) ->
@@ -39,6 +45,6 @@ define [
 				"/project/#{@$scope.project_id}/references/index",
 				opts,
 				(data) =>
-					console.log ">> got keys ", data
+					# console.log ">> got keys ", data
 					@_storeReferencesKeys(data.keys)
 			)

services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee

@@ -44,6 +44,9 @@ describe 'ReferencesSearchHandler', ->
 					findPopulatedById: sinon.stub().callsArgWith(1, null, @fakeProject)
 				}
 			}
+			'../DocumentUpdater/DocumentUpdaterHandler': @DocumentUpdaterHandler = {
+				flushDocToMongo: sinon.stub().callsArgWith(2, null)
+			}
 		@fakeResponseData =
 			projectId: @projectId
 			keys: ['k1', 'k2']
@@ -73,6 +76,13 @@ describe 'ReferencesSearchHandler', ->
 					@Project.findPopulatedById.calledWith(@projectId).should.equal true
 					done()
 
+			it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) ->
+				@call (err, data) =>
+					@DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2
+					@docIds.forEach (docId) =>
+						@DocumentUpdaterHandler.flushDocToMongo.calledWith(@projectId, docId).should.equal true
+					done()
+
 			it 'should make a request to references service', (done) ->
 				@call (err, data) =>
 					@request.post.callCount.should.equal 1
@@ -106,6 +116,11 @@ describe 'ReferencesSearchHandler', ->
 					@handler._findBibDocIds.calledWith(@fakeProject).should.equal true
 					done()
 
+			it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) ->
+				@call (err, data) =>
+					@DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2
+					done()
+
 			it 'should not produce an error', (done) ->
 				@call (err, data) =>
 					expect(err).to.equal null
@@ -153,11 +168,32 @@ describe 'ReferencesSearchHandler', ->
 					@request.post.callCount.should.equal 0
 					done()
 
+		describe 'when flushDocToMongo produces an error', ->
+
+			beforeEach ->
+				@Project.findPopulatedById.callsArgWith(1, null, @fakeProject)
+				@handler._isFullIndex.callsArgWith(1, false)
+				@DocumentUpdaterHandler.flushDocToMongo.callsArgWith(2, new Error('woops'))
+
+			it 'should produce an error', (done) ->
+				@call (err, data) =>
+					expect(err).to.not.equal null
+					expect(err).to.be.instanceof Error
+					expect(data).to.equal undefined
+					done()
+
+			it 'should not send request', (done) ->
+				@call (err, data) =>
+					@request.post.callCount.should.equal 0
+					done()
+
+
 		describe 'when request produces an error', ->
 
 			beforeEach ->
 				@Project.findPopulatedById.callsArgWith(1, null, @fakeProject)
 				@handler._isFullIndex.callsArgWith(1, null, false)
+				@DocumentUpdaterHandler.flushDocToMongo.callsArgWith(2, null)
 				@request.post.callsArgWith(1, new Error('woops'))
 
 			it 'should produce an error', (done) ->