wip: change how indexing of references works.

services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchController.coffee

@@ -37,13 +37,17 @@ module.exports = ReferencesSearchController =
 				return res.send 500
 			return res.json data
 
-	loadReferencesKeys: (req, res) ->
-		project_id = req.params.Project_id
+	index: (req, res) ->
+		projectId = req.params.Project_id
 		shouldBroadcast = req.body.shouldBroadcast
-		logger.log {project_id}, "loading project references keys"
-		ReferencesSearchHandler.loadReferencesKeys project_id, (err, data) ->
+		docIds = req.body.docIds
+		if (not docIds instanceof Array) and (docIds != "ALL")
+			logger.err {projectId, docIds}, "docIds is not valid, should be either Array or String 'ALL'"
+			return res.send 400
+		logger.log {projectId, docIds}, "index references for project"
+		ReferencesSearchHandler.index projectId, docIds, (err, data) ->
 			if err
-				logger.err {err, project_id}, "error getting references keys"
+				logger.err {err, projectId}, "error indexing references"
 				return res.send 500
 			# TODO: optionally broadcast to all connected clients
 			return res.json data

services/web/app/coffee/Features/ReferencesSearch/ReferencesSearchHandler.coffee

@@ -29,35 +29,40 @@ module.exports = ReferencesSearchHandler =
 		return ids
 
 	_isFullIndex: (project, callback = (err, result) ->) ->
-		UserGetter.getUser project.owner_ref, {features: 1}, (err, owner) ->
-			return callback(err) if err
-			callback(null, owner.features.references == true)
+		owner = project.owner_ref
+		callback(null, owner.features.references == true)
 
-	loadReferencesKeys: (projectId, callback=(err, data)->) ->
-		logger.log {projectId}, "load references keys for project"
+	# projectId: String, docIds: List[String]|Null
+	index: (projectId, docIds, callback=(err, data)->) ->
 		Project.findPopulatedById projectId, (err, project) ->
 			if err
+				logger.err {err, projectId}, "error finding project"
 				return callback(err)
+			if docIds == "ALL"
+				logger.log {projectId}, "indexing all bib files in project"
+				docIds = ReferencesSearchHandler._findBibDocIds(project)
 			ReferencesSearchHandler._isFullIndex project, (err, isFullIndex) ->
 				if err
+					logger.err {err, projectId}, "error checking whether to do full index"
 					return callback(err)
-				bibDocIds = ReferencesSearchHandler._findBibDocIds(project)
-				bibDocUrls = bibDocIds.map (docId) ->
+				bibDocUrls = docIds.map (docId) ->
 					ReferencesSearchHandler._buildDocUrl projectId, docId
-				logger.log {projectId, isFullIndex, bibDocIds}, "sending request to references service"
+				logger.log {projectId, isFullIndex, docIds}, "sending request to references service"
 				request.post {
-					url: "#{settings.apis.references.url}/project/#{projectId}/loadreferenceskeys"
+					url: "#{settings.apis.references.url}/project/#{projectId}/index"
 					json:
 						docUrls: bibDocUrls
 						fullIndex: isFullIndex
-				}, (err, res, result) ->
+				}, (err, res, data) ->
 					if err
+						logger.err {err, projectId}, "error communicating with references api"
 						return callback(err)
 					if 200 <= res.statusCode < 300
-						return callback(null)
+						logger.log {projectId}, "got keys from references api"
+						return callback(null, data)
 					else
 						err = new Error("references api responded with non-success code: #{res.statusCode}")
-						logger.log {err, projectId, fileUrl}, "error updating references"
+						logger.log {err, projectId}, "error updating references"
 						return callback(err)
 
 	## ## ## ##

services/web/app/coffee/router.coffee

@@ -173,6 +173,7 @@ module.exports = class Router
 
 		webRouter.post "/project/:Project_id/references", SecurityManager.requestCanAccessProject, ReferencesSearchController.indexFile
 		webRouter.get "/project/:Project_id/references/keys", SecurityManager.requestCanAccessProject, ReferencesSearchController.getKeys
+		webRouter.post "/project/:Project_id/references/index", SecurityManager.requestCanAccessProject, ReferencesSearchController.index
 
 		#Admin Stuff
 		webRouter.get  '/admin', SecurityManager.requestIsAdmin, AdminController.index

services/web/public/coffee/ide/references-search/ReferencesSearchManager.coffee

@@ -10,41 +10,29 @@ define [
 				 	entity = @ide.fileTreeManager.findEntityById doc.doc_id
 					if entity?.name?.match /.*\.bib$/
 						@$scope.$emit 'references:changed', entity
-						@indexReferences doc.doc_id
+						@indexReferences([doc.doc_id], true)
 
 			@$scope.$on 'project:joined', (e) =>
-				@loadReferencesKeys()
+				@indexReferences("ALL", false)
 
-		loadReferencesKeys: () ->
+		# docIds: List[String]|String('ALL'), shouldBroadcast: Bool
+		indexReferences: (docIds, shouldBroadcast) ->
 			if window._ENABLE_REFERENCES_AUTOCOMPLETE != true
 				return
+			opts =
+				docIds: docIds
+				shouldBroadcast: shouldBroadcast
+				_csrf: window.csrfToken
+			console.log ">>", opts
 			$.post(
-				"/project/#{@$scope.project_id}/referenceskeys",
-				{
-					shouldBroadcast: false
-					_csrf: window.csrfToken
-				},
+				"/project/#{@$scope.project_id}/references/index",
+				opts,
 				(data) =>
-					console.log ">> ", data
-			)
-
-		indexReferences: (doc_id) ->
-			if window._ENABLE_REFERENCES_AUTOCOMPLETE != true
-				return
-			$.post(
-				"/project/#{@$scope.project_id}/references",
-				{
-					docId: doc_id,
-					_csrf: window.csrfToken
-				},
-				(data) =>
-					setTimeout(
-						( () -> @getReferenceKeys() ).bind(this),
-						500
-					)
+					console.log ">> done ", data
+					@$scope.$root._references.keys = data.keys
 			)
 
-		getReferenceKeys: (callback) ->
+		getReferenceKeys: (callback=(keys)->) ->
 			if window._ENABLE_REFERENCES_AUTOCOMPLETE != true
 				return
 			$.get(

services/web/test/UnitTests/coffee/ReferencesSearch/ReferencesSearchHandlerTests.coffee

@@ -7,115 +7,115 @@ modulePath = "../../../../app/js/Features/ReferencesSearch/ReferencesSearchHandl
 
 describe 'ReferencesSearchHandler', ->
 
-	beforeEach ->
-		@project_id = '222'
-		@file_id = '111111'
-		@handler = SandboxedModule.require modulePath, requires:
-			'logger-sharelatex': {
-				log: ->
-				err: ->
-			}
-			'settings-sharelatex': @settings = {
-				apis:
-					references: {url: 'http://some.url'}
-					web: {url: 'http://some.url'}
-			}
-			'request': @request = {
-				get: sinon.stub()
-				post: sinon.stub()
-			}
-			'../../models/Project': @Project = {
-				Project: {
-					findById: sinon.stub().callsArgWith(2, null, {owner_ref: '111'})
-				}
-			}
-			'../User/UserGetter': @UserGetter = {
-				getUser: sinon.stub().callsArgWith(2, null, {features: {references: false}})
-			}
-
-	describe 'indexFile', ->
-
-		describe 'full index or not', ->
-
-			beforeEach ->
-				@request.post.callsArgWith(1, null, {statusCode: 200}, {})
-
-			describe 'when full index is not required', ->
-
-				beforeEach ->
-					@UserGetter.getUser.callsArgWith(2, null, {features: {references: false}})
-
-				it 'should set fullIndex to true', (done) ->
-					@handler.indexFile @project_id, @file_id, (err) =>
-						@request.post.calledOnce.should.equal true
-						options = @request.post.firstCall.args[0]
-						options.json.fullIndex.should.equal false
-						done()
-
-			describe 'when full index is required', ->
-
-				beforeEach ->
-					@UserGetter.getUser.callsArgWith(2, null, {features: {references: true}})
-
-				it 'should set fullIndex to true', (done) ->
-					@handler.indexFile @project_id, @file_id, (err) =>
-						@request.post.calledOnce.should.equal true
-						options = @request.post.firstCall.args[0]
-						options.json.fullIndex.should.equal true
-						done()
-
-		describe 'when index operation is successful', ->
-			beforeEach ->
-				@request.post.callsArgWith(1, null, {statusCode: 201}, {})
-
-			it 'should not produce an error', (done) ->
-				@handler.indexFile @project_id, @file_id, (err) =>
-					expect(err).to.equal null
-					@request.post.calledOnce.should.equal true
-					options = @request.post.firstCall.args[0]
-					options.json.fullIndex.should.equal false
-					options.json.referencesUrl.should.not.be.undefined
-					options.url.should.not.be.undefined
-					done()
-
-		describe 'when index operation fails', ->
-			beforeEach ->
-				@request.post.callsArgWith(1, null, {statusCode: 500}, {})
-
-			it 'should produce an error', (done) ->
-				@handler.indexFile @project_id, @file_id, (err) =>
-					expect(err).to.not.equal null
-					done()
-
-	describe 'getKeys', ->
-
-		describe 'when request is successful', ->
-			beforeEach ->
-				@data =
-					projectId: @projectId
-					keys: ['a', 'b', 'c']
-				@request.get.callsArgWith(1, null, {statusCode: 200}, @data)
-
-			it 'should not produce an error', ->
-				@handler.getKeys @project_id, (err, result) =>
-					expect(err).to.equal null
-
-			it 'should produce a result object', ->
-				@handler.getKeys @project_id, (err, result) =>
-					expect(result).to.not.equal null
-					expect(result).to.deep.equal @data
-
-		describe 'when request fails', ->
-			beforeEach ->
-				@data =
-					projectId: @project_Id
-					keys: ['a', 'b', 'c']
-				@request.get.callsArgWith(1, null, {statusCode: 500}, null)
-
-			it 'should produce an error', ->
-				@handler.getKeys @project_id, (err, result) =>
-					expect(err).to.not.equal null
-
-			it 'should not produce a result', ->
-				@handler.getKeys @project_id, (err, result) =>
-					expect(result).to.not.equal null
+	# beforeEach ->
+	# 	@project_id = '222'
+	# 	@file_id = '111111'
+	# 	@handler = SandboxedModule.require modulePath, requires:
+	# 		'logger-sharelatex': {
+	# 			log: ->
+	# 			err: ->
+	# 		}
+	# 		'settings-sharelatex': @settings = {
+	# 			apis:
+	# 				references: {url: 'http://some.url'}
+	# 				web: {url: 'http://some.url'}
+	# 		}
+	# 		'request': @request = {
+	# 			get: sinon.stub()
+	# 			post: sinon.stub()
+	# 		}
+	# 		'../../models/Project': @Project = {
+	# 			Project: {
+	# 				findById: sinon.stub().callsArgWith(2, null, {owner_ref: '111'})
+	# 			}
+	# 		}
+	# 		'../User/UserGetter': @UserGetter = {
+	# 			getUser: sinon.stub().callsArgWith(2, null, {features: {references: false}})
+	# 		}
+
+	# describe 'indexFile', ->
+
+	# 	describe 'full index or not', ->
+
+	# 		beforeEach ->
+	# 			@request.post.callsArgWith(1, null, {statusCode: 200}, {})
+
+	# 		describe 'when full index is not required', ->
+
+	# 			beforeEach ->
+	# 				@UserGetter.getUser.callsArgWith(2, null, {features: {references: false}})
+
+	# 			it 'should set fullIndex to true', (done) ->
+	# 				@handler.indexFile @project_id, @file_id, (err) =>
+	# 					@request.post.calledOnce.should.equal true
+	# 					options = @request.post.firstCall.args[0]
+	# 					options.json.fullIndex.should.equal false
+	# 					done()
+
+	# 		describe 'when full index is required', ->
+
+	# 			beforeEach ->
+	# 				@UserGetter.getUser.callsArgWith(2, null, {features: {references: true}})
+
+	# 			it 'should set fullIndex to true', (done) ->
+	# 				@handler.indexFile @project_id, @file_id, (err) =>
+	# 					@request.post.calledOnce.should.equal true
+	# 					options = @request.post.firstCall.args[0]
+	# 					options.json.fullIndex.should.equal true
+	# 					done()
+
+	# 	describe 'when index operation is successful', ->
+	# 		beforeEach ->
+	# 			@request.post.callsArgWith(1, null, {statusCode: 201}, {})
+
+	# 		it 'should not produce an error', (done) ->
+	# 			@handler.indexFile @project_id, @file_id, (err) =>
+	# 				expect(err).to.equal null
+	# 				@request.post.calledOnce.should.equal true
+	# 				options = @request.post.firstCall.args[0]
+	# 				options.json.fullIndex.should.equal false
+	# 				options.json.referencesUrl.should.not.be.undefined
+	# 				options.url.should.not.be.undefined
+	# 				done()
+
+	# 	describe 'when index operation fails', ->
+	# 		beforeEach ->
+	# 			@request.post.callsArgWith(1, null, {statusCode: 500}, {})
+
+	# 		it 'should produce an error', (done) ->
+	# 			@handler.indexFile @project_id, @file_id, (err) =>
+	# 				expect(err).to.not.equal null
+	# 				done()
+
+	# describe 'getKeys', ->
+
+	# 	describe 'when request is successful', ->
+	# 		beforeEach ->
+	# 			@data =
+	# 				projectId: @projectId
+	# 				keys: ['a', 'b', 'c']
+	# 			@request.get.callsArgWith(1, null, {statusCode: 200}, @data)
+
+	# 		it 'should not produce an error', ->
+	# 			@handler.getKeys @project_id, (err, result) =>
+	# 				expect(err).to.equal null
+
+	# 		it 'should produce a result object', ->
+	# 			@handler.getKeys @project_id, (err, result) =>
+	# 				expect(result).to.not.equal null
+	# 				expect(result).to.deep.equal @data
+
+	# 	describe 'when request fails', ->
+	# 		beforeEach ->
+	# 			@data =
+	# 				projectId: @project_Id
+	# 				keys: ['a', 'b', 'c']
+	# 			@request.get.callsArgWith(1, null, {statusCode: 500}, null)
+
+	# 		it 'should produce an error', ->
+	# 			@handler.getKeys @project_id, (err, result) =>
+	# 				expect(err).to.not.equal null
+
+	# 		it 'should not produce a result', ->
+	# 			@handler.getKeys @project_id, (err, result) =>
+	# 				expect(result).to.not.equal null