Merge branch 'master' into clsi-dynamic-load

services/web/app/coffee/Features/References/ReferencesHandler.coffee

@@ -16,19 +16,31 @@ module.exports = ReferencesHandler =
 	_buildDocUrl: (projectId, docId) ->
 		"#{settings.apis.docstore.url}/project/#{projectId}/doc/#{docId}/raw"
 
-	_findBibDocIds: (project) ->
+	_buildFileUrl: (projectId, fileId) ->
+		"#{settings.apis.filestore.url}/project/#{projectId}/file/#{fileId}"
+
+	_findBibFileIds: (project) ->
 		ids = []
+		_process = (folder) ->
+			(folder.fileRefs or []).forEach (file) ->
+				if file?.name?.match(/^.*\.bib$/)
+					ids.push(file._id)
+			(folder.folders or []).forEach (folder) ->
+				_process(folder)
+		(project.rootFolder or []).forEach (rootFolder) ->
+			_process(rootFolder)
+		return ids
 
+	_findBibDocIds: (project) ->
+		ids = []
 		_process = (folder) ->
 			(folder.docs or []).forEach (doc) ->
 				if doc?.name?.match(/^.*\.bib$/)
 					ids.push(doc._id)
 			(folder.folders or []).forEach (folder) ->
 				_process(folder)
-
 		(project.rootFolder or []).forEach (rootFolder) ->
 			_process(rootFolder)
-
 		return ids
 
 	_isFullIndex: (project, callback = (err, result) ->) ->
@@ -43,16 +55,17 @@ module.exports = ReferencesHandler =
 				return callback(err)
 			logger.log {projectId}, "indexing all bib files in project"
 			docIds = ReferencesHandler._findBibDocIds(project)
-			ReferencesHandler._doIndexOperation(projectId, project, docIds, callback)
+			fileIds = ReferencesHandler._findBibFileIds(project)
+			ReferencesHandler._doIndexOperation(projectId, project, docIds, fileIds, callback)
 
 	index: (projectId, docIds, callback=(err, data)->) ->
 		ProjectGetter.getProject projectId, {rootFolder: true, owner_ref: 1}, (err, project) ->
 			if err
 				logger.err {err, projectId}, "error finding project"
 				return callback(err)
-			ReferencesHandler._doIndexOperation(projectId, project, docIds, callback)
+			ReferencesHandler._doIndexOperation(projectId, project, docIds, [], callback)
 
-	_doIndexOperation: (projectId, project, docIds, callback) ->
+	_doIndexOperation: (projectId, project, docIds, fileIds, callback) ->
 		ReferencesHandler._isFullIndex project, (err, isFullIndex) ->
 			if err
 				logger.err {err, projectId}, "error checking whether to do full index"
@@ -67,11 +80,14 @@ module.exports = ReferencesHandler =
 						return callback(err)
 					bibDocUrls = docIds.map (docId) ->
 						ReferencesHandler._buildDocUrl projectId, docId
+					bibFileUrls = fileIds.map (fileId) ->
+						ReferencesHandler._buildFileUrl projectId, fileId
+					allUrls = bibDocUrls.concat(bibFileUrls)
 					logger.log {projectId, isFullIndex, docIds, bibDocUrls}, "sending request to references service"
 					request.post {
 						url: "#{settings.apis.references.url}/project/#{projectId}/index"
 						json:
-							docUrls: bibDocUrls
+							docUrls: allUrls
 							fullIndex: isFullIndex
 					}, (err, res, data) ->
 						if err

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -97,7 +97,7 @@ define [
 				$scope.pdf.qs = if qs_args.length then "?" + qs_args.join("&") else ""
 				$scope.pdf.url += $scope.pdf.qs
 
-				fetchLogs(fileByPath['output.log'])
+				fetchLogs(fileByPath['output.log'], fileByPath['output.blg'])
 
 			IGNORE_FILES = ["output.fls", "output.fdb_latexmk"]
 			$scope.pdf.outputFiles = []
@@ -116,62 +116,77 @@ define [
 						file.url = file.url + "?clsiserverid=#{response.clsiServerId}"
 					$scope.pdf.outputFiles.push file
 
-		fetchLogs = (outputFile) ->
 
-			opts =
-				method:"GET"
-				params:
-					build:outputFile.build
-					clsiserverid:ide.clsiServerId
-			if outputFile?.build?
-				opts.url = "/project/#{$scope.project_id}/build/#{outputFile.build}/output/output.log"
-			else
-				opts.url = "/project/#{$scope.project_id}/output/output.log"
-			$http opts
-				.success (log) ->
-					#console.log ">>", log
-					$scope.pdf.rawLog = log
-					logEntries = LogParser.parse(log, ignoreDuplicates: true)
-					#console.log ">>", logEntries
-					$scope.pdf.logEntries = logEntries
-					$scope.pdf.logEntries.all = logEntries.errors.concat(logEntries.warnings).concat(logEntries.typesetting)
-					# # # #
-					proceed = () ->
-						$scope.pdf.logEntryAnnotations = {}
-						for entry in logEntries.all
-							if entry.file?
-								entry.file = normalizeFilePath(entry.file)
-								entity = ide.fileTreeManager.findEntityByPath(entry.file)
-								if entity?
-									$scope.pdf.logEntryAnnotations[entity.id] ||= []
-									$scope.pdf.logEntryAnnotations[entity.id].push {
-										row: entry.line - 1
-										type: if entry.level == "error" then "error" else "warning"
-										text: entry.message
-									}
-					# Get the biber log and parse it
-					if outputFile?.build?
-						opts.url = "/project/#{$scope.project_id}/build/#{outputFile.build}/output/output.blg"
-					else
-						opts.url = "/project/#{$scope.project_id}/output/output.blg"
-					$http opts 
-						.success (log) ->
-							window._s = $scope
-							biberLogEntries = BibLogParser.parse(log, {})
-							if $scope.pdf.logEntries
-								entries = $scope.pdf.logEntries
-								all = biberLogEntries.errors.concat(biberLogEntries.warnings)
-								entries.all = entries.all.concat(all)
-								entries.errors = entries.errors.concat(biberLogEntries.errors)
-								entries.warnings = entries.warnings.concat(biberLogEntries.warnings)
-							proceed()
-						.error (e) ->
-							# it's not an error for the output.blg file to not be present
-							proceed()
-					# # # #
-				.error () ->
-					$scope.pdf.logEntries = []
-					$scope.pdf.rawLog = ""
+		fetchLogs = (logFile, blgFile) ->
+
+			getFile = (name, file) ->
+				opts =
+					method:"GET"
+					params:
+						build:file.build
+						clsiserverid:ide.clsiServerId
+				if file?.build?
+					opts.url = "/project/#{$scope.project_id}/build/#{file.build}/output/#{name}"
+				else
+					opts.url = "/project/#{$scope.project_id}/output/#{name}"
+				return $http(opts)
+
+			# accumulate the log entries
+			logEntries =
+				all: []
+				errors: []
+				warnings: []
+
+			accumulateResults = (newEntries) ->
+				for key in ['all', 'errors', 'warnings']
+					logEntries[key] = logEntries[key].concat newEntries[key]
+
+			# use the parsers for each file type
+			processLog = (log) ->
+				$scope.pdf.rawLog = log
+				{errors, warnings, typesetting} = LogParser.parse(log, ignoreDuplicates: true)
+				all = [].concat errors, warnings, typesetting
+				accumulateResults {all, errors, warnings}
+
+			processBiber = (log) ->
+				{errors, warnings} = BibLogParser.parse(log, {})
+				all = [].concat errors, warnings
+				accumulateResults {all, errors, warnings}
+
+			# output the results
+			handleError = () ->
+				$scope.pdf.logEntries = []
+				$scope.pdf.rawLog = ""
+
+			annotateFiles = () ->
+				$scope.pdf.logEntries = logEntries
+				$scope.pdf.logEntryAnnotations = {}
+				for entry in logEntries.all
+					if entry.file?
+						entry.file = normalizeFilePath(entry.file)
+						entity = ide.fileTreeManager.findEntityByPath(entry.file)
+						if entity?
+							$scope.pdf.logEntryAnnotations[entity.id] ||= []
+							$scope.pdf.logEntryAnnotations[entity.id].push {
+								row: entry.line - 1
+								type: if entry.level == "error" then "error" else "warning"
+								text: entry.message
+							}
+
+			# retrieve the logfile and process it
+			response = getFile('output.log', logFile)
+				.success processLog
+				.error handleError
+
+			if blgFile?	# retrieve the blg file if present
+				response.success () ->
+					getFile('output.blg', blgFile)
+						# ignore errors in biber file
+						.success processBiber
+						# display the combined result
+						.then annotateFiles
+			else # otherwise just display the result
+				response.success annotateFiles
 
 		getRootDocOverride_id = () ->
 			doc = ide.editorManager.getCurrentDocValue()

services/web/public/coffee/ide/references/ReferencesManager.coffee

@@ -11,6 +11,9 @@ define [
 					if entity?.name?.match /.*\.bib$/
 						@indexReferences([doc.doc_id], true)
 
+			@$scope.$on 'references:should-reindex', (e, data) =>
+				@indexAllReferences(true)
+
 			# When we join the project:
 			#   index all references files
 			#   and don't broadcast to all clients

services/web/test/UnitTests/coffee/References/ReferencesHandlerTests.coffee

@@ -21,7 +21,11 @@ describe 'ReferencesHandler', ->
 					{name: 'two.txt', _id: 'bbb'},
 				]
 				folders: [
-					{docs: [{name: 'three.bib', _id: 'ccc'}], folders: []}
+					{
+						docs: [{name: 'three.bib', _id: 'ccc'}],
+						fileRefs: [{name: 'four.bib', _id: 'ghg'}],
+						folders: []
+					}
 				]
 			]
 		@docIds = ['aaa', 'ccc']
@@ -34,6 +38,7 @@ describe 'ReferencesHandler', ->
 				apis:
 					references: {url: 'http://some.url/references'}
 					docstore: {url: 'http://some.url/docstore'}
+					filestore: {url: 'http://some.url/filestore'}
 			}
 			'request': @request = {
 				get: sinon.stub()
@@ -56,6 +61,7 @@ describe 'ReferencesHandler', ->
 
 		beforeEach ->
 			sinon.stub(@handler, '_findBibDocIds')
+			sinon.stub(@handler, '_findBibFileIds')
 			sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true)
 			@request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData)
 			@call = (callback) =>
@@ -198,6 +204,7 @@ describe 'ReferencesHandler', ->
 
 		beforeEach ->
 			sinon.stub(@handler, '_findBibDocIds').returns(['aaa', 'ccc'])
+			sinon.stub(@handler, '_findBibFileIds').returns(['fff', 'ggg'])
 			sinon.stub(@handler, '_isFullIndex').callsArgWith(1, null, true)
 			@request.post.callsArgWith(1, null, {statusCode: 200}, @fakeResponseData)
 			@call = (callback) =>
@@ -209,6 +216,12 @@ describe 'ReferencesHandler', ->
 				@handler._findBibDocIds.calledWith(@fakeProject).should.equal true
 				done()
 
+		it 'should call _findBibFileIds', (done) ->
+			@call (err, data) =>
+				@handler._findBibDocIds.callCount.should.equal 1
+				@handler._findBibDocIds.calledWith(@fakeProject).should.equal true
+				done()
+
 		it 'should call DocumentUpdaterHandler.flushDocToMongo', (done) ->
 			@call (err, data) =>
 				@DocumentUpdaterHandler.flushDocToMongo.callCount.should.equal 2
@@ -219,7 +232,7 @@ describe 'ReferencesHandler', ->
 				@request.post.callCount.should.equal 1
 				arg = @request.post.firstCall.args[0]
 				expect(arg.json).to.have.all.keys 'docUrls', 'fullIndex'
-				expect(arg.json.docUrls.length).to.equal 2
+				expect(arg.json.docUrls.length).to.equal 4
 				expect(arg.json.fullIndex).to.equal true
 				done()
 
@@ -309,6 +322,32 @@ describe 'ReferencesHandler', ->
 			result = @handler._findBibDocIds(@fakeProject)
 			expect(result).to.deep.equal @expectedIds
 
+	describe '_findBibFileIds', ->
+
+		beforeEach ->
+			@fakeProject =
+				rootFolder: [
+					docs: [
+						{name: 'one.bib', _id: 'aaa'},
+						{name: 'two.txt', _id: 'bbb'},
+					]
+					fileRefs: [
+						{name: 'other.bib', _id: 'ddd'}
+					],
+					folders: [
+						{
+							docs: [{name: 'three.bib', _id: 'ccc'}],
+							fileRefs: [{name: 'four.bib', _id: 'ghg'}],
+							folders: []
+						}
+					]
+				]
+			@expectedIds = ['ddd', 'ghg']
+
+		it 'should select the correct docIds', ->
+			result = @handler._findBibFileIds(@fakeProject)
+			expect(result).to.deep.equal @expectedIds
+
 	describe '_isFullIndex', ->
 
 		beforeEach ->