added clsiformat checker, wired in

services/web/app/coffee/Features/Compile/ClsiFormatChecker.coffee

@@ -0,0 +1,68 @@
+_ = require("lodash")
+async = require("async")
+
+module.exports = ClsiFormatChecker =
+
+	checkRecoursesForProblems: (resources, callback)->
+		jobs = 
+			conflictedPaths: (cb)->
+				ClsiFormatChecker._checkForConflictingPaths resources, cb
+
+			sizeCheck: (cb)->
+				ClsiFormatChecker._checkDocsAreUnderSizeLimit resources, cb
+
+		async.series jobs, (err, problems)->
+			if err? 
+				return callback(err)
+			
+			problems = _.omitBy(problems, _.isEmpty)
+
+			if _.isEmpty(problems)
+				return callback()
+			else
+				callback(null, problems)
+
+
+	_checkForConflictingPaths: (resources, callback)->
+		paths = _.map(resources, 'path')
+
+		conflicts = _.filter paths, (path)->
+			matchingPaths = _.filter paths, (checkPath)->
+				return checkPath.indexOf(path+"/") != -1
+
+			return matchingPaths.length > 0
+
+		conflictObjects = _.map conflicts, (conflict)->
+			path:conflict
+
+		callback null, conflictObjects
+
+	_checkDocsAreUnderSizeLimit: (resources, callback)->
+		
+		FIVEMB = 1000 * 1000 * 5
+
+		totalSize = 0
+
+		sizedResources = _.map resources, (resource)->
+			result = {path:resource.path}
+			if resource.content?
+				result.size = resource.content.replace(/\n/g).length
+				result.kbSize = Math.ceil(result.size / 1000)
+			else
+				result.size = 0
+			totalSize += result.size
+			return result
+
+		tooLarge = totalSize > FIVEMB
+		if !tooLarge
+			return callback()
+		else
+			sizedResources = _.sortBy(sizedResources, "size").reverse().slice(0, 10)
+			return callback(null, {resources:sizedResources, totalSize:totalSize})
+
+
+
+
+
+
+

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -9,21 +9,21 @@ url = require("url")
 ClsiCookieManager = require("./ClsiCookieManager")
 _ = require("underscore")
 async = require("async")
-
+ClsiFormatChecker = require("./ClsiFormatChecker")
 
 module.exports = ClsiManager =
 
-	sendRequest: (project_id, options = {}, callback = (error, success) ->) ->
+	sendRequest: (project_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
 		ClsiManager._buildRequest project_id, options, (error, req) ->
 			return callback(error) if error?
 			logger.log project_id: project_id, "sending compile to CLSI"
-			ClsiManager._checkRecoursesForErrors req.compile.resources, (err, problems)->
+			ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)->
 				if err?
 					logger.err err, project_id, "could not check resources for potential problems before sending to clsi"
 					return callback(err)
-				if problems?
-					logger.log project_id:project_id, problems:problems, "problems with users latex before compile was attempted"
-					return callback(null, problems)
+				if validationProblems?
+					logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted"
+					return callback(null, "validation-problems", null, null, validationProblems)
 				ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) ->
 					if error?
 						logger.err err:error, project_id:project_id, "error sending request to clsi"
@@ -162,79 +162,3 @@ module.exports = ClsiManager =
 					logger.error err: error, project_id: project_id, "CLSI returned failure code"
 					callback error, body
 
-	_checkRecoursesForErrors: (resources, callback)->
-		jobs = 
-			duplicatePaths: (cb)->
-				ClsiManager._checkForDuplicatePaths resources, cb
-
-			conflictedPaths: (cb)->
-				ClsiManager._checkForConflictingPaths resources, cb
-
-			sizeCheck: (cb)->
-				ClsiManager._checkDocsAreUnderSizeLimit resources, cb
-
-		async.series jobs, (err, problems)->
-			if err? 
-				return callback(err)
-			problems = _.omit(problems, _.isEmpty)
-			if _.isEmpty(problems)
-				return callback()
-			else
-				callback(null, problems)
-
-
-	_checkForDuplicatePaths: (resources, callback)->
-		paths = _.pluck(resources, 'path')
-
-		duplicates = _.filter paths, (path)->
-			return _.countBy(paths)[path] > 1
-
-		duplicates = _.uniq(duplicates)
-
-		duplicateObjects = _.map duplicates, (dup)->
-			path:dup
-
-		callback(null, duplicateObjects)
-
-	_checkForConflictingPaths: (resources, callback)->
-		paths = _.pluck(resources, 'path')
-
-		conflicts = _.filter paths, (path)->
-			matchingPaths = _.filter paths, (checkPath)->
-				return checkPath.indexOf(path+"/") != -1
-
-			return matchingPaths.length > 0
-
-		conflictObjects = _.map conflicts, (conflict)->
-			path:conflict
-
-		callback null, conflictObjects
-
-	_checkDocsAreUnderSizeLimit: (resources, callback)->
-		
-		FIVEMB = 1000 * 1000 * 5
-
-		totalSize = 0
-
-		sizedResources = _.map resources, (resource)->
-			result = {path:resource.path}
-			if resource.content?
-				result.size = resource.content.join("").length
-			else
-				result.size = 0
-			totalSize += result.size
-			return result
-
-		tooLarge = totalSize > FIVEMB
-		if !tooLarge
-			return callback()
-		else
-			sizedResources = _.sortBy(sizedResources, "size").slice(10).reverse()
-			return callback(null, {resources:sizedResources, totalSize:totalSize})
-
-
-
-
-
-
-

services/web/app/coffee/Features/Compile/CompileController.coffee

@@ -30,7 +30,7 @@ module.exports = CompileController =
 			if req.body?.draft
 				options.draft = req.body.draft
 			logger.log {options, project_id}, "got compile request"
-			CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits) ->
+			CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits, validationProblems) ->
 				return next(error) if error?
 				res.contentType("application/json")
 				res.status(200).send JSON.stringify {
@@ -38,6 +38,7 @@ module.exports = CompileController =
 					outputFiles: outputFiles
 					compileGroup: limits?.compileGroup
 					clsiServerId:clsiServerId
+					validationProblems:validationProblems
 				}
 
 	downloadPdf: (req, res, next = (error) ->)->

services/web/app/coffee/Features/Compile/CompileManager.coffee

@@ -37,10 +37,10 @@ module.exports = CompileManager =
 							return callback(error) if error?
 							for key, value of limits
 								options[key] = value
-							ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId) ->
+							ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
 								return callback(error) if error?
 								logger.log files: outputFiles, "output files"
-								callback(null, status, outputFiles, clsiServerId, limits)
+								callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
 								
 	deleteAuxFiles: (project_id, callback = (error) ->) ->
 		CompileManager.getProjectCompileLimits project_id, (error, limits) ->

services/web/app/views/project/editor/pdf.jade

@@ -154,6 +154,27 @@ div.full-size.pdf(ng-controller="PdfController")
 				ng-if="settings.pdfViewer == 'native'"
 			)
 
+		.pdf-validation-problems(ng-switch-when="validation-problems")
+
+			.alert.alert-danger(ng-show="pdf.validation.duplicatePaths")
+				strong #{translate("latex_error")}
+				span #{translate("duplicate_paths_found")}
+
+
+			.alert.alert-danger(ng-show="pdf.validation.sizeCheck")
+				strong #{translate("project_too_large")} 
+				div #{translate("project_too_large_please_reduce")}
+				div
+					li(ng-repeat="entry in pdf.validation.sizeCheck.resources") {{ '/'+entry['path'] }} - {{entry['kbSize']}}kb
+	
+			.alert.alert-danger(ng-show="pdf.validation.conflictedPaths")
+				div
+					strong #{translate("conflicting_paths_found")}
+				div #{translate("following_paths_conflict")}
+				div
+					li(ng-repeat="entry in pdf.validation.conflictedPaths") {{ '/'+entry['path'] }}
+
+
 		.pdf-errors(ng-switch-when="errors")
 	
 			.alert.alert-danger(ng-show="pdf.error")

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -42,11 +42,11 @@ define [
 			$scope.pdf.error      = false
 			$scope.pdf.timedout   = false
 			$scope.pdf.failure    = false
-			$scope.pdf.projectTooLarge = false
 			$scope.pdf.url        = null
 			$scope.pdf.clsiMaintenance = false
 			$scope.pdf.tooRecentlyCompiled = false
 			$scope.pdf.renderingError = false
+			$scope.pdf.projectTooLarge = false
 
 			if response.status == "timedout"
 				$scope.pdf.view = 'errors'
@@ -60,13 +60,15 @@ define [
 				$scope.pdf.view = 'errors'
 				$scope.pdf.failure = true
 				$scope.shouldShowLogs = true
-				fetchLogs()
 			else if response.status == 'clsi-maintenance'
 				$scope.pdf.view = 'errors'
 				$scope.pdf.clsiMaintenance = true
 			else if response.status == "too-recently-compiled"
 				$scope.pdf.view = 'errors'
 				$scope.pdf.tooRecentlyCompiled = true
+			else if response.status == "validation-problems"
+				$scope.pdf.view = "validation-problems"
+				$scope.pdf.validation = response.validationProblems
 			else if response.status == "success"
 				$scope.pdf.view = 'pdf'
 				$scope.shouldShowLogs = false

services/web/public/stylesheets/app/editor/pdf.less

@@ -3,7 +3,7 @@
 	top: 58px;
 }
 
-.pdf-logs, .pdf-errors, .pdf-uncompiled {
+.pdf-logs, .pdf-errors, .pdf-uncompiled, .pdf-validation-problems{
 	padding: @line-height-computed / 2;
 }
 

services/web/test/UnitTests/coffee/Compile/ClsiFormatCheckerTests.coffee

@@ -0,0 +1,151 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = "../../../../app/js/Features/Compile/ClsiFormatChecker.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "ClsiFormatChecker", ->
+	beforeEach ->
+		@ClsiFormatChecker = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @settings ={}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
+		@project_id = "project-id"
+
+
+	describe "checkRecoursesForProblems", ->
+
+		beforeEach ->
+			@resources = [{
+				path:    "main.tex"
+				content: "stuff"
+			}, {
+				path:    "chapters/chapter1"
+				content: "other stuff"
+			}, {
+				path: "stuff/image/image.png"
+				url:  "http:somewhere.com/project/#{@project_id}/file/1234124321312"
+				modified: "more stuff"
+			}]
+
+		it "should call _checkForDuplicatePaths and _checkForConflictingPaths", (done)->
+
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null)
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1)
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				@ClsiFormatChecker._checkForConflictingPaths.called.should.equal true
+				@ClsiFormatChecker._checkDocsAreUnderSizeLimit.called.should.equal true
+				done()
+
+		it "should remove undefined errors", (done)->
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {})
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				expect(problems).to.not.exist
+				expect(problems).to.not.exist
+				done()
+
+		it "should keep populated arrays", (done)->
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [{path:"somewhere/main.tex"}])
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {})
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				problems.conflictedPaths[0].path.should.equal "somewhere/main.tex"
+				expect(problems.sizeCheck).to.not.exist
+				done()
+
+		it "should keep populated object", (done)->
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {resources:[{"a.tex"},{"b.tex"}], totalSize:1000000})
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				problems.sizeCheck.resources.length.should.equal 2
+				problems.sizeCheck.totalSize.should.equal 1000000
+				expect(problems.conflictedPaths).to.not.exist
+				done()
+
+		describe "_checkForConflictingPaths", ->
+
+			beforeEach ->
+
+				@resources.push({
+					path:    "chapters/chapter1.tex"
+					content: "other stuff"
+				})
+
+				@resources.push({
+					path:    "chapters.tex"
+					content: "other stuff"
+				})
+
+			it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)->
+				@resources.push({
+					path: "stuff/image"
+					url: "http://somwhere.com"
+				})
+
+				@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
+					conflictPathErrors.length.should.equal 1
+					conflictPathErrors[0].path.should.equal "stuff/image"
+					done()
+				
+			it "should flag up when a root level file has folder with same subpath as file elsewhere", (done)->
+				@resources.push({
+					path:    "stuff"
+					content: "other stuff"
+				})
+
+				@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
+					conflictPathErrors.length.should.equal 1
+					conflictPathErrors[0].path.should.equal "stuff"
+					done()
+
+			it "should not flag up when the file is a substring of a path", (done)->
+				@resources.push({
+					path:    "stuf"
+					content: "other stuff"
+				})
+
+				@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
+					conflictPathErrors.length.should.equal 0
+					done()
+				
+
+		describe "_checkDocsAreUnderSizeLimit", ->
+
+			it "should error when there is more than 5mb of data", (done)->
+
+				@resources.push({
+					path:    "massive.tex"
+					content: require("crypto").randomBytes(1000 * 1000 * 5).toString("hex")
+				})
+
+				while @resources.length < 20
+					@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
+
+				@ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
+					sizeError.totalSize.should.equal 10000016
+					sizeError.resources.length.should.equal 10
+					sizeError.resources[0].path.should.equal "massive.tex"
+					sizeError.resources[0].size.should.equal 1000 * 1000 * 10
+					done()
+			
+
+			it "should return nothing when project is correct size", (done)->
+
+				@resources.push({
+					path:    "massive.tex"
+					content: require("crypto").randomBytes(1000 * 1000 * 1).toString("hex")
+				})
+
+				while @resources.length < 20
+					@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
+
+				@ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
+					expect(sizeError).to.not.exist
+					done()
+
+
+
+
+
+
+

services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee

@@ -12,6 +12,8 @@ describe "ClsiManager", ->
 			getCookieJar: sinon.stub().callsArgWith(1, null, @jar)
 			setServerId: sinon.stub().callsArgWith(2)
 			_getServerId:sinon.stub()
+		@ClsiFormatChecker =
+			checkRecoursesForProblems:sinon.stub().callsArgWith(1)
 		@ClsiManager = SandboxedModule.require modulePath, requires:
 			"settings-sharelatex": @settings =
 				apis:
@@ -27,6 +29,7 @@ describe "ClsiManager", ->
 			"./ClsiCookieManager": @ClsiCookieManager
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
 			"request": @request = sinon.stub()
+			"./ClsiFormatChecker": @ClsiFormatChecker
 		@project_id = "project-id"
 		@callback = sinon.stub()
 
@@ -320,106 +323,6 @@ describe "ClsiManager", ->
 
 
 
-	describe "_checkRecoursesForErrors", ->
-
-		beforeEach ->
-			@resources = [{
-				path:    "main.tex"
-				content: ["stuff"]
-			}, {
-				path:    "chapters/chapter1"
-				content: ["other stuff"]
-			}, {
-				path: "stuff/image/image.png"
-				url:  "#{@settings.apis.filestore.url}/project/#{@project_id}/file/1234124321312"
-				modified: ["more stuff"]
-			}]
-
-		it "should call _checkForDuplicatePaths and _checkForConflictingPaths", (done)->
-
-			@ClsiManager._checkForDuplicatePaths = sinon.stub().callsArgWith(1)
-			@ClsiManager._checkForConflictingPaths = sinon.stub().callsArgWith(1)
-			@ClsiManager._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1)
-			@ClsiManager._checkRecoursesForErrors @resources, (err, problems)=>
-				@ClsiManager._checkForDuplicatePaths.called.should.equal true
-				@ClsiManager._checkForConflictingPaths.called.should.equal true
-				@ClsiManager._checkDocsAreUnderSizeLimit.called.should.equal true
-				expect(problems).to.not.exist
-				done()
-
-
-		it "should remove undefined errors", (done)->
-			@ClsiManager._checkForDuplicatePaths = sinon.stub().callsArgWith(1, null, [path:"something/here"])
-			@ClsiManager._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
-			@ClsiManager._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null)
-			@ClsiManager._checkRecoursesForErrors @resources, (err, problems)=>
-				problems.duplicatePaths[0].path.should.equal "something/here"
-				expect(problems.conflictedPaths).to.not.exist
-				expect(problems.sizeCheck).to.not.exist
-
-				done()
-
-		describe "_checkForDuplicatePaths", ->
-
-			it "should flag up 2 nested files with same path", (done)->
-
-				@resources.push({
-					path: "chapters/chapter1"
-					url: "http://somwhere.com"
-				})
-
-				@ClsiManager._checkForDuplicatePaths @resources, (err, duplicateErrors)->
-					duplicateErrors.length.should.equal 1
-					duplicateErrors[0].path.should.equal "chapters/chapter1"
-					done()
-
-		describe "_checkForConflictingPaths", ->
-
-			it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)->
-				@resources.push({
-					path: "stuff/image"
-					url: "http://somwhere.com"
-				})
-
-				@resources.push({
-					path:    "chapters/chapter1.tex"
-					content: ["other stuff"]
-				})
-
-				@resources.push({
-					path:    "chapters.tex"
-					content: ["other stuff"]
-				})
-
-				@ClsiManager._checkForConflictingPaths @resources, (err, conflictPathErrors)->
-					conflictPathErrors.length.should.equal 1
-					conflictPathErrors[0].path.should.equal "stuff/image"
-					done()
-				
-
-		describe "_checkDocsAreUnderSizeLimit", ->
-
-			it "should error when there is more than 2mb of data", (done)->
-
-				@resources.push({
-					path:    "massive.tex"
-					content: [require("crypto").randomBytes(1000 * 1000 * 5).toString("hex")]
-				})
-
-				while @resources.length < 20
-					@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
-
-				@ClsiManager._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
-					sizeError.totalSize.should.equal 10000016
-					sizeError.resources.length.should.equal 10
-					sizeError.resources[0].path.should.equal "massive.tex"
-					sizeError.resources[0].size.should.equal 1000 * 1000 * 10
-					done()
-			
-
-
-
-