Merge pull request #37 from sharelatex/bg-rate-limit-autocompile

rate limit autocompile (connects to #18)

services/web/app/coffee/Features/Compile/CompileManager.coffee

@@ -18,7 +18,7 @@ module.exports = CompileManager =
 			timer.done()
 			_callback(args...)
 
-		@_checkIfAutoCompileLimitHasBeenHit options.isAutoCompile, (err, canCompile)->
+		@_checkIfAutoCompileLimitHasBeenHit options.isAutoCompile, "everyone", (err, canCompile)->
 			if !canCompile
 				return callback null, "autocompile-backoff", []
 			logger.log project_id: project_id, user_id: user_id, "compiling project"
@@ -34,12 +34,16 @@ module.exports = CompileManager =
 						return callback(error) if error?
 						for key, value of limits
 							options[key] = value
-						# only pass user_id down to clsi if this is a per-user compile
-						compileAsUser = if Settings.disablePerUserCompiles then undefined else user_id
-						ClsiManager.sendRequest project_id, compileAsUser, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
-							return callback(error) if error?
-							logger.log files: outputFiles, "output files"
-							callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
+						# Put a lower limit on autocompiles for free users, based on compileGroup
+						CompileManager._checkCompileGroupAutoCompileLimit options.isAutoCompile, limits.compileGroup, (err, canCompile)->
+							if !canCompile
+								return callback null, "autocompile-backoff", []
+							# only pass user_id down to clsi if this is a per-user compile
+							compileAsUser = if Settings.disablePerUserCompiles then undefined else user_id
+							ClsiManager.sendRequest project_id, compileAsUser, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
+								return callback(error) if error?
+								logger.log files: outputFiles, "output files"
+								callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
 
 
 	stopCompile: (project_id, user_id, callback = (error) ->) ->
@@ -72,18 +76,27 @@ module.exports = CompileManager =
 			else
 				return callback null, true
 
-	_checkIfAutoCompileLimitHasBeenHit: (isAutoCompile, callback = (err, canCompile)->)->
+	_checkCompileGroupAutoCompileLimit: (isAutoCompile, compileGroup, callback = (err, canCompile)->)->
+		if compileGroup is "default"
+			CompileManager._checkIfAutoCompileLimitHasBeenHit isAutoCompile, compileGroup, callback
+		else
+			Metrics.inc "auto-compile-#{compileGroup}"
+			return callback(null, true)	# always allow priority group users to compile
+
+	_checkIfAutoCompileLimitHasBeenHit: (isAutoCompile, compileGroup, callback = (err, canCompile)->)->
 		if !isAutoCompile
 			return callback(null, true)
-		opts = 
+		Metrics.inc "auto-compile-#{compileGroup}"
+		opts =
 			endpointName:"auto_compile"
 			timeInterval:20
-			subjectName:"everyone"
-			throttle: 25
+			subjectName:compileGroup
+			throttle: Settings?.rateLimit?.autoCompile?[compileGroup] || 25
 		rateLimiter.addCount opts, (err, canCompile)->
 			if err?
 				canCompile = false
-			logger.log canCompile:canCompile, opts:opts, "checking if auto compile limit has been hit"
+			if !canCompile
+				Metrics.inc "auto-compile-#{compileGroup}-limited"
 			callback err, canCompile
 
 	_ensureRootDocumentIsSet: (project_id, callback = (error) ->) ->

services/web/app/views/project/editor/pdf.pug

@@ -393,6 +393,12 @@ div.full-size.pdf(ng-controller="PdfController")
 						ng-click="startFreeTrial('compile-timeout')"
 					) #{translate("start_free_trial")}
 
+
+			.alert.alert-danger(ng-show="pdf.autoCompileDisabled")
+				p
+					strong #{translate("autocompile_disabled")}. 
+					span #{translate("autocompile_disabled_reason")}
+
 			.alert.alert-danger(ng-show="pdf.projectTooLarge")
 				strong #{translate("project_too_large")} 
 				span #{translate("project_too_large_please_reduce")}

services/web/config/settings.defaults.coffee

@@ -437,3 +437,8 @@ module.exports = settings =
 	#	name : "all projects",
 	#	url: "/templates/all"
 	#}]
+
+	rateLimits:
+		autoCompile:
+			everyone: 100
+			standard: 25

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -68,7 +68,7 @@ define [
 		$scope.$on "project:joined", () ->
 			return if !autoCompile
 			autoCompile = false
-			$scope.recompile(isAutoCompile: true)
+			$scope.recompile(isAutoCompileOnLoad: true)
 			$scope.hasPremiumCompile = $scope.project.features.compileGroup == "priority"
 
 		$scope.$on "pdf:error:display", () ->
@@ -86,7 +86,7 @@ define [
 
 			if isTimeNonMonotonic || timeSinceLastCompile >= AUTO_COMPILE_TIMEOUT
 				if (!ide.$scope.hasLintingError)
-					$scope.recompile(isBackgroundAutoCompile: true)
+					$scope.recompile(isAutoCompileOnChange: true)
 			else
 				# Extend remainder of timeout
 				autoCompileTimeout = setTimeout () ->
@@ -127,7 +127,7 @@ define [
 		sendCompileRequest = (options = {}) ->
 			url = "/project/#{$scope.project_id}/compile"
 			params = {}
-			if options.isAutoCompile
+			if options.isAutoCompileOnLoad or options.isAutoCompileOnChange
 				params["auto_compile"]=true
 			# if the previous run was a check, clear the error logs
 			$scope.pdf.logEntries = [] if $scope.check
@@ -168,6 +168,7 @@ define [
 			$scope.pdf.compileExited = false
 			$scope.pdf.failedCheck = false
 			$scope.pdf.compileInProgress = false
+			$scope.pdf.autoCompileDisabled = false
 
 			# make a cache to look up files by name
 			fileByPath = {}
@@ -206,7 +207,13 @@ define [
 				$scope.shouldShowLogs = true
 				fetchLogs(fileByPath)
 			else if response.status == "autocompile-backoff"
-				$scope.pdf.view = 'uncompiled'
+				if $scope.pdf.isAutoCompileOnLoad # initial autocompile
+					$scope.pdf.view = 'uncompiled'
+				else # background autocompile from typing
+					$scope.pdf.view = 'errors'
+					$scope.pdf.autoCompileDisabled = true
+					$scope.autocompile_enabled = false # disable any further autocompiles
+					event_tracking.sendMB "autocompile-rate-limited", {hasPremiumCompile: $scope.hasPremiumCompile}
 			else if response.status == "project-too-large"
 				$scope.pdf.view = 'errors'
 				$scope.pdf.projectTooLarge = true
@@ -418,6 +425,7 @@ define [
 			event_tracking.sendMBSampled "editor-recompile-sampled", options
 
 			$scope.pdf.compiling = true
+			$scope.pdf.isAutoCompileOnLoad = options?.isAutoCompileOnLoad # initial autocompile
 
 			if options?.force
 				# for forced compile, turn off validation check and ignore errors

services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee

@@ -44,7 +44,7 @@ describe "CompileManager", ->
 
 		describe "succesfully", ->
 			beforeEach ->
-				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (_, cb)-> cb(null, true)
+				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (isAutoCompile, compileGroup, cb)-> cb(null, true)
 				@CompileManager.compile @project_id, @user_id, {}, @callback
 
 			it "should check the project has not been recently compiled", ->
@@ -84,7 +84,7 @@ describe "CompileManager", ->
 				
 		describe "when the project has been recently compiled", ->
 			it "should return", (done)->
-				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (_, cb)-> cb(null, true)
+				@CompileManager._checkIfAutoCompileLimitHasBeenHit = (isAutoCompile, compileGroup, cb)-> cb(null, true)
 				@CompileManager._checkIfRecentlyCompiled = sinon.stub().callsArgWith(2, null, true)
 				@CompileManager.compile @project_id, @user_id, {}, (err, status)->
 					status.should.equal "too-recently-compiled"
@@ -92,7 +92,7 @@ describe "CompileManager", ->
 
 		describe "should check the rate limit", ->
 			it "should return", (done)->
-				@CompileManager._checkIfAutoCompileLimitHasBeenHit = sinon.stub().callsArgWith(1, null, false)
+				@CompileManager._checkIfAutoCompileLimitHasBeenHit = sinon.stub().callsArgWith(2, null, false)
 				@CompileManager.compile @project_id, @user_id, {}, (err, status)->
 					status.should.equal "autocompile-backoff"
 					done()
@@ -222,14 +222,14 @@ describe "CompileManager", ->
 	describe "_checkIfAutoCompileLimitHasBeenHit", ->
 
 		it "should be able to compile if it is not an autocompile", (done)->
-			@ratelimiter.addCount.callsArgWith(1, null, true)
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit false, (err, canCompile)=>
+			@ratelimiter.addCount.callsArgWith(2, null, true)
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit false, "everyone", (err, canCompile)=>
 				canCompile.should.equal true
 				done()
 
 		it "should be able to compile if rate limit has remianing", (done)->
 			@ratelimiter.addCount.callsArgWith(1, null, true)
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=>
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
 				args = @ratelimiter.addCount.args[0][0]
 				args.throttle.should.equal 25
 				args.subjectName.should.equal "everyone"
@@ -240,13 +240,13 @@ describe "CompileManager", ->
 
 		it "should be not able to compile if rate limit has no remianing", (done)->
 			@ratelimiter.addCount.callsArgWith(1, null, false)
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=>
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
 				canCompile.should.equal false
 				done()
 
 		it "should return false if there is an error in the rate limit", (done)->
 			@ratelimiter.addCount.callsArgWith(1, "error")
-			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=>
+			@CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
 				canCompile.should.equal false
 				done()