Merge pull request #414 from sharelatex/afc-retries-with-exp-backoff

Use exponential backoff in request retries to analytics

services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee

@@ -5,15 +5,51 @@ request = require "requestretry"
 Errors = require '../Errors/Errors'
 
 
+makeFaultTolerantRequest = (userId, options, callback) ->
+	if userId+"" == settings.smokeTest?.userId+""
+		return callback()
+
+	options = Object.assign(options, {
+		delayStrategy: exponentialBackoffStrategy()
+		timeout: 30000
+	})
+
+	if settings.overleaf?
+		options.qs = Object.assign({}, options.qs, { fromV2: 1 })
+
+	makeRequest(options, callback)
+
 makeRequest = (opts, callback)->
 	if settings.apis?.analytics?.url?
 		urlPath = opts.url
 		opts.url = "#{settings.apis.analytics.url}#{urlPath}"
-		request opts, callback
+		request opts, (err) ->
+			if err?
+				logger.err { err: err }, 'Request to analytics failed'
+
+		callback() # Do not wait for all the attempts
 	else
 		callback(new Errors.ServiceNotConfiguredError('Analytics service not configured'))
 
 
+# Set an exponential backoff to retry calls to analytics. First retry will
+# happen after 4s, then 8, 16, 32, 64...
+exponentialBackoffStrategy = () ->
+	attempts = 1 # This won't be called until there has been 1 failure
+
+	() ->
+		attempts += 1
+		exponentialBackoffDelay(attempts)
+
+exponentialBackoffDelay = (attempts) ->
+	delay = 2 ** attempts * 1000
+
+	logger.warn "Error comunicating with the analytics service. " +
+		"Will try again attempt #{attempts} in #{delay}ms"
+
+	delay
+
+
 module.exports =
 
 	identifyUser: (user_id, old_user_id, callback = (error)->)->
@@ -27,40 +63,33 @@ module.exports =
 		makeRequest opts, callback
 
 	recordEvent: (user_id, event, segmentation = {}, callback = (error) ->) ->
-		if user_id+"" == settings.smokeTest?.userId+""
-			return callback()
 		opts =
 			body:
 				event:event
 				segmentation:segmentation
 			json:true
 			method:"POST"
-			timeout:1000
 			url: "/user/#{user_id}/event"
-			maxAttempts: 20
-			retryDelay: 5000
-		if settings.overleaf?
-			opts.qs = {fromV2: 1}
-		makeRequest opts, callback
+			maxAttempts: 7 # Give up after ~ 8min
+
+		makeFaultTolerantRequest user_id, opts, callback
+
 
 	updateEditingSession: (userId, projectId, countryCode, callback = (error) ->) ->
-		if userId+"" == settings.smokeTest?.userId+""
-			return callback()
 		query =
 			userId: userId
 			projectId: projectId
+
 		if countryCode
 			query.countryCode = countryCode
+
 		opts =
 			method: "PUT"
-			timeout: 1000
 			url: "/editingSession"
 			qs: query
-			maxAttempts: 20
-			retryDelay: 5000
-		if settings.overleaf?
-			opts.qs.fromV2 = 1
-		makeRequest opts, callback
+			maxAttempts: 6 # Give up after ~ 4min
+
+		makeFaultTolerantRequest userId, opts, callback
 
 
 	getLastOccurrence: (user_id, event, callback = (error) ->) ->