Refactor TCP redis monitoring to be more explicit

services/document-updater/app/coffee/RedisBackend.coffee

@@ -18,7 +18,7 @@ class Client
 			}
 		)
 
-	monitorAndReconnect: () ->
+	monitorTcpAndReconnect: () ->
 		for client in @clients
 			if client.driver == "ioredis"
 				@_monitorCluster(client.rclient)
@@ -28,18 +28,18 @@ class Client
 			# Nodes can come and go as the cluster moves/heals, so each heartbeat
 			# we ask again for the currently known nodes.
 			for node in rclient.nodes("all")
-				do (node) =>
-					timer = setTimeout () =>
-						logger.error {err: new Error("Node timed out, reconnecting"), key: node.options.key}
-						node.stream.destroy()
-						timer = null
-					, @HEARTBEAT_TIMEOUT
-					node.ping (err) ->
-						if !err?
-							clearTimeout timer
-							timer = null
+				@_checkNode(node)
 		, @HEARTBEAT_INTERVAL
-
+	
+	_checkNode: (node) ->
+		timer = setTimeout () ->
+			logger.error {err: new Error("Node timed out, reconnecting"), key: node.options.key}
+			# Discussion of application layer monitoring recommends this way of reconnecting at https://github.com/luin/ioredis/issues/275
+			node.stream.destroy()
+		, @HEARTBEAT_TIMEOUT
+		node.ping (err) ->
+			if !err?
+				clearTimeout timer
 
 class MultiClient
 	constructor: (@clients) ->

services/document-updater/app/coffee/RedisManager.coffee

@@ -10,7 +10,7 @@ Errors = require "./Errors"
 # Make times easy to read
 minutes = 60 # seconds for Redis expire
 
-rclient.monitorAndReconnect()
+rclient.monitorTcpAndReconnect()
 
 module.exports = RedisManager =
 	putDocInMemory : (project_id, doc_id, docLines, version, _callback)->

services/document-updater/config/settings.defaults.coffee

@@ -21,33 +21,31 @@ module.exports =
 			host:"localhost"
 			password:""
 		documentupdater:
-			port:"6379"
-			host:"localhost"
-			password:""
-			key_schema:
-				blockingKey: ({doc_id}) -> "Blocking:#{doc_id}"
-				docLines: ({doc_id}) -> "doclines:#{doc_id}"
-				docOps: ({doc_id}) -> "DocOps:#{doc_id}"
-				docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
-				projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
-				pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
-				docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
+			# port:"6379"
+			# host:"localhost"
+			# password:""
+			# key_schema:
+			# 	blockingKey: ({doc_id}) -> "Blocking:#{doc_id}"
+			# 	docLines: ({doc_id}) -> "doclines:#{doc_id}"
+			# 	docOps: ({doc_id}) -> "DocOps:#{doc_id}"
+			# 	docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
+			# 	projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
+			# 	docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
 			# To use Redis cluster, configure the backend as follows:
-			# [{
-			# 	primary: true
-			# 	cluster: [{
-			# 		port: "7000"
-			# 		host: "localhost"
-			# 	}]
-			# 	key_schema:
-			# 		blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}"
-			# 		docLines: ({doc_id}) -> "doclines:{#{doc_id}}"
-			# 		docOps: ({doc_id}) -> "DocOps:{#{doc_id}}"
-			# 		docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}"
-			# 		projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}"
-			# 		pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}"
-			# 		docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}"
-			# }]
+			[{
+				primary: true
+				cluster: [{
+					port: "7000"
+					host: "localhost"
+				}]
+				key_schema:
+					blockingKey: ({doc_id}) -> "Blocking:{#{doc_id}}"
+					docLines: ({doc_id}) -> "doclines:{#{doc_id}}"
+					docOps: ({doc_id}) -> "DocOps:{#{doc_id}}"
+					docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}"
+					projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}"
+					docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}"
+			}]
 	
 	max_doc_length: 2 * 1024 * 1024 # 2mb
 

services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee

@@ -307,10 +307,10 @@ describe "RedisBackend", ->
 					}, "error in redis backend")
 					.should.equal true
 
-	describe "monitorAndReconnect", ->
+	describe "monitorTcpAndReconnect", ->
 		beforeEach ->
 			@client._monitorCluster = sinon.stub()
-			@client.monitorAndReconnect()
+			@client.monitorTcpAndReconnect()
 	
 		it "should monitor the cluster client", ->
 			@client._monitorCluster

services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee

@@ -10,7 +10,7 @@ describe "RedisManager", ->
 		@rclient =
 			auth: () ->
 			exec: sinon.stub()
-			monitorAndReconnect: () ->
+			monitorTcpAndReconnect: () ->
 		@rclient.multi = () => @rclient
 		@RedisManager = SandboxedModule.require modulePath, requires:
 			"./RedisBackend":