Move all redis end points to be cluster compatible

services/web/app/coffee/Features/Compile/ClsiCookieManager.coffee

@@ -1,7 +1,7 @@
 Settings = require "settings-sharelatex"
 request = require('request')
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.web)
+RedisWrapper = require("../../infrastructure/RedisWrapper")
+rclient = RedisWrapper.client("clsi_cookie")
 Cookie = require('cookie')
 logger = require "logger-sharelatex"
 

services/web/app/coffee/Features/Compile/CompileManager.coffee

@@ -1,6 +1,6 @@
 Settings = require('settings-sharelatex')
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.web)
+RedisWrapper = require("../../infrastructure/RedisWrapper")
+rclient = RedisWrapper.client("clsi_recently_compiled")
 DocumentUpdaterHandler = require "../DocumentUpdater/DocumentUpdaterHandler"
 Project = require("../../models/Project").Project
 ProjectRootDocManager = require "../Project/ProjectRootDocManager"

services/web/app/coffee/Features/DocumentUpdater/DocumentUpdaterHandler.coffee

@@ -5,24 +5,10 @@ _ = require 'underscore'
 async = require 'async'
 logger = require('logger-sharelatex')
 metrics = require('metrics-sharelatex')
-redis = require("redis-sharelatex")
-rclient = redis.createClient(settings.redis.web)
 Project = require("../../models/Project").Project
 ProjectLocator = require('../../Features/Project/ProjectLocator')
 
 module.exports = DocumentUpdaterHandler =
-	
-	queueChange : (project_id, doc_id, change, callback = ()->)->
-		jsonChange = JSON.stringify change
-		doc_key = keys.combineProjectIdAndDocId(project_id, doc_id)
-		multi = rclient.multi()
-		multi.rpush keys.pendingUpdates(doc_id:doc_id), jsonChange
-		multi.sadd  keys.docsWithPendingUpdates, doc_key
-		multi.rpush "pending-updates-list", doc_key
-		multi.exec (error) ->
-			return callback(error) if error?
-			callback()
-
 	flushProjectToMongo: (project_id, callback = (error) ->)->
 		logger.log project_id:project_id, "flushing project from document updater"
 		timer = new metrics.Timer("flushing.mongo.project")

services/web/app/coffee/Features/Editor/EditorRealTimeController.coffee

@@ -1,14 +1,10 @@
 Settings = require 'settings-sharelatex'
-redis = require("redis-sharelatex")
-rclientPub = redis.createClient(Settings.redis.web)
-rclientSub = redis.createClient(Settings.redis.web)
+RedisWrapper = require("../../infrastructure/RedisWrapper")
+rclient = RedisWrapper.client("realtime")
 
 module.exports = EditorRealTimeController =
-	rclientPub: rclientPub
-	rclientSub: rclientSub
-
 	emitToRoom: (room_id, message, payload...) ->
-		@rclientPub.publish "editor-events", JSON.stringify
+		rclient.publish "editor-events", JSON.stringify
 			room_id: room_id
 			message: message
 			payload: payload

services/web/app/coffee/Features/HealthCheck/HealthCheckController.coffee

@@ -1,8 +1,8 @@
 Mocha = require "mocha"
 Base = require("mocha/lib/reporters/base")
-redis = require("redis-sharelatex")
+RedisWrapper = require("../../infrastructure/RedisWrapper")
+rclient = RedisWrapper.client("health_check")
 settings = require("settings-sharelatex")
-redisCheck = redis.activeHealthCheckRedis(settings.redis.web)
 logger = require "logger-sharelatex"
 domain = require "domain"
 
@@ -31,10 +31,12 @@ module.exports = HealthCheckController =
 				delete require.cache[path]
 
 	checkRedis: (req, res, next)->
-		if redisCheck.isAlive()
-			res.sendStatus 200
-		else
-			res.sendStatus 500
+		rclient.healthCheck (error) ->
+			if error?
+				logger.err {err: error}, "failed redis health check"
+				res.sendStatus 500
+			else
+				res.sendStatus 200
 		
 Reporter = (res) ->
 	(runner) ->

services/web/app/coffee/Features/Security/OneTimeTokenHandler.coffee

@@ -1,6 +1,6 @@
 Settings = require('settings-sharelatex')
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.web)
+RedisWrapper = require("../../infrastructure/RedisWrapper")
+rclient = RedisWrapper.client("one_time_token")
 crypto = require("crypto")
 logger = require("logger-sharelatex")
 

services/web/app/coffee/Features/User/UserSessionsManager.coffee

@@ -3,7 +3,6 @@ logger = require("logger-sharelatex")
 Async = require('async')
 _ = require('underscore')
 UserSessionsRedis = require('./UserSessionsRedis')
-
 rclient = UserSessionsRedis.client()
 
 module.exports = UserSessionsManager =

services/web/app/coffee/Features/User/UserSessionsRedis.coffee

@@ -1,21 +1,9 @@
-Settings = require 'settings-sharelatex'
-redis = require 'redis-sharelatex'
-ioredis = require 'ioredis'
-logger = require 'logger-sharelatex'
-
-redisSessionsSettings = Settings.redis.websessions or Settings.redis.web
+RedisWrapper = require("../../infrastructure/RedisWrapper")
+rclient = RedisWrapper.client("websessions")
 
 module.exports = Redis =
 	client: () ->
-		if redisSessionsSettings?.cluster?
-			logger.log {}, "using redis cluster for web sessions"
-			rclient = new ioredis.Cluster(redisSessionsSettings.cluster)
-		else
-			rclient = redis.createClient(redisSessionsSettings)
 		return rclient
 
 	sessionSetKey: (user) ->
-		if redisSessionsSettings?.cluster?
-			return "UserSessions:{#{user._id}}"
-		else
-			return "UserSessions:#{user._id}"
+		return "UserSessions:{#{user._id}}"

services/web/app/coffee/infrastructure/LockManager.coffee

@@ -1,7 +1,7 @@
 metrics = require('metrics-sharelatex')
 Settings = require('settings-sharelatex')
-redis = require("redis-sharelatex")
-rclient = redis.createClient(Settings.redis.web)
+RedisWrapper = require("./RedisWrapper")
+rclient = RedisWrapper.client("lock")
 logger = require "logger-sharelatex"
 
 module.exports = LockManager =

services/web/app/coffee/infrastructure/RedisWrapper.coffee

@@ -1,28 +1,14 @@
 Settings = require 'settings-sharelatex'
 redis = require 'redis-sharelatex'
-ioredis = require 'ioredis'
-logger = require 'logger-sharelatex'
-
 
 # A per-feature interface to Redis,
 # looks up the feature in `settings.redis`
 # and returns an appropriate client.
 # Necessary because we don't want to migrate web over
 # to redis-cluster all at once.
-
-# TODO: consider merging into `redis-sharelatex`
-
-
 module.exports = Redis =
-
 	# feature = 'websessions' | 'ratelimiter' | ...
 	client: (feature) ->
 		redisFeatureSettings = Settings.redis[feature] or Settings.redis.web
-		if redisFeatureSettings?.cluster?
-			logger.log {feature}, "creating redis-cluster client"
-			rclient = new ioredis.Cluster(redisFeatureSettings.cluster)
-			rclient.__is_redis_cluster = true
-		else
-			logger.log {feature}, "creating redis client"
-			rclient = redis.createClient(redisFeatureSettings)
+		rclient = redis.createClient(redisFeatureSettings)
 		return rclient

services/web/test/UnitTests/coffee/Compile/ClsiCookieManagerTests.coffee

@@ -34,8 +34,8 @@ describe "ClsiCookieManager", ->
 				ttl:Math.random()
 				key: "coooookie"
 		@requires = 
-			"redis-sharelatex" :
-				createClient: =>
+			"../../infrastructure/RedisWrapper":
+				client: =>
 					@redis
 			"settings-sharelatex": @settings
 			"request": @request

services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee

@@ -15,8 +15,8 @@ describe "CompileManager", ->
 		@CompileManager = SandboxedModule.require modulePath, requires:
 			"settings-sharelatex": @settings =
 				redis: web: {host: "localhost", port: 42}
-			"redis-sharelatex":
-				createClient: () => @rclient = { auth: () -> }
+			"../../infrastructure/RedisWrapper":
+				client: () => @rclient = { auth: () -> }
 			"../DocumentUpdater/DocumentUpdaterHandler": @DocumentUpdaterHandler = {}
 			"../Project/ProjectRootDocManager": @ProjectRootDocManager = {}
 			"../../models/Project": Project: @Project = {}

services/web/test/UnitTests/coffee/DocumentUpdater/DocumentUpdaterHandlerTests.coffee

@@ -20,10 +20,8 @@ describe 'DocumentUpdaterHandler', ->
 
 		@request = {}
 		@projectEntityHandler = {}
-		@rclient = {auth:->}
 		@settings = 
 			apis : documentupdater: url : "http://something.com"
-			redis:{web:{}}
 		@handler = SandboxedModule.require modulePath, requires:
 			'request': defaults:=> return @request
 			'settings-sharelatex':@settings
@@ -31,52 +29,10 @@ describe 'DocumentUpdaterHandler', ->
 			'../Project/ProjectEntityHandler':@projectEntityHandler
 			"../../models/Project": Project: @Project={}
 			'../../Features/Project/ProjectLocator':{}
-			'redis-sharelatex' : createClient: () => @rclient
 			"metrics-sharelatex": 
 				Timer:->
 					done:->
 
-	describe 'queueChange', ->
-		beforeEach ->
-			@change = {
-				"action":"removeText",
-				"range":{"start":{"row":2,"column":2},"end":{"row":2,"column":3}},
-				"text":"e"
-			}
-			@rclient.multi = sinon.stub().returns @rclient
-			@rclient.exec = sinon.stub().callsArg(0)
-			@rclient.rpush = sinon.stub()
-			@rclient.sadd = sinon.stub()
-			@callback = sinon.stub()
-
-		describe "successfully", ->
-			beforeEach ->
-				@handler.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should push the change", ->
-				@rclient.rpush
-					.calledWith("PendingUpdates:#{@doc_id}", JSON.stringify(@change))
-					.should.equal true
-
-			it "should notify the doc updater of the change via the pending-updates-list queue", ->
-				@rclient.rpush
-					.calledWith("pending-updates-list", "#{@project_id}:#{@doc_id}")
-					.should.equal true
-
-			it "should push the doc id into the pending updates set", ->
-				@rclient.sadd
-					.calledWith("DocsWithPendingUpdates", "#{@project_id}:#{@doc_id}")
-					.should.equal true
-
-		describe "with error connecting to redis during exec", ->
-			beforeEach ->
-				@rclient.exec = sinon.stub().callsArgWith(0, new Error("something went wrong"))
-				@handler.queueChange(@project_id, @doc_id, @change, @callback)
-
-			it "should return an error", ->
-				@callback.calledWithExactly(sinon.match(Error)).should.equal true
-
-
 	describe 'flushProjectToMongo', ->
 		beforeEach ->
 			@callback = sinon.stub()

services/web/test/UnitTests/coffee/Editor/EditorRealTimeControllerTests.coffee

@@ -5,16 +5,13 @@ modulePath = require('path').join __dirname, '../../../../app/js/Features/Editor
 
 describe "EditorRealTimeController", ->
 	beforeEach ->
+		@rclient =
+			publish: sinon.stub()
 		@EditorRealTimeController = SandboxedModule.require modulePath, requires:
-			"redis-sharelatex": 
-				createClient: () ->
-					auth:->
+			"../../infrastructure/RedisWrapper": 
+				client: () => @rclient
 			"../../infrastructure/Server" : io: @io = {}
 			"settings-sharelatex":{redis:{}}
-		@EditorRealTimeController.rclientPub = publish: sinon.stub()
-		@EditorRealTimeController.rclientSub =
-			subscribe: sinon.stub()
-			on: sinon.stub()
 		
 		@room_id = "room-id"
 		@message = "message-to-editor"
@@ -25,7 +22,7 @@ describe "EditorRealTimeController", ->
 			@EditorRealTimeController.emitToRoom(@room_id, @message, @payload...)
 
 		it "should publish the message to redis", ->
-			@EditorRealTimeController.rclientPub.publish
+			@rclient.publish
 				.calledWith("editor-events", JSON.stringify(
 					room_id: @room_id,
 					message: @message

services/web/test/UnitTests/coffee/Security/OneTimeTokenHandlerTests.coffee

@@ -23,8 +23,8 @@ describe "OneTimeTokenHandler", ->
 			exec:sinon.stub()
 		self = @
 		@OneTimeTokenHandler = SandboxedModule.require modulePath, requires:
-			"redis-sharelatex" :
-				createClient: =>
+			"../../infrastructure/RedisWrapper" :
+				client: =>
 					auth:->
 					multi: -> return self.redisMulti
 

services/web/test/UnitTests/coffee/infrastructure/LockManager/CheckingTheLock.coffee

@@ -7,7 +7,7 @@ doc_id     = 5678
 blockingKey = "Blocking:#{doc_id}"
 SandboxedModule = require('sandboxed-module')
 
-describe 'Lock Manager - checking the lock', ()->
+describe 'LockManager - checking the lock', ()->
 
 	existsStub = sinon.stub()
 	setStub = sinon.stub()
@@ -17,8 +17,8 @@ describe 'Lock Manager - checking the lock', ()->
 	mocks =
 		"logger-sharelatex": log:->
 
-		"redis-sharelatex":
-			createClient : ()->
+		"./RedisWrapper":
+			client: ()->
 				auth:->
 				multi: ->
 					exists: existsStub

services/web/test/UnitTests/coffee/infrastructure/LockManager/ReleasingTheLock.coffee

@@ -12,8 +12,8 @@ describe 'LockManager - releasing the lock', ()->
 	mocks =
 		"logger-sharelatex": log:->
 
-		"redis-sharelatex":
-			createClient : ()->
+		"./RedisWrapper":
+			client: ()->
 				auth:->
 				del:deleteStub
 	

services/web/test/UnitTests/coffee/infrastructure/LockManager/getLockTests.coffee

@@ -9,8 +9,8 @@ describe 'LockManager - getting the lock', ->
 	beforeEach ->
 		@LockManager = SandboxedModule.require modulePath, requires:
 			"logger-sharelatex": log:->
-			"redis-sharelatex":
-				createClient : () =>
+			"./RedisWrapper":
+				client: ()->
 					auth:->
 			"settings-sharelatex":{redis:{}}
 			"metrics-sharelatex": inc:->

services/web/test/UnitTests/coffee/infrastructure/LockManager/tryLockTests.coffee

@@ -9,8 +9,8 @@ describe 'LockManager - trying the lock', ->
 	beforeEach ->
 		@LockManager = SandboxedModule.require modulePath, requires:
 			"logger-sharelatex": log:->
-			"redis-sharelatex":
-				createClient : () =>
+			"./RedisWrapper":
+				client: () =>
 					auth:->
 					set: @set = sinon.stub()
 			"settings-sharelatex":{redis:{}}

services/web/test/UnitTests/coffee/infrastructure/RedisWrapperTests.coffee

@@ -9,58 +9,28 @@ SandboxedModule = require('sandboxed-module')
 describe 'RedisWrapper', ->
 
 	beforeEach ->
-		@featureName = 'somefeature'
-		@settings =
-			redis:
-				web:
-					port:"1234"
-					host:"somewhere"
-					password: "password"
-				somefeature: {}
-		@normalRedisInstance =
-			thisIsANormalRedisInstance: true
-			n: 1
-		@clusterRedisInstance =
-			thisIsAClusterRedisInstance: true
-			n: 2
+		@settings = { redis: {} }
 		@redis =
-			createClient: sinon.stub().returns(@normalRedisInstance)
-		@ioredis =
-			Cluster: sinon.stub().returns(@clusterRedisInstance)
-		@logger = {log: sinon.stub()}
-
+			createClient: sinon.stub()
 		@RedisWrapper = SandboxedModule.require modulePath, requires:
-			'logger-sharelatex': @logger
 			'settings-sharelatex': @settings
 			'redis-sharelatex': @redis
-			'ioredis': @ioredis
 
 	describe 'client', ->
-
-		beforeEach ->
-			@call = () =>
-				@RedisWrapper.client(@featureName)
-
-		describe 'when feature uses cluster', ->
-
-			beforeEach ->
-				@settings.redis.somefeature =
-					cluster: [1, 2, 3]
-
-			it 'should return a cluster client', ->
-				client = @call()
-				expect(client).to.equal @clusterRedisInstance
-				expect(client.__is_redis_cluster).to.equal true
-
-		describe 'when feature uses normal redis', ->
-
-			beforeEach ->
-				@settings.redis.somefeature =
-					port:"1234"
-					host:"somewhere"
-					password: "password"
-
-			it 'should return a regular redis client', ->
-				client = @call()
-				expect(client).to.equal @normalRedisInstance
-				expect(client.__is_redis_cluster).to.equal undefined
+		it "should use the feature settings if present", ->
+			@settings.redis =
+				my_feature:
+					port:"23456"
+					host:"otherhost"
+					password: "banana"
+			@RedisWrapper.client("my_feature")
+			@redis.createClient.calledWith(@settings.redis.my_feature).should.equal true
+
+		it "should use the web settings if feature not present", ->
+			@settings.redis =
+				web:
+					port:"43"
+					host:"otherhost"
+					password: "banana"
+			@RedisWrapper.client("my_feature")
+			@redis.createClient.calledWith(@settings.redis.web).should.equal true