add sha1 hash support on writes

services/document-updater/app/coffee/RedisKeyBuilder.coffee

@@ -28,6 +28,8 @@ module.exports = RedisKeyBuilder =
 		return (key_schema) -> key_schema.docOps({doc_id})
 	docVersion: ({doc_id}) ->
 		return (key_schema) -> key_schema.docVersion({doc_id})
+	docHash: ({doc_id}) ->
+		return (key_schema) -> key_schema.docHash({doc_id})
 	projectKey: ({doc_id}) ->
 		return (key_schema) -> key_schema.projectKey({doc_id})
 	uncompressedHistoryOp: ({doc_id}) ->

services/document-updater/app/coffee/RedisManager.coffee

@@ -6,6 +6,7 @@ keys = require('./RedisKeyBuilder')
 logger = require('logger-sharelatex')
 metrics = require('./Metrics')
 Errors = require "./Errors"
+crypto = require "crypto"
 
 # Make times easy to read
 minutes = 60 # seconds for Redis expire
@@ -18,12 +19,15 @@ module.exports = RedisManager =
 		callback = (error) ->
 			timer.done()
 			_callback(error)
-		logger.log project_id:project_id, doc_id:doc_id, version: version, "putting doc in redis"
+		docLines = JSON.stringify(docLines)
+		docHash = RedisManager._computeHash(docLines)
+		logger.log project_id:project_id, doc_id:doc_id, version: version, hash:docHash, "putting doc in redis"
 		ranges = RedisManager._serializeRanges(ranges)
 		multi = rclient.multi()
-		multi.set keys.docLines(doc_id:doc_id), JSON.stringify(docLines)
+		multi.set keys.docLines(doc_id:doc_id), docLines
 		multi.set keys.projectKey({doc_id:doc_id}), project_id
 		multi.set keys.docVersion(doc_id:doc_id), version
+		multi.set keys.docHash(doc_id:doc_id), docHash
 		if ranges?
 			multi.set keys.ranges(doc_id:doc_id), ranges
 		else
@@ -46,6 +50,7 @@ module.exports = RedisManager =
 		multi.del keys.docLines(doc_id:doc_id)
 		multi.del keys.projectKey(doc_id:doc_id)
 		multi.del keys.docVersion(doc_id:doc_id)
+		multi.del keys.docHash(doc_id:doc_id)
 		multi.del keys.ranges(doc_id:doc_id)
 		multi.exec (error) ->
 			return callback(error) if error?
@@ -56,11 +61,17 @@ module.exports = RedisManager =
 		multi = rclient.multi()
 		multi.get keys.docLines(doc_id:doc_id)
 		multi.get keys.docVersion(doc_id:doc_id)
+		multi.get keys.docHash(doc_id:doc_id)
 		multi.get keys.projectKey(doc_id:doc_id)
 		multi.get keys.ranges(doc_id:doc_id)
-		multi.exec (error, [docLines, version, doc_project_id, ranges])->
+		multi.exec (error, [docLines, version, storedHash, doc_project_id, ranges])->
 			timer.done()
 			return callback(error) if error?
+			if docLines?
+				computedHash = RedisManager._computeHash(docLines)
+				if computedHash isnt storedHash
+					logger.error project_id: project_id, doc_id: doc_id, doc_project_id: doc_project_id, computedHash: computedHash, storedHash: storedHash, "hash mismatch on retrieved document"
+
 			try
 				docLines = JSON.parse docLines
 				ranges = RedisManager._deserializeRanges(ranges)
@@ -121,8 +132,11 @@ module.exports = RedisManager =
 				return callback(error)
 			jsonOps = appliedOps.map (op) -> JSON.stringify op
 			multi = rclient.multi()
-			multi.set    keys.docLines(doc_id:doc_id), JSON.stringify(docLines)
+			newDocLines = JSON.stringify(docLines)
+			newHash = RedisManager._computeHash(newDocLines)
+			multi.set    keys.docLines(doc_id:doc_id), newDocLines
 			multi.set    keys.docVersion(doc_id:doc_id), newVersion
+			multi.set    keys.docHash(doc_id:doc_id), newHash
 			if jsonOps.length > 0
 				multi.rpush  keys.docOps(doc_id: doc_id), jsonOps...
 			multi.expire keys.docOps(doc_id: doc_id), RedisManager.DOC_OPS_TTL
@@ -150,4 +164,8 @@ module.exports = RedisManager =
 		if !ranges? or ranges == ""
 			return {}
 		else
-			return JSON.parse(ranges)
+			return JSON.parse(ranges)
+
+	_computeHash: (docLines) ->
+		# use sha1 checksum of doclines to detect data corruption
+		return crypto.createHash('sha1').update(docLines).digest('hex')

services/document-updater/config/settings.defaults.coffee

@@ -30,6 +30,7 @@ module.exports =
 				docLines: ({doc_id}) -> "doclines:#{doc_id}"
 				docOps: ({doc_id}) -> "DocOps:#{doc_id}"
 				docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
+				docHash: ({doc_id}) -> "DocHash:#{doc_id}"
 				projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
 				docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
 				ranges: ({doc_id}) -> "Ranges:#{doc_id}"

services/document-updater/test/unit/coffee/RedisBackend/RedisBackendTests.coffee

@@ -19,6 +19,7 @@ describe "RedisBackend", ->
 						docLines: ({doc_id}) -> "doclines:#{doc_id}"
 						docOps: ({doc_id}) -> "DocOps:#{doc_id}"
 						docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
+						docHash: ({doc_id}) -> "DocHash:#{doc_id}"
 						projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
 						pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
 						docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
@@ -33,6 +34,7 @@ describe "RedisBackend", ->
 						docLines: ({doc_id}) -> "doclines:{#{doc_id}}"
 						docOps: ({doc_id}) -> "DocOps:{#{doc_id}}"
 						docVersion: ({doc_id}) -> "DocVersion:{#{doc_id}}"
+						docHash: ({doc_id}) -> "DocHash:{#{doc_id}}"
 						projectKey: ({doc_id}) -> "ProjectId:{#{doc_id}}"
 						pendingUpdates: ({doc_id}) -> "PendingUpdates:{#{doc_id}}"
 						docsInProject: ({project_id}) -> "DocsIn:{#{project_id}}"

services/document-updater/test/unit/coffee/RedisManager/RedisManagerTests.coffee

@@ -4,6 +4,7 @@ should = chai.should()
 modulePath = "../../../../app/js/RedisManager.js"
 SandboxedModule = require('sandboxed-module')
 Errors = require "../../../../app/js/Errors"
+crypto = require "crypto"
 
 describe "RedisManager", ->
 	beforeEach ->
@@ -19,6 +20,7 @@ describe "RedisManager", ->
 				docLines: ({doc_id}) -> "doclines:#{doc_id}"
 				docOps: ({doc_id}) -> "DocOps:#{doc_id}"
 				docVersion: ({doc_id}) -> "DocVersion:#{doc_id}"
+				docHash: ({doc_id}) -> "DocHash:#{doc_id}"
 				projectKey: ({doc_id}) -> "ProjectId:#{doc_id}"
 				pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
 				docsInProject: ({project_id}) -> "DocsIn:#{project_id}"
@@ -38,10 +40,11 @@ describe "RedisManager", ->
 			@lines = ["one", "two", "three"]
 			@jsonlines = JSON.stringify @lines
 			@version = 42
+			@hash = crypto.createHash('sha1').update(@jsonlines).digest('hex')
 			@ranges = { comments: "mock", entries: "mock" }
 			@json_ranges = JSON.stringify @ranges
 			@rclient.get = sinon.stub()
-			@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @project_id, @json_ranges])
+			@rclient.exec = sinon.stub().callsArgWith(0, null, [@jsonlines, @version, @hash, @project_id, @json_ranges])
 
 		describe "successfully", ->
 			beforeEach ->