Merge branch 'master' into sk-unlisted-projects

services/real-time/.nvmrc

@@ -1 +1 @@
-4.2.1
+6.9.5

services/real-time/Jenkinsfile

@@ -0,0 +1,88 @@
+pipeline {
+  
+  agent any
+
+  environment  {
+      HOME = "/tmp"
+  }
+
+  triggers {
+    pollSCM('* * * * *')
+    cron('@daily')
+  }
+
+  stages {
+    stage('Install') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
+          reuseNode true
+        }
+      }
+      steps {
+        // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
+        // which does not exist in the container's /etc/passwd file, causing the clone to fail.
+        sh 'git config --global core.logallrefupdates false'
+        sh 'rm -fr node_modules'
+        sh 'npm install'
+        sh 'npm rebuild'
+        sh 'npm install --quiet grunt-cli'
+      }
+    }
+    stage('Compile and Test') {
+      agent {
+        docker {
+          image 'node:6.9.5'
+          args "-v /var/lib/jenkins/.npm:/tmp/.npm -e HOME=/tmp"
+          reuseNode true
+        }
+      }
+      steps {
+        sh 'node_modules/.bin/grunt install'
+        sh 'node_modules/.bin/grunt compile:acceptance_tests'
+        sh 'node_modules/.bin/grunt test:unit'
+      }
+    }
+    stage('Acceptance Tests') {
+      steps {
+        sh 'docker pull sharelatex/acceptance-test-runner'
+        sh 'docker run --rm -v $(pwd):/app sharelatex/acceptance-test-runner'
+      }
+    }
+    stage('Package') {
+      steps {
+        sh 'echo ${BUILD_NUMBER} > build_number.txt'
+        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
+        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
+      }
+    }
+    stage('Publish') {
+      steps {
+        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
+            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
+            // The deployment process uses this file to figure out the latest build
+            s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
+        }
+      }
+    }
+  }
+  
+  post {
+    failure {
+      mail(from: "${EMAIL_ALERT_FROM}", 
+           to: "${EMAIL_ALERT_TO}", 
+           subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
+           body: "Build: ${BUILD_URL}")
+    }
+  }
+  
+  // The options directive is for configuration that applies to the whole job.
+  options {
+    // we'd like to make sure remove old builds, so we don't fill up our storage!
+    buildDiscarder(logRotator(numToKeepStr:'50'))
+    
+    // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
+    timeout(time: 30, unit: 'MINUTES')
+  }
+}

services/real-time/app.coffee

@@ -5,6 +5,8 @@ express = require("express")
 session = require("express-session")
 redis = require("redis-sharelatex")
 Settings = require "settings-sharelatex"
+if Settings.sentry?.dsn?
+	logger.initializeErrorReporting(Settings.sentry.dsn)
 
 sessionRedisClient = redis.createClient(Settings.redis.websessions)
 

services/real-time/app/coffee/DocumentUpdaterManager.coffee

@@ -54,6 +54,10 @@ module.exports = DocumentUpdaterManager =
 				
 	queueChange: (project_id, doc_id, change, callback = ()->)->
 		jsonChange = JSON.stringify change
+		if jsonChange.indexOf("\u0000") != -1
+			error = new Error("null bytes found in op")
+			logger.error err: error, project_id: project_id, doc_id: doc_id, jsonChange: jsonChange, error.message
+			return callback(error)
 		doc_key = "#{project_id}:#{doc_id}"
 		# Push onto pendingUpdates for doc_id first, because once the doc updater
 		# gets an entry on pending-updates-list, it starts processing.

services/real-time/app/coffee/Router.coffee

@@ -81,14 +81,30 @@ module.exports = Router =
 					if err?
 						Router._handleError null, err, client, "leaveProject"
 
-
-			client.on "joinDoc", (doc_id, fromVersion, callback) ->
-				# fromVersion is optional
-				if typeof fromVersion == "function"
+			# Variadic. The possible arguments:
+			# doc_id, callback
+			# doc_id, fromVersion, callback
+			# doc_id, options, callback
+			# doc_id, fromVersion, options, callback
+			client.on "joinDoc", (doc_id, fromVersion, options, callback) ->
+				if typeof fromVersion == "function" and !options
 					callback = fromVersion
 					fromVersion = -1
+					options = {}
+				else if typeof fromVersion == "number" and typeof options == "function"
+					callback = options
+					options = {}
+				else if typeof fromVersion == "object" and typeof options == "function"
+					callback = options
+					options = fromVersion
+					fromVersion = -1
+				else if typeof fromVersion == "number" and typeof options == "object"
+					# Called with 4 args, things are as expected
+				else
+					logger.error { arguments: arguments }, "unexpected arguments"
+					return callback?(new Error("unexpected arguments"))
 
-				WebsocketController.joinDoc client, doc_id, fromVersion, (err, args...) ->
+				WebsocketController.joinDoc client, doc_id, fromVersion, options, (err, args...) ->
 					if err?
 						Router._handleError callback, err, client, "joinDoc", {doc_id, fromVersion}
 					else

services/real-time/app/coffee/WebsocketController.coffee

@@ -80,7 +80,7 @@ module.exports = WebsocketController =
 				callback()
 			, WebsocketController.FLUSH_IF_EMPTY_DELAY
 			
-	joinDoc: (client, doc_id, fromVersion = -1, callback = (error, doclines, version, ops, ranges) ->) ->
+	joinDoc: (client, doc_id, fromVersion = -1, options, callback = (error, doclines, version, ops, ranges) ->) ->
 		metrics.inc "editor.join-doc"
 		Utils.getClientAttributes client, ["project_id", "user_id"], (error, {project_id, user_id}) ->
 			return callback(error) if error?
@@ -91,16 +91,29 @@ module.exports = WebsocketController =
 				return callback(error) if error?
 				DocumentUpdaterManager.getDocument project_id, doc_id, fromVersion, (error, lines, version, ranges, ops) ->
 					return callback(error) if error?
+
 					# Encode any binary bits of data so it can go via WebSockets
 					# See http://ecmanaut.blogspot.co.uk/2006/07/encoding-decoding-utf8-in-javascript.html
+					encodeForWebsockets = (text) -> unescape(encodeURIComponent(text))
 					escapedLines = []
 					for line in lines
 						try
-							line = unescape(encodeURIComponent(line))
+							line = encodeForWebsockets(line)
 						catch err
 							logger.err {err, project_id, doc_id, fromVersion, line, client_id: client.id}, "error encoding line uri component"
 							return callback(err)
 						escapedLines.push line
+					if options.encodeRanges
+						try
+							for comment in ranges?.comments or []
+								comment.op.c = encodeForWebsockets(comment.op.c) if comment.op.c?
+							for change in ranges?.changes or []
+								change.op.i = encodeForWebsockets(change.op.i) if change.op.i?
+								change.op.d = encodeForWebsockets(change.op.d) if change.op.d?
+						catch err
+							logger.err {err, project_id, doc_id, fromVersion, ranges, client_id: client.id}, "error encoding range uri component"
+							return callback(err)
+
 					AuthorizationManager.addAccessToDoc client, doc_id
 					client.join(doc_id)
 					callback null, escapedLines, version, ops, ranges

services/real-time/package.json

@@ -15,10 +15,9 @@
     "cookie-parser": "^1.3.3",
     "express": "^4.10.1",
     "express-session": "^1.9.1",
-    "ioredis": "^2.4.0",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0",
+    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6",
     "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.4.0",
-    "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.2",
+    "redis-sharelatex": "git+https://github.com/sharelatex/redis-sharelatex.git#v1.0.4",
     "request": "~2.34.0",
     "session.socket.io": "^0.1.6",
     "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",

services/real-time/test/acceptance/coffee/JoinDocTests.coffee

@@ -166,4 +166,81 @@ describe "joinDoc", ->
 		it "should have joined the doc room", (done) ->
 			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
 				expect(@doc_id in client.rooms).to.equal true
-				done()
+				done()
+
+	describe "with options", ->
+		before (done) ->
+			@options = { encodeRanges: true }
+			async.series [
+				(cb) =>
+					FixturesManager.setUpProject {
+						privilegeLevel: "readAndWrite"
+					}, (e, {@project_id, @user_id}) =>
+						cb(e)
+
+				(cb) =>
+					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
+						cb(e)
+
+				(cb) =>
+					@client = RealTimeClient.connect()
+					@client.on "connectionAccepted", cb
+
+				(cb) =>
+					@client.emit "joinProject", project_id: @project_id, cb
+
+				(cb) =>
+					@client.emit "joinDoc", @doc_id, @options, (error, @returnedArgs...) => cb(error)
+			], done
+
+		it "should get the doc from the doc updater with the default fromVersion", ->
+			MockDocUpdaterServer.getDocument
+				.calledWith(@project_id, @doc_id, -1)
+				.should.equal true
+
+		it "should return the doc lines, version, ranges and ops", ->
+			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
+
+		it "should have joined the doc room", (done) ->
+			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
+				expect(@doc_id in client.rooms).to.equal true
+				done()
+
+	describe "with fromVersion and options", ->
+		before (done) ->
+			@fromVersion = 36
+			@options = { encodeRanges: true }
+			async.series [
+				(cb) =>
+					FixturesManager.setUpProject {
+						privilegeLevel: "readAndWrite"
+					}, (e, {@project_id, @user_id}) =>
+						cb(e)
+
+				(cb) =>
+					FixturesManager.setUpDoc @project_id, {@lines, @version, @ops, @ranges}, (e, {@doc_id}) =>
+						cb(e)
+
+				(cb) =>
+					@client = RealTimeClient.connect()
+					@client.on "connectionAccepted", cb
+
+				(cb) =>
+					@client.emit "joinProject", project_id: @project_id, cb
+
+				(cb) =>
+					@client.emit "joinDoc", @doc_id, @fromVersion, @options, (error, @returnedArgs...) => cb(error)
+			], done
+
+		it "should get the doc from the doc updater with the fromVersion", ->
+			MockDocUpdaterServer.getDocument
+				.calledWith(@project_id, @doc_id, @fromVersion)
+				.should.equal true
+
+		it "should return the doc lines, version, ranges and ops", ->
+			@returnedArgs.should.deep.equal [@lines, @version, @ops, @ranges]
+
+		it "should have joined the doc room", (done) ->
+			RealTimeClient.getConnectedClient @client.socket.sessionid, (error, client) =>
+				expect(@doc_id in client.rooms).to.equal true
+				done()

services/real-time/test/acceptance/coffee/helpers/MockDocUpdaterServer.coffee

@@ -38,5 +38,9 @@ module.exports = MockDocUpdaterServer =
 		app.listen 3003, (error) ->
 			MockDocUpdaterServer.running = true
 			callback(error)
+		.on "error", (error) ->
+			console.error "error starting MockDocUpdaterServer:", error.message
+			process.exit(1)
+
 			
-sinon.spy MockDocUpdaterServer, "getDocument"
+sinon.spy MockDocUpdaterServer, "getDocument"

services/real-time/test/acceptance/coffee/helpers/MockWebServer.coffee

@@ -35,5 +35,9 @@ module.exports = MockWebServer =
 		app.listen 3000, (error) ->
 			MockWebServer.running = true
 			callback(error)
+		.on "error", (error) ->
+			console.error "error starting MockWebServer:", error.message
+			process.exit(1)
+
 			
-sinon.spy MockWebServer, "joinProject"
+sinon.spy MockWebServer, "joinProject"

services/real-time/test/unit/coffee/DocumentUpdaterManagerTests.coffee

@@ -17,14 +17,17 @@ describe 'DocumentUpdaterManager', ->
 					pendingUpdates: ({doc_id}) -> "PendingUpdates:#{doc_id}"
 		@rclient = {auth:->}
 			
-		@DocumentUpdaterManager = SandboxedModule.require modulePath, requires:
-			'settings-sharelatex':@settings
-			'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()}
-			'request': @request = {}
-			'redis-sharelatex' : createClient: () => @rclient
-			'metrics-sharelatex': @Metrics =
-				Timer: class Timer
-					done: () ->
+		@DocumentUpdaterManager = SandboxedModule.require modulePath,
+			requires:
+				'settings-sharelatex':@settings
+				'logger-sharelatex': @logger = {log: sinon.stub(), error: sinon.stub(), warn: sinon.stub()}
+				'request': @request = {}
+				'redis-sharelatex' : createClient: () => @rclient
+				'metrics-sharelatex': @Metrics =
+					Timer: class Timer
+						done: () ->
+			globals:
+				JSON: @JSON = Object.create(JSON) # avoid modifying JSON object directly
 
 	describe "getDocument", ->
 		beforeEach ->
@@ -147,3 +150,14 @@ describe 'DocumentUpdaterManager', ->
 
 			it "should return an error", ->
 				@callback.calledWithExactly(sinon.match(Error)).should.equal true
+
+		describe "with null byte corruption", ->
+			beforeEach ->
+				@JSON.stringify = () -> return '["bad bytes! \u0000 <- here"]'
+				@DocumentUpdaterManager.queueChange(@project_id, @doc_id, @change, @callback)
+
+			it "should return an error", ->
+				@callback.calledWithExactly(sinon.match(Error)).should.equal true
+
+			it "should not push the change onto the pending-updates-list queue", ->
+				@rclient.rpush.called.should.equal false

services/real-time/test/unit/coffee/WebsocketControllerTests.coffee

@@ -221,49 +221,59 @@ describe 'WebsocketController', ->
 			@version = 42
 			@ops = ["mock", "ops"]
 			@ranges = { "mock": "ranges" }
+			@options = {}
 			
 			@client.params.project_id = @project_id
 			@AuthorizationManager.addAccessToDoc = sinon.stub()
 			@AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, null)
 			@DocumentUpdaterManager.getDocument = sinon.stub().callsArgWith(3, null, @doc_lines, @version, @ranges, @ops)
-			
-		describe "with a fromVersion", ->
+
+		describe "works", ->
 			beforeEach ->
-				@fromVersion = 40
-				@WebsocketController.joinDoc @client, @doc_id, @fromVersion, @callback
-				
+				@WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback
+
 			it "should check that the client is authorized to view the project", ->
 				@AuthorizationManager.assertClientCanViewProject
 					.calledWith(@client)
 					.should.equal true
-					
-			it "should get the document from the DocumentUpdaterManager", ->
+
+			it "should get the document from the DocumentUpdaterManager with fromVersion", ->
 				@DocumentUpdaterManager.getDocument
-					.calledWith(@project_id, @doc_id, @fromVersion)
+					.calledWith(@project_id, @doc_id, -1)
 					.should.equal true
 
 			it "should add permissions for the client to access the doc", ->
 				@AuthorizationManager.addAccessToDoc
 					.calledWith(@client, @doc_id)
 					.should.equal true
-					
+
 			it "should join the client to room for the doc_id", ->
 				@client.join
 					.calledWith(@doc_id)
 					.should.equal true
-					
+
 			it "should call the callback with the lines, version, ranges and ops", ->
 				@callback
 					.calledWith(null, @doc_lines, @version, @ops, @ranges)
 					.should.equal true
-					
+
 			it "should increment the join-doc metric", ->
 				@metrics.inc.calledWith("editor.join-doc").should.equal true
+
+		describe "with a fromVersion", ->
+			beforeEach ->
+				@fromVersion = 40
+				@WebsocketController.joinDoc @client, @doc_id, @fromVersion, @options, @callback
 					
+			it "should get the document from the DocumentUpdaterManager with fromVersion", ->
+				@DocumentUpdaterManager.getDocument
+					.calledWith(@project_id, @doc_id, @fromVersion)
+					.should.equal true
+
 		describe "with doclines that need escaping", ->
 			beforeEach ->
 				@doc_lines.push ["räksmörgås"]
-				@WebsocketController.joinDoc @client, @doc_id, -1, @callback
+				@WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback
 						
 			it "should call the callback with the escaped lines", ->
 				escaped_lines = @callback.args[0][1]
@@ -271,11 +281,41 @@ describe 'WebsocketController', ->
 				escaped_word.should.equal 'räksmörgås'
 				# Check that unescaping works
 				decodeURIComponent(escape(escaped_word)).should.equal "räksmörgås"
-				
+
+		describe "with comments that need encoding", ->
+			beforeEach ->
+				@ranges.comments = [{ op: { c: "räksmörgås" } }]
+				@WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback
+
+			it "should call the callback with the encoded comment", ->
+				encoded_comments = @callback.args[0][4]
+				encoded_comment = encoded_comments.comments.pop()
+				encoded_comment_text = encoded_comment.op.c
+				encoded_comment_text.should.equal 'räksmörgås'
+
+		describe "with changes that need encoding", ->
+			it "should call the callback with the encoded insert change", ->
+				@ranges.changes = [{ op: { i: "räksmörgås" } }]
+				@WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback
+
+				encoded_changes = @callback.args[0][4]
+				encoded_change = encoded_changes.changes.pop()
+				encoded_change_text = encoded_change.op.i
+				encoded_change_text.should.equal 'räksmörgås'
+
+			it "should call the callback with the encoded delete change", ->
+				@ranges.changes = [{ op: { d: "räksmörgås" } }]
+				@WebsocketController.joinDoc @client, @doc_id, -1, { encodeRanges: true }, @callback
+
+				encoded_changes = @callback.args[0][4]
+				encoded_change = encoded_changes.changes.pop()
+				encoded_change_text = encoded_change.op.d
+				encoded_change_text.should.equal 'räksmörgås'
+
 		describe "when not authorized", ->
 			beforeEach ->
 				@AuthorizationManager.assertClientCanViewProject = sinon.stub().callsArgWith(1, @err = new Error("not authorized"))
-				@WebsocketController.joinDoc @client, @doc_id, -1, @callback
+				@WebsocketController.joinDoc @client, @doc_id, -1, @options, @callback
 				
 			it "should call the callback with an error", ->
 				@callback.calledWith(@err).should.equal true