dockerise app

- 1.1.3 build scripts
- add env vars for configuring s3
- add docker file

services/track-changes/.dockerignore

@@ -0,0 +1,9 @@
+node_modules/*
+gitrev
+.git
+.gitignore
+.npm
+.nvmrc
+nodemon.json
+app.js
+**/js/*

services/track-changes/Dockerfile

@@ -0,0 +1,22 @@
+FROM node:6.11.2 as app
+
+WORKDIR /app
+
+#wildcard as some files may not be in all repos
+COPY package*.json npm-shrink*.json /app/
+
+RUN npm install --quiet
+
+COPY . /app
+
+
+RUN npm run compile:all
+
+FROM node:6.11.2
+
+COPY --from=app /app /app
+
+WORKDIR /app
+USER node
+
+CMD ["node","app.js"]

services/track-changes/Jenkinsfile

@@ -1,57 +1,44 @@
-pipeline {
-
-  agent {
-    docker {
-      image 'node:6.11.2'
-      args "-v /var/lib/jenkins/.npm:/tmp/.npm"
-    }
-  }
+String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
 
-  environment  {
-      HOME = "/tmp"
-  }
+pipeline {
+  agent any
 
   triggers {
     pollSCM('* * * * *')
-    cron('@daily')
+    cron(cron_string)
   }
 
   stages {
-    stage('Set up') {
-      steps {
-        // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
-        // which does not exist in the container's /etc/passwd file, causing the clone to fail.
-        sh 'git config --global core.logallrefupdates false'
-      }
-    }
-    stage('Install') {
+    stage('Build') {
       steps {
-        sh 'rm -fr node_modules'
-        sh 'npm install'
-        sh 'npm install --quiet grunt-cli'
+        sh 'make build'
       }
     }
-    stage('Compile') {
+
+    stage('Unit Tests') {
       steps {
-        sh 'node_modules/.bin/grunt compile'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
       }
     }
-    stage('Test') {
+
+    stage('Acceptance Tests') {
       steps {
-        sh 'NODE_ENV=development node_modules/.bin/grunt test:unit'
+        withCredentials([usernamePassword(credentialsId: 'S3_DOCSTORE_TEST_AWS_KEYS', passwordVariable: 'AWS_SECRET_ACCESS_KEY', usernameVariable: 'AWS_ACCESS_KEY_ID')]) {
+          sh 'AWS_BUCKET="sl-doc-archive-testing" AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
+        }
       }
     }
-    stage('Package') {
+
+    stage('Package and publish build') {
       steps {
-        sh 'echo ${BUILD_NUMBER} > build_number.txt'
-        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
-        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
+        sh 'make publish'
       }
     }
-    stage('Publish') {
+
+    stage('Publish build number') {
       steps {
+        sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
         withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
-            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
             // The deployment process uses this file to figure out the latest build
             s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
         }
@@ -60,6 +47,10 @@ pipeline {
   }
 
   post {
+    always {
+      sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
+    }
+
     failure {
       mail(from: "${EMAIL_ALERT_FROM}",
            to: "${EMAIL_ALERT_TO}",

services/track-changes/Makefile

@@ -0,0 +1,45 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.3
+
+BUILD_NUMBER ?= local
+BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+PROJECT_NAME = track-changes
+DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
+DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	AWS_BUCKET=${AWS_BUCKET} \
+	AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
+	AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
+	docker-compose ${DOCKER_COMPOSE_FLAGS} 
+
+
+clean:
+	rm -f app.js
+	rm -rf app/js
+	rm -rf test/unit/js
+	rm -rf test/acceptance/js
+
+test: test_unit test_acceptance
+
+test_unit:
+	@[ ! -d test/unit ] && echo "track-changes has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
+
+test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run
+	@[ ! -d test/acceptance ] && echo "track-changes has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
+
+test_clean:
+	$(DOCKER_COMPOSE) down -v -t 0
+
+test_acceptance_pre_run:
+	@[ ! -f test/acceptance/scripts/pre-run ] && echo "track-changes has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
+build:
+	docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
+
+publish:
+	docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+
+.PHONY: clean test test_unit test_acceptance test_clean build publish

services/track-changes/app.coffee

@@ -92,9 +92,13 @@ app.use (error, req, res, next) ->
 
 port = Settings.internal?.trackchanges?.port or 3015
 host = Settings.internal?.trackchanges?.host or "localhost"
-app.listen port, host, (error) ->
-	if error?
-		logger.error err: error, "could not start track-changes server"
-	else
-		logger.info "trackchanges starting up, listening on #{host}:#{port}"
+
+if !module.parent # Called directly
+	app.listen port, host, (error) ->
+		if error?
+			logger.error err: error, "could not start track-changes server"
+		else
+			logger.info "trackchanges starting up, listening on #{host}:#{port}"
+
+module.exports = app
 

services/track-changes/config/settings.defaults.coffee

@@ -35,11 +35,10 @@ module.exports =
 
 	trackchanges:
 		s3:
-			key: ""
-			secret: ""
+			key: process.env['AWS_ACCESS_KEY_ID']
+			secret: process.env['AWS_SECRET_ACCESS_KEY']
 		stores:
-			doc_history: ""
-
-
+			doc_history: process.env['AWS_BUCKET']
+			
 	path:
 		dumpFolder:   Path.join(TMP_DIR, "dumpFolder")

services/track-changes/docker-compose.ci.yml

@@ -0,0 +1,35 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.3
+
+version: "2"
+
+services:
+  test_unit:
+    image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    user: node
+    command: npm run test:unit:_run
+
+  test_acceptance:
+    build: .
+    image: gcr.io/csh-gcdm-test/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    environment:
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_BUCKET: ${AWS_BUCKET}
+    depends_on:
+      - mongo
+      - redis
+    user: node
+    command: npm run test:acceptance:_run
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4
+

services/track-changes/docker-compose.yml

@@ -0,0 +1,42 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.3
+
+version: "2"
+
+services:
+  test_unit:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      MOCHA_GREP: ${MOCHA_GREP}
+    command: npm run test:unit
+    user: node
+
+  test_acceptance:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_BUCKET: ${AWS_BUCKET}
+      MOCHA_GREP: ${MOCHA_GREP}
+    user: node
+    depends_on:
+      - mongo
+      - redis
+    command: npm run test:acceptance
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4
+

services/track-changes/nodemon.json

@@ -0,0 +1,19 @@
+{
+  "ignore": [
+    ".git",
+    "node_modules/"
+  ],
+  "verbose": true,
+  "legacyWatch": true,
+  "execMap": {
+    "js": "npm run start"
+  },
+
+  "watch": [
+    "app/coffee/",
+    "app.coffee",
+    "config/"
+  ],
+  "ext": "coffee"
+
+}

services/track-changes/package.json

@@ -7,8 +7,16 @@
     "url": "https://github.com/sharelatex/track-changes-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
-    "start": "npm run compile:app && node app.js"
+    "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
+    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY AWS_BUCKET=$AWS_BUCKET AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
+    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
+    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
+    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
+    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
+    "nodemon": "nodemon --config nodemon.json"
   },
   "dependencies": {
     "JSONStream": "^1.0.4",
@@ -44,6 +52,7 @@
     "grunt-available-tasks": "~0.4.2",
     "grunt-contrib-coffee": "~0.10.1",
     "bunyan": "~2.0.2",
+    "mocha": "^4.0.1",
     "grunt-bunyan": "~0.5.0",
     "grunt-forever": "~0.4.2",
     "timekeeper": "0.0.4",

services/track-changes/test/acceptance/coffee/AppendingUpdatesTests.coffee

@@ -6,12 +6,18 @@ mongojs = require "../../../app/js/mongojs"
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 request = require "request"
-rclient = require("redis").createClient() # Only works locally for now
+console.log "hiiiiis"
+console.log Settings.redis.history
+rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
 
+TrackChangesApp = require "./helpers/TrackChangesApp"
 TrackChangesClient = require "./helpers/TrackChangesClient"
 MockWebApi = require "./helpers/MockWebApi"
 
 describe "Appending doc ops to the history", ->
+	before (done)->
+		TrackChangesApp.ensureRunning done
+
 	describe "when the history does not exist yet", ->
 		before (done) ->
 			@project_id = ObjectId().toString()
@@ -81,6 +87,7 @@ describe "Appending doc ops to the history", ->
 
 		describe "when the updates are recent and from the same user", ->
 			beforeEach (done) ->
+				console.log 1
 				TrackChangesClient.pushRawUpdates @project_id, @doc_id, [{
 					op: [{ i: "b", p: 6 }]
 					meta: { ts: Date.now(), user_id: @user_id }
@@ -94,9 +101,13 @@ describe "Appending doc ops to the history", ->
 					meta: { ts: Date.now(), user_id: @user_id }
 					v: 8
 				}], (error) =>
+					console.log 2, error
 					throw error if error?
+					console.log 3
 					TrackChangesClient.flushAndGetCompressedUpdates @project_id, @doc_id, (error, @updates) =>
+						console.log 4, error
 						throw error if error?
+						console.log 5
 						done()
 
 			it "should combine all the updates into one pack", ->

services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee

@@ -7,8 +7,9 @@ db = mongojs.db
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 request = require "request"
-rclient = require("redis").createClient() # Only works locally for now
+rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
 
+TrackChangesApp = require "./helpers/TrackChangesApp"
 TrackChangesClient = require "./helpers/TrackChangesClient"
 MockDocStoreApi = require "./helpers/MockDocStoreApi"
 MockWebApi = require "./helpers/MockWebApi"
@@ -53,12 +54,12 @@ describe "Archiving updates", ->
 				meta: { ts: @now + (i-2048) * @hours + 10*@minutes, user_id: @user_id }
 				v: 2 * i + 2
 			}
-
-		TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-			throw error if error?
-			TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
+		TrackChangesApp.ensureRunning =>
+			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
 				throw error if error?
-				done()
+				TrackChangesClient.flushDoc @project_id, @doc_id, (error) ->
+					throw error if error?
+					done()
 
 	after (done) ->
 		MockWebApi.getUserInfo.restore()

services/track-changes/test/acceptance/coffee/FlushingUpdatesTests.coffee

@@ -6,12 +6,16 @@ mongojs = require "../../../app/js/mongojs"
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 request = require "request"
-rclient = require("redis").createClient() # Only works locally for now
+rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
 
+TrackChangesApp = require "./helpers/TrackChangesApp"
 TrackChangesClient = require "./helpers/TrackChangesClient"
 MockWebApi = require "./helpers/MockWebApi"
 
 describe "Flushing updates", ->
+	before (done)->
+		TrackChangesApp.ensureRunning done
+		
 	describe "flushing a doc's updates", ->
 		before (done) ->
 			@project_id = ObjectId().toString()

services/track-changes/test/acceptance/coffee/GettingADiffTests.coffee

@@ -7,12 +7,14 @@ db = mongojs.db
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 
+TrackChangesApp = require "./helpers/TrackChangesApp"
 TrackChangesClient = require "./helpers/TrackChangesClient"
 MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi"
 MockWebApi = require "./helpers/MockWebApi"
 
 describe "Getting a diff", ->
-	before (done) ->
+
+	beforeEach (done) ->
 		sinon.spy MockDocUpdaterApi, "getDoc"
 
 		@now = Date.now()
@@ -58,15 +60,15 @@ describe "Getting a diff", ->
 		MockDocUpdaterApi.docs[@doc_id] =
 			lines: @lines
 			version: 7
-
-		TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-			throw error if error?
-			TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) =>
+		TrackChangesApp.ensureRunning =>
+			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
 				throw error if error?
-				@diff = diff.diff
-				done()
+				TrackChangesClient.getDiff @project_id, @doc_id, @fromVersion, @toVersion, (error, diff) =>
+					throw error if error?
+					@diff = diff.diff
+					done()
 
-	after () ->
+	afterEach () ->
 		MockDocUpdaterApi.getDoc.restore()
 		MockWebApi.getUserInfo.restore()
 

services/track-changes/test/acceptance/coffee/GettingUpdatesTests.coffee

@@ -7,6 +7,7 @@ db = mongojs.db
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 
+TrackChangesApp = require "./helpers/TrackChangesApp"
 TrackChangesClient = require "./helpers/TrackChangesClient"
 MockWebApi = require "./helpers/MockWebApi"
 
@@ -46,10 +47,11 @@ describe "Getting updates", ->
 				v: 2 * i + 2
 			}
 		@updates[0].meta.user_id = @deleted_user_id
-
-		TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-			throw error if error?
-			done()
+		
+		TrackChangesApp.ensureRunning =>
+			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
+				throw error if error?
+				done()
 
 	after: () ->
 		MockWebApi.getUserInfo.restore()

services/track-changes/test/acceptance/coffee/LockManagerTests.coffee

@@ -6,9 +6,14 @@ mongojs = require "../../../app/js/mongojs"
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 LockManager = require "../../../app/js/LockManager"
-rclient = require("redis").createClient() # Only works locally for now
+rclient = require("redis").createClient(Settings.redis.history) # Only works locally for now
+TrackChangesApp = require "./helpers/TrackChangesApp"
 
 describe "Locking document", ->
+
+	before (done)->
+		TrackChangesApp.ensureRunning done
+		
 	describe "when the lock has expired in redis", ->
 		before (done) ->
 			LockManager.LOCK_TTL = 1 # second

services/track-changes/test/acceptance/coffee/RestoringVersions.coffee

@@ -7,6 +7,7 @@ db = mongojs.db
 ObjectId = mongojs.ObjectId
 Settings = require "settings-sharelatex"
 
+TrackChangesApp = require "./helpers/TrackChangesApp"
 TrackChangesClient = require "./helpers/TrackChangesClient"
 MockDocUpdaterApi = require "./helpers/MockDocUpdaterApi"
 MockWebApi = require "./helpers/MockWebApi"
@@ -54,11 +55,12 @@ describe "Restoring a version", ->
 			lines: @lines
 			version: 7
 
-		TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
-			throw error if error?
-			TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) =>
+		TrackChangesApp.ensureRunning =>
+			TrackChangesClient.pushRawUpdates @project_id, @doc_id, @updates, (error) =>
 				throw error if error?
-				done()
+				TrackChangesClient.restoreDoc @project_id, @doc_id, @beforeVersion, @user_id, (error) =>
+					throw error if error?
+					done()
 
 	after () ->
 		MockDocUpdaterApi.setDoc.restore()

services/track-changes/test/acceptance/coffee/helpers/TrackChangesApp.coffee

@@ -0,0 +1,24 @@
+app = require('../../../../app')
+require("logger-sharelatex").logger.level("info")
+logger = require("logger-sharelatex")
+Settings = require("settings-sharelatex")
+
+module.exports =
+	running: false
+	initing: false
+	callbacks: []
+	ensureRunning: (callback = (error) ->) ->
+		if @running
+			return callback()
+		else if @initing
+			@callbacks.push callback
+		else
+			@initing = true
+			@callbacks.push callback
+			app.listen Settings.internal?.trackchanges?.port, "localhost", (error) => 
+				throw error if error?
+				@running = true
+				logger.log("track changes running in dev mode")
+
+				for callback in @callbacks
+					callback()