Merge pull request #18 from sharelatex/ho-docker

Move to dual Linode / GKE deploy

services/spelling/.dockerignore

@@ -0,0 +1,9 @@
+node_modules/*
+gitrev
+.git
+.gitignore
+.npm
+.nvmrc
+nodemon.json
+app.js
+**/js/*

services/spelling/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,38 @@
+<!-- BUG REPORT TEMPLATE -->
+
+## Steps to Reproduce
+<!-- Describe the steps leading up to when / where you found the bug. -->
+<!-- Screenshots may be helpful here. -->
+
+1.
+2.
+3.
+
+## Expected Behaviour
+<!-- What should have happened when you completed the steps above? -->
+
+## Observed Behaviour
+<!-- What actually happened when you completed the steps above? -->
+<!-- Screenshots may be helpful here. -->
+
+## Context
+<!-- How has this issue affected you? What were you trying to accomplish? -->
+
+## Technical Info
+<!-- Provide any technical details that may be applicable (or N/A if not applicable). -->
+
+* URL:
+* Browser Name and version:
+* Operating System and version (desktop or mobile):
+* Signed in as:
+* Project and/or file:
+
+## Analysis
+<!--- Optionally, document investigation of / suggest a fix for the bug, e.g. 'comes from this line / commit' -->
+
+## Who Needs to Know?
+<!-- If you want to bring this to the attention of particular people, @-mention them below. -->
+<!-- If a user reported this bug and should be notified when it is fixed, provide the Front conversation link. -->
+
+-
+-

services/spelling/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,45 @@
+<!-- Please review https://github.com/overleaf/write_latex/blob/master/.github/CONTRIBUTING.md for guidance on what is expected in each section. -->
+
+### Description
+
+
+
+#### Screenshots
+
+
+
+#### Related Issues / PRs
+
+
+
+### Review
+
+
+
+#### Potential Impact
+
+
+
+#### Manual Testing Performed
+
+- [ ]
+- [ ]
+
+#### Accessibility
+
+
+
+### Deployment
+
+
+
+#### Deployment Checklist
+
+- [ ] Update documentation not included in the PR (if any)
+- [ ]
+
+#### Metrics and Monitoring
+
+
+
+#### Who Needs to Know?

services/spelling/.gitignore

@@ -5,4 +5,5 @@ app.js
 test/UnitTests/js/*
 node_modules/*
 test/unit/js/
-cache/spell.cache
+cache/spell.cache
+**/*.map

services/spelling/Dockerfile

@@ -1,3 +1,23 @@
+FROM node:6.9.5 as app
+
+WORKDIR /app
+
+#wildcard as some files may not be in all repos
+COPY package*.json npm-shrink*.json /app/
+
+RUN npm install --quiet
+
+COPY . /app
+
+
+RUN npm run compile:all
+
 FROM node:6.9.5
 
-RUN apt-get update && apt-get install -y aspell aspell-en aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-alt aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-or aspell-pa aspell-pl aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-sv aspell-ta aspell-te aspell-tl aspell-uk aspell-uz
+COPY --from=app /app /app
+
+WORKDIR /app
+RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
+USER node
+
+CMD ["node", "--expose-gc", "app.js"]

services/spelling/Gruntfile.coffee

@@ -1,54 +0,0 @@
-module.exports = (grunt) ->
-	grunt.initConfig
-		coffee:
-			app_src:
-				expand: true,
-				cwd: "app/coffee"
-				src: ['**/*.coffee'],
-				dest: 'app/js/',
-				ext: '.js'
-
-			app:
-				src: "app.coffee"
-				dest: "app.js"
-
-			unit_tests:
-				expand: true
-				cwd:  "test/unit/coffee"
-				src: ["**/*.coffee"]
-				dest: "test/unit/js/"
-				ext:  ".js"
-
-		clean:
-			app: ["app/js/"]
-			unit_tests: ["test/unit/js"]
-
-		execute:
-			app:
-				src: "app.js"
-
-		mochaTest:
-			unit:
-				options:
-					reporter: grunt.option('reporter') or 'spec'
-					grep: grunt.option("grep")
-					timeout: grunt.option("timeout")
-				src: ["test/unit/js/**/*.js"]
-
-	grunt.loadNpmTasks 'grunt-contrib-coffee'
-	grunt.loadNpmTasks 'grunt-contrib-clean'
-	grunt.loadNpmTasks 'grunt-mocha-test'
-	grunt.loadNpmTasks 'grunt-execute'
-	grunt.loadNpmTasks 'grunt-bunyan'
-
-	grunt.registerTask 'compile:app', ['clean:app', 'coffee:app', 'coffee:app_src']
-	grunt.registerTask 'run',         ['compile:app', 'bunyan', 'execute']
-
-	grunt.registerTask 'compile:unit_tests', ['clean:unit_tests', 'coffee:unit_tests']
-	grunt.registerTask 'test:unit',          ['compile:app', 'compile:unit_tests', 'mochaTest:unit']
-
-	grunt.registerTask 'install', 'compile:app'
-
-	grunt.registerTask 'default', ['run']
-
-

services/spelling/Jenkinsfile

@@ -1,79 +1,122 @@
+String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
+
 pipeline {
-  
-  agent {
-    docker {
-      image 'sharelatex/node-aspell:6.9.5'
-      args "-v /var/lib/jenkins/.npm:/tmp/.npm"
-    }
-  }
+  agent any
 
-  environment  {
-      HOME = "/tmp"
+  environment {
+    GIT_PROJECT = "spelling-sharelatex"
+    JENKINS_WORKFLOW = "spelling-sharelatex"
+    TARGET_URL = "${env.JENKINS_URL}blue/organizations/jenkins/${JENKINS_WORKFLOW}/detail/$BRANCH_NAME/$BUILD_NUMBER/pipeline"
+    GIT_API_URL = "https://api.github.com/repos/sharelatex/${GIT_PROJECT}/statuses/$GIT_COMMIT"
   }
 
   triggers {
     pollSCM('* * * * *')
-    cron('@daily')
+    cron(cron_string)
   }
 
   stages {
-    stage('Set up') {
+    stage('Install') {
       steps {
-        // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
-        // which does not exist in the container's /etc/passwd file, causing the clone to fail.
-        sh 'git config --global core.logallrefupdates false'
+        withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+          sh "curl $GIT_API_URL \
+            --data '{ \
+            \"state\" : \"pending\", \
+            \"target_url\": \"$TARGET_URL\", \
+            \"description\": \"Your build is underway\", \
+            \"context\": \"ci/jenkins\" }' \
+            -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+        }
       }
     }
-    stage('Install') {
+
+    stage('Build') {
       steps {
-        sh 'rm -fr node_modules'
-        sh 'npm install'
-        sh 'npm rebuild'
-        sh 'npm install --quiet grunt-cli'
+        sh 'make build'
       }
     }
-    stage('Compile') {
+
+    stage('Unit Tests') {
       steps {
-        sh 'node_modules/.bin/grunt install'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
       }
     }
-    stage('Test') {
+
+    stage('Acceptance Tests') {
       steps {
-        sh 'node_modules/.bin/grunt test:unit --timeout 10000'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
       }
     }
-    stage('Package') {
+
+    stage('Package and docker push') {
       steps {
         sh 'echo ${BUILD_NUMBER} > build_number.txt'
         sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
-        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make tar'
+        
+        withCredentials([file(credentialsId: 'gcr.io_overleaf-ops', variable: 'DOCKER_REPO_KEY_PATH')]) {
+          sh 'docker login -u _json_key --password-stdin https://gcr.io/overleaf-ops < ${DOCKER_REPO_KEY_PATH}'
+        }
+        sh 'DOCKER_REPO=gcr.io/overleaf-ops make publish'
+        sh 'docker logout https://gcr.io/overleaf-ops'
+        
       }
     }
-    stage('Publish') {
+
+    stage('Publish to s3') {
       steps {
+        sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
         withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
             s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
+        }
+        withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
             // The deployment process uses this file to figure out the latest build
             s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
         }
       }
     }
   }
-  
+
   post {
+    always {
+      sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
+      sh 'make clean'
+    }
+
+    success {
+      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+        sh "curl $GIT_API_URL \
+          --data '{ \
+          \"state\" : \"success\", \
+          \"target_url\": \"$TARGET_URL\", \
+          \"description\": \"Your build succeeded!\", \
+          \"context\": \"ci/jenkins\" }' \
+          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+      }
+    }
+
     failure {
-      mail(from: "${EMAIL_ALERT_FROM}", 
-           to: "${EMAIL_ALERT_TO}", 
+      mail(from: "${EMAIL_ALERT_FROM}",
+           to: "${EMAIL_ALERT_TO}",
            subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
            body: "Build: ${BUILD_URL}")
+      withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {
+        sh "curl $GIT_API_URL \
+          --data '{ \
+          \"state\" : \"failure\", \
+          \"target_url\": \"$TARGET_URL\", \
+          \"description\": \"Your build failed\", \
+          \"context\": \"ci/jenkins\" }' \
+          -u $GH_AUTH_USERNAME:$GH_AUTH_PASSWORD"
+      }
     }
   }
-  
+
   // The options directive is for configuration that applies to the whole job.
   options {
     // we'd like to make sure remove old builds, so we don't fill up our storage!
     buildDiscarder(logRotator(numToKeepStr:'50'))
-    
+
     // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
     timeout(time: 30, unit: 'MINUTES')
   }

services/spelling/Makefile

@@ -0,0 +1,49 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.12
+
+BUILD_NUMBER ?= local
+BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
+PROJECT_NAME = spelling
+DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
+DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	docker-compose ${DOCKER_COMPOSE_FLAGS}
+
+clean:
+	docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+	rm -f app.js
+	rm -rf app/js
+	rm -rf test/unit/js
+	rm -rf test/acceptance/js
+
+test: test_unit test_acceptance
+
+test_unit:
+	@[ ! -d test/unit ] && echo "spelling has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit
+
+test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run
+	@[ ! -d test/acceptance ] && echo "spelling has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
+
+test_clean:
+	$(DOCKER_COMPOSE) down -v -t 0
+
+test_acceptance_pre_run:
+	@[ ! -f test/acceptance/scripts/pre-run ] && echo "spelling has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
+build:
+	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		.
+
+tar:
+	$(DOCKER_COMPOSE) up tar
+
+publish:
+
+	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+
+.PHONY: clean test test_unit test_acceptance test_clean build publish

services/spelling/app.coffee

@@ -1,17 +1,20 @@
+metrics = require("metrics-sharelatex")
+metrics.initialize("spelling")
+
+
 Settings = require 'settings-sharelatex'
 logger = require 'logger-sharelatex'
-logger.initialize("spelling-sharelatex")
+logger.initialize("spelling")
 if Settings.sentry?.dsn?
 	logger.initializeErrorReporting(Settings.sentry.dsn)
+metrics.memory.monitor(logger)
 
 SpellingAPIController = require './app/js/SpellingAPIController'
 express = require('express')
 Path = require("path")
 server = express()
+metrics.injectMetricsRoute(server)
 bodyParser = require('body-parser')
-metrics = require("metrics-sharelatex")
-metrics.initialize("spelling")
-metrics.memory.monitor(logger)
 HealthCheckController = require("./app/js/HealthCheckController")
 
 
@@ -20,6 +23,7 @@ server.use bodyParser.json(limit: "2mb")
 server.use metrics.http.monitor(logger)
 
 server.del "/user/:user_id", SpellingAPIController.deleteDic
+server.get "/user/:user_id", SpellingAPIController.getDic
 server.post "/user/:user_id/check", SpellingAPIController.check
 server.post "/user/:user_id/learn", SpellingAPIController.learn
 server.get "/status", (req, res)->

services/spelling/app/coffee/ASpell.coffee

@@ -6,8 +6,9 @@ fs = require 'fs'
 settings = require("settings-sharelatex")
 Path = require("path")
 
-cache = LRU(10000)
 OneMinute = 60 * 1000
+opts = {max:10000, maxAge: OneMinute * 60 * 10}
+cache = LRU(opts)
 
 cacheFsPath = Path.resolve(settings.cacheDir, "spell.cache")
 cacheFsPathTmp = cacheFsPath + ".tmp"
@@ -63,7 +64,7 @@ class ASpellRunner
 			for k, v of addToCache
 				cache.set(k, v)
 
-			logger.log hits: hits, total: words.length, hitrate: (hits/words.length).toFixed(2), "cache hit rate"
+			logger.info hits: hits, total: words.length, hitrate: (hits/words.length).toFixed(2), "cache hit rate"
 			callback null, results
 
 	getSuggestions: (language, output) ->

services/spelling/app/coffee/ASpellWorker.coffee

@@ -10,12 +10,12 @@ class ASpellWorker
 		@language = language
 		@count = 0
 		@pipe = child_process.spawn("aspell", ["pipe", "-t", "--encoding=utf-8", "-d", language])
-		logger.log process: @pipe.pid, lang: @language, "starting new aspell worker"
-		metrics.inc "aspellWorker-start-" + @language
+		logger.info process: @pipe.pid, lang: @language, "starting new aspell worker"
+		metrics.inc "aspellWorker", 1, {status: "start", method: @language}
 		@pipe.on 'exit', () =>
 			@state = 'killed'
-			logger.log process: @pipe.pid, lang: @language, "aspell worker has exited"
-			metrics.inc "aspellWorker-exit-" + @language
+			logger.info process: @pipe.pid, lang: @language, "aspell worker has exited"
+			metrics.inc "aspellWorker" , 1, {status: "exit", method: @language}
 		@pipe.on 'close', () =>
 			@state = 'closed' unless @state == 'killed'
 			if @callback?
@@ -30,7 +30,7 @@ class ASpellWorker
 				@callback = null
 		@pipe.stdin.on 'error', (err) =>
 			@state = 'error' unless @state == 'killed'
-			logger.log process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error on stdin"
+			logger.info process: @pipe.pid, error: err, stdout: output.slice(-1024), stderr: error.slice(-1024), lang: @language, "aspell worker error on stdin"
 			if @callback?
 				@callback err, []
 				@callback = null
@@ -89,12 +89,12 @@ class ASpellWorker
 		@sendCommand("$$l")
 
 	shutdown: (reason) ->
-		logger.log process: @pipe.pid, reason: reason, 'shutting down'
+		logger.info process: @pipe.pid, reason: reason, 'shutting down'
 		@state = "closing"
 		@pipe.stdin.end()
 
 	kill: (reason) ->
-		logger.log process: @pipe.pid, reason: reason, 'killing'
+		logger.info process: @pipe.pid, reason: reason, 'killing'
 		return if @state == 'killed'
 		@pipe.kill('SIGKILL')
 

services/spelling/app/coffee/ASpellWorkerPool.coffee

@@ -24,7 +24,7 @@ class ASpellWorkerPool
 			if worker.idleTimer?
 				clearTimeout worker.idleTimer
 				worker.idleTimer = null
-			logger.log process: worker.pipe.pid, lang: language, "removing aspell worker from pool"
+			logger.info process: worker.pipe.pid, lang: language, "removing aspell worker from pool"
 			@cleanup()
 		@PROCESS_POOL.push(worker)
 		metrics.gauge 'aspellWorkerPool-size', @PROCESS_POOL.length

services/spelling/app/coffee/LearnedWordsManager.coffee

@@ -17,10 +17,10 @@ module.exports = LearnedWordsManager =
 	getLearnedWords: (user_token, callback = (error, words)->) ->
 		mongoCachedWords = mongoCache.get(user_token)
 		if mongoCachedWords?
-			metrics.inc "mongoCache-hit", 0.1
+			metrics.inc "mongoCache", 0.1, {status: "hit"}
 			return callback(null, mongoCachedWords)
 
-		metrics.inc "mongoCache-miss", 0.1
+		metrics.inc "mongoCache", 0.1, {status: "miss"}
 		logger.info user_token:user_token, "mongoCache miss"
 		
 		db.spellingPreferences.findOne token: user_token, (error, preferences) ->

services/spelling/app/coffee/MongoCache.coffee

@@ -1,7 +1,7 @@
 LRU = require("lru-cache")
 cacheOpts = 
-	max: 5000
-	maxAge: 1000 * 60 * 60
+	max: 15000
+	maxAge: 1000 * 60 * 60 * 10
 
 cache = LRU(cacheOpts)
 

services/spelling/app/coffee/SpellingAPIController.coffee

@@ -5,7 +5,7 @@ metrics = require('metrics-sharelatex')
 module.exports = SpellingAPIController =
 	check: (req, res, next) ->
 		metrics.inc "spelling-check", 0.1
-		logger.log token: req?.params?.user_id, word_count: req?.body?.words?.length, "running check"
+		logger.info token: req?.params?.user_id, word_count: req?.body?.words?.length, "running check"
 		SpellingAPIManager.runRequest req.params.user_id, req.body, (error, result) ->
 			if error?
 				logger.err err:error, user_id:req?.params?.user_id, word_count: req?.body?.words?.length, "error processing spelling request"
@@ -14,7 +14,7 @@ module.exports = SpellingAPIController =
 
 	learn: (req, res, next) ->
 		metrics.inc "spelling-learn", 0.1
-		logger.log token: req?.params?.user_id, word: req?.body?.word, "learning word"
+		logger.info token: req?.params?.user_id, word: req?.body?.word, "learning word"
 		SpellingAPIManager.learnWord req.params.user_id, req.body, (error, result) ->
 			return next(error) if error?
 			res.sendStatus(200)
@@ -27,3 +27,9 @@ module.exports = SpellingAPIController =
 			return next(error) if error?
 			res.sendStatus(204)
 
+
+	getDic: (req, res, next)->
+		logger.info token: req?.params?.user_id, "getting user dictionary"
+		SpellingAPIManager.getDic req.params.user_id, (error, words)->
+			return next(error) if error?
+			res.send(words)

services/spelling/app/coffee/SpellingAPIManager.coffee

@@ -49,3 +49,6 @@ module.exports = SpellingAPIManager =
 	deleteDic: (token, callback)->
 		LearnedWordsManager.deleteUsersLearnedWords token, callback
 
+	getDic: (token, callback)->
+		LearnedWordsManager.getLearnedWords token, callback
+

services/spelling/buildscript.txt

@@ -0,0 +1,8 @@
+spelling
+--language=coffeescript
+--node-version=6.9.5
+--acceptance-creds=None
+--dependencies=mongo,redis
+--docker-repos=gcr.io/overleaf-ops
+--build-target=docker
+--script-version=1.1.12

services/spelling/config/settings.defaults.coffee

@@ -5,14 +5,9 @@ module.exports = Settings =
 		spelling:
 			port: 3005
 			host: process.env["LISTEN_ADDRESS"] or "localhost"
-			
-	redis:
-		port: 6379
-		host: process.env["REDIS_HOST"] or "localhost"
-		password:""
 		
 	mongo:
-		url : "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
+		url: process.env['MONGO_CONNECTION_STRING'] or "mongodb://#{process.env["MONGO_HOST"] or "localhost"}/sharelatex"
 
 	cacheDir: Path.resolve "cache"
 

services/spelling/docker-compose.ci.yml

@@ -0,0 +1,44 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.12
+
+version: "2"
+
+services:
+  test_unit:
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    user: node
+    command: npm run test:unit:_run
+
+
+  test_acceptance:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    environment:
+      ELASTIC_SEARCH_DSN: es:9200
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      MOCHA_GREP: ${MOCHA_GREP}
+    depends_on:
+      - mongo
+      - redis
+    user: node
+    command: npm run test:acceptance:_run
+
+
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4

services/spelling/docker-compose.yml

@@ -0,0 +1,51 @@
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.12
+
+version: "2"
+
+services:
+  test_unit:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      MOCHA_GREP: ${MOCHA_GREP}
+    command: npm run test:unit
+    user: node
+
+  test_acceptance:
+    build: .
+    volumes:
+      - .:/app
+    working_dir: /app
+    environment:
+      ELASTIC_SEARCH_DSN: es:9200
+      REDIS_HOST: redis
+      MONGO_HOST: mongo
+      POSTGRES_HOST: postgres
+      MOCHA_GREP: ${MOCHA_GREP}
+    user: node
+    depends_on:
+      - mongo
+      - redis
+    command: npm run test:acceptance
+
+
+
+  tar:
+    build: .
+    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
+    volumes:
+      - ./:/tmp/build/
+    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    user: root
+
+  redis:
+    image: redis
+
+  mongo:
+    image: mongo:3.4
+

services/spelling/install_deps.sh

@@ -0,0 +1,8 @@
+# some spelling packages are missing from debian and are on ubuntulo
+echo "deb http://us.archive.ubuntu.com/ubuntu/ bionic main universe" >> /etc/apt/sources.list
+apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32
+apt-get update
+apt-get install -y aspell aspell-en aspell-af aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-alt aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-id aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-no aspell-nr aspell-ns aspell-or aspell-pa aspell-pl aspell-pt aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sk aspell-ss aspell-st aspell-sv aspell-ta aspell-te aspell-tl aspell-tn aspell-ts aspell-uk aspell-uz aspell-xh aspell-zu
+mkdir /app/cache
+chown node:node /app/cache
+chmod 0777 /app/cache

services/spelling/nodemon.json

@@ -0,0 +1,19 @@
+{
+  "ignore": [
+    ".git",
+    "node_modules/"
+  ],
+  "verbose": true,
+  "legacyWatch": true,
+  "execMap": {
+    "js": "npm run start"
+  },
+
+  "watch": [
+    "app/coffee/",
+    "app.coffee",
+    "config/"
+  ],
+  "ext": "coffee"
+
+}

services/spelling/package.json

@@ -7,8 +7,17 @@
     "url": "https://github.com/sharelatex/spelling-sharelatex.git"
   },
   "scripts": {
-    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
-    "start": "npm run compile:app && node app.js"
+    "compile:app": "([ -e app/coffee ] && coffee -m $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee -m $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
+    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 30000 --exit $@ test/acceptance/js",
+    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
+    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
+    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- --grep=$MOCHA_GREP",
+    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
+    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
+    "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests && npm run compile:smoke_tests",
+    "nodemon": "nodemon --config nodemon.json",
+    "compile:smoke_tests": "[ ! -e test/smoke/coffee ] &&  echo 'No smoke tests to compile' || coffee -o test/smoke/js -c test/smoke/coffee"
   },
   "version": "0.1.4",
   "dependencies": {
@@ -16,26 +25,21 @@
     "body-parser": "^1.12.0",
     "coffee-script": "^1.9.1",
     "express": "^4.12.1",
-    "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.1.0",
+    "logger-sharelatex": "^1.6.0",
     "lru-cache": "^4.0.0",
-    "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.7.1",
+    "metrics-sharelatex": "^2.1.1",
     "mongojs": "2.4.0",
     "node-statsd": "0.0.3",
     "redis": "~0.8.4",
     "request": "^2.53.0",
-    "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0",
+    "settings-sharelatex": "^1.1.0",
     "underscore": "1.4.4",
     "v8-profiler": "^5.2.4"
   },
   "devDependencies": {
     "bunyan": "^1.0.0",
     "chai": "",
-    "grunt": "^0.4.5",
-    "grunt-bunyan": "^0.5.0",
-    "grunt-contrib-clean": "^0.6.0",
-    "grunt-contrib-coffee": "^0.11.0",
-    "grunt-execute": "^0.2.2",
-    "grunt-mocha-test": "^0.11.0",
+    "mocha": "^4.1.0",
     "sandboxed-module": "",
     "sinon": "^1.17.0"
   }

services/spelling/test/unit/coffee/ASpellTests.coffee

@@ -9,6 +9,7 @@ describe "ASpell", ->
 		@ASpell = SandboxedModule.require "../../../app/js/ASpell", requires:
 			"logger-sharelatex":
 				log:->
+				info:->
 				err:->
 			'metrics-sharelatex':
 				gauge:->