Merge pull request #47 from overleaf/jpa-build-scripts-1-3-5

[misc] update the build scripts to 1.3.5

services/spelling/.dockerignore

@@ -5,3 +5,4 @@ gitrev
 .npm
 .nvmrc
 nodemon.json
+cache/

services/spelling/.eslintrc

@@ -1,13 +1,16 @@
 // this file was auto-generated, do not edit it directly.
 // instead run bin/update_build_scripts from
 // https://github.com/sharelatex/sharelatex-dev-environment
-// Version: 1.1.21
+// Version: 1.3.5
 {
   "extends": [
     "standard",
     "prettier",
-    "prettier/standard",
+    "prettier/standard"
   ],
+  "parserOptions": {
+    "ecmaVersion": 2017
+  },
   "plugins": [
     "mocha",
     "chai-expect",
@@ -17,25 +20,46 @@
     "node": true,
     "mocha": true
   },
-  "globals": {
-    "expect": true,
-    "define": true
-  },
   "rules": {
-    // Add some mocha specific rules
-    "mocha/handle-done-callback": "error",
-    "mocha/no-exclusive-tests": "error",
-    "mocha/no-global-tests": "error",
-    "mocha/no-identical-title": "error",
-    "mocha/no-nested-tests": "error",
-    "mocha/no-pending-tests": "error",
-    "mocha/no-skipped-tests": "error",
-
-    // Add some chai specific rules
-    "chai-expect/missing-assertion": "error",
-    "chai-expect/terminating-properties": "error",
     // Swap the no-unused-expressions rule with a more chai-friendly one
     "no-unused-expressions": 0,
     "chai-friendly/no-unused-expressions": "error"
-  }
+  },
+  "overrides": [
+    {
+      // Test specific rules
+      "files": ["test/**/*.js"],
+      "globals": {
+        "expect": true
+      },
+      "rules": {
+        // mocha-specific rules
+        "mocha/handle-done-callback": "error",
+        "mocha/no-exclusive-tests": "error",
+        "mocha/no-global-tests": "error",
+        "mocha/no-identical-title": "error",
+        "mocha/no-nested-tests": "error",
+        "mocha/no-pending-tests": "error",
+        "mocha/no-skipped-tests": "error",
+        "mocha/no-mocha-arrows": "error",
+
+        // chai-specific rules
+        "chai-expect/missing-assertion": "error",
+        "chai-expect/terminating-properties": "error",
+
+        // prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
+        // we don't enforce this at the top-level - just in tests to manage `this` scope
+        // based on mocha's context mechanism
+        "mocha/prefer-arrow-callback": "error"
+      }
+    },
+    {
+      // Backend specific rules
+      "files": ["app/**/*.js", "app.js", "index.js"],
+      "rules": {
+        // don't allow console.log in backend code
+        "no-console": "error"
+      }
+    }
+  ]
 }

services/spelling/.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,4 @@
+
 <!-- Please review https://github.com/overleaf/write_latex/blob/master/.github/CONTRIBUTING.md for guidance on what is expected in each section. -->
 
 ### Description

services/spelling/.prettierrc

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.21
+# Version: 1.3.5
 {
   "semi": false,
   "singleQuote": true

services/spelling/Dockerfile

@@ -1,6 +1,15 @@
-FROM node:10.19.0 as app
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.3.5
+
+FROM node:10.19.0 as base
 
 WORKDIR /app
+COPY install_deps.sh /app
+RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
+
+FROM base as app
 
 #wildcard as some files may not be in all repos
 COPY package*.json npm-shrink*.json /app/
@@ -11,12 +20,11 @@ COPY . /app
 
 
 
-FROM node:10.19.0
+FROM base
 
 COPY --from=app /app /app
-
-WORKDIR /app
-RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
+RUN mkdir -p cache \
+&&  chown node:node cache
 USER node
 
 CMD ["node", "--expose-gc", "app.js"]

services/spelling/Jenkinsfile

@@ -16,6 +16,7 @@ pipeline {
   }
 
   stages {
+
     stage('Install') {
       steps {
         withCredentials([usernamePassword(credentialsId: 'GITHUB_INTEGRATION', usernameVariable: 'GH_AUTH_USERNAME', passwordVariable: 'GH_AUTH_PASSWORD')]) {

services/spelling/Makefile

@@ -1,7 +1,7 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.21
+# Version: 1.3.5
 
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
@@ -33,14 +33,20 @@ test_unit:
 
 test_acceptance: test_clean test_acceptance_pre_run test_acceptance_run
 
+test_acceptance_debug: test_clean test_acceptance_pre_run test_acceptance_run_debug
+
 test_acceptance_run:
 	@[ ! -d test/acceptance ] && echo "spelling has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance
 
+test_acceptance_run_debug:
+	@[ ! -d test/acceptance ] && echo "spelling has no acceptance tests" || $(DOCKER_COMPOSE) run -p 127.0.0.9:19999:19999 --rm test_acceptance npm run test:acceptance -- --inspect=0.0.0.0:19999 --inspect-brk
+
 test_clean:
 	$(DOCKER_COMPOSE) down -v -t 0
 
 test_acceptance_pre_run:
-	@[ ! -f test/acceptance/scripts/pre-run ] && echo "spelling has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
+	@[ ! -f test/acceptance/js/scripts/pre-run ] && echo "spelling has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
+
 build:
 	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
 		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
@@ -53,4 +59,5 @@ publish:
 
 	docker push $(DOCKER_REPO)/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
+
 .PHONY: clean test test_unit test_acceptance test_clean build publish

services/spelling/app/js/ASpell.js

@@ -32,7 +32,7 @@ try {
 }
 
 // write the cache every 30 minutes
-setInterval(function() {
+const cacheDump = setInterval(function() {
   const dump = JSON.stringify(cache.dump())
   return fs.writeFile(cacheFsPathTmp, dump, function(err) {
     if (err != null) {
@@ -186,3 +186,4 @@ ASpell.promises = promises
 module.exports = ASpell
 
 var WorkerPool = new ASpellWorkerPool()
+module.exports.cacheDump = cacheDump

services/spelling/buildscript.txt

@@ -1,8 +1,11 @@
 spelling
+--public-repo=False
+--language=es
+--env-add=
 --node-version=10.19.0
---script-version=1.1.21
---build-target=docker
+--acceptance-creds=None
 --dependencies=mongo,redis
---language=es
 --docker-repos=gcr.io/overleaf-ops
---acceptance-creds=None
+--env-pass-through=
+--data-dirs=cache
+--script-version=1.3.5

services/spelling/docker-compose.ci.yml

@@ -1,9 +1,9 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.21
+# Version: 1.3.5
 
-version: "2"
+version: "2.3"
 
 services:
   test_unit:
@@ -25,13 +25,14 @@ services:
       MOCHA_GREP: ${MOCHA_GREP}
       NODE_ENV: test
     depends_on:
-      - mongo
-      - redis
+      mongo:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
     user: node
     command: npm run test:acceptance:_run
 
 
-
   tar:
     build: .
     image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
@@ -39,9 +40,8 @@ services:
       - ./:/tmp/build/
     command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
     user: root
-
   redis:
     image: redis
 
   mongo:
-    image: mongo:3.4
+    image: mongo:3.6

services/spelling/docker-compose.yml

@@ -1,13 +1,15 @@
 # This file was auto-generated, do not edit it directly.
 # Instead run bin/update_build_scripts from
 # https://github.com/sharelatex/sharelatex-dev-environment
-# Version: 1.1.21
+# Version: 1.3.5
 
-version: "2"
+version: "2.3"
 
 services:
   test_unit:
-    build: .
+    build:
+      context: .
+      target: base
     volumes:
       - .:/app
     working_dir: /app
@@ -18,7 +20,9 @@ services:
     user: node
 
   test_acceptance:
-    build: .
+    build:
+      context: .
+      target: base
     volumes:
       - .:/app
     working_dir: /app
@@ -32,24 +36,15 @@ services:
       NODE_ENV: test
     user: node
     depends_on:
-      - mongo
-      - redis
+      mongo:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
     command: npm run test:acceptance
 
-
-
-  tar:
-    build: .
-    image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
-    volumes:
-      - ./:/tmp/build/
-    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
-    user: root
-
   redis:
     image: redis
 
   mongo:
-    image: mongo:3.4
-
+    image: mongo:3.6
 

services/spelling/install_deps.sh

@@ -13,6 +13,3 @@ apt-get update
 apt-get install -y aspell aspell-en aspell-af aspell-am aspell-ar aspell-ar-large aspell-bg aspell-bn aspell-br aspell-ca aspell-cs aspell-cy aspell-da aspell-de aspell-de-alt aspell-el aspell-eo aspell-es aspell-et aspell-eu-es aspell-fa aspell-fo aspell-fr aspell-ga aspell-gl-minimos aspell-gu aspell-he aspell-hi aspell-hr aspell-hsb aspell-hu aspell-hy aspell-id aspell-is aspell-it aspell-kk aspell-kn aspell-ku aspell-lt aspell-lv aspell-ml aspell-mr aspell-nl aspell-nr aspell-ns  aspell-pa aspell-pl aspell-pt aspell-pt-br aspell-ro aspell-ru aspell-sk aspell-sl aspell-ss aspell-st aspell-sv aspell-tl aspell-tn aspell-ts aspell-uk aspell-uz aspell-xh aspell-zu
 
 apt-get install aspell-or=0.03-1-6 aspell-te=0.01-2-6 aspell-no=2.2-4 aspell-ta=20040424-1-2
-mkdir /app/cache
-chown node:node /app/cache
-chmod 0777 /app/cache

services/spelling/package.json

@@ -11,7 +11,7 @@
     "start": "node $NODE_APP_OPTIONS app.js",
     "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
     "test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
-    "test:unit:_run": "mocha --recursive --reporter spec --exit $@ test/unit/js",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
     "test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
     "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
     "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",

services/spelling/test/acceptance/js/CheckTest.js

@@ -12,19 +12,19 @@ const checkWord = (words, language) =>
     })
   })
 
-describe('checking words', () => {
+describe('checking words', function() {
   let response
 
-  describe('on successful response', () => {
-    beforeEach(async () => {
+  describe('on successful response', function() {
+    beforeEach(async function () {
       response = await checkWord(['anather'])
     })
 
-    it('should return status 200', async () => {
+    it('should return status 200', async function () {
       expect(response.statusCode).to.equal(200)
     })
 
-    it('should return the list of misspellings', async () => {
+    it('should return the list of misspellings', async function () {
       const body = JSON.parse(response.body)
       expect(body).to.deep.equal({
         misspellings: [{ index: 0, suggestions: ['anther', 'another'] }]
@@ -32,23 +32,23 @@ describe('checking words', () => {
     })
   })
 
-  describe('when multiple words are submitted', () => {
-    beforeEach(async () => {
+  describe('when multiple words are submitted', function() {
+    beforeEach(async function () {
       response = await checkWord(['anather', 'anather', 'theorie'])
     })
 
-    it('should return the misspellings for all the words', async () => {
+    it('should return the misspellings for all the words', async function () {
       const body = JSON.parse(response.body)
       expect(body.misspellings.length).to.equal(3)
     })
 
-    it('should have misspelling suggestions with consecutive indexes', async () => {
+    it('should have misspelling suggestions with consecutive indexes', async function () {
       const body = JSON.parse(response.body)
       const indexes = body.misspellings.map(mspl => mspl.index)
       expect(indexes).to.deep.equal([0, 1, 2])
     })
 
-    it('should return identical suggestions for the same entry', async () => {
+    it('should return identical suggestions for the same entry', async function () {
       const body = JSON.parse(response.body)
       expect(body.misspellings[0].suggestions).to.deep.equal(
         body.misspellings[1].suggestions
@@ -56,8 +56,8 @@ describe('checking words', () => {
     })
   })
 
-  describe('when a very long list of words if submitted', () => {
-    beforeEach(async () => {
+  describe('when a very long list of words if submitted', function() {
+    beforeEach(async function () {
       let words = []
       for (let i = 0; i <= 20000; i++) {
         words.push('anather')
@@ -65,12 +65,12 @@ describe('checking words', () => {
       response = await checkWord(words)
     })
 
-    it('should return misspellings for the first 10K results only', async () => {
+    it('should return misspellings for the first 10K results only', async function () {
       const body = JSON.parse(response.body)
       expect(body.misspellings.length).to.equal(10000)
     })
 
-    it('should have misspelling suggestions with consecutive indexes', async () => {
+    it('should have misspelling suggestions with consecutive indexes', async function () {
       const body = JSON.parse(response.body)
       const indexList = body.misspellings.map(mspl => mspl.index)
       expect(indexList.length).to.equal(10000) // avoid testing over an incorrect array
@@ -80,8 +80,8 @@ describe('checking words', () => {
     })
   })
 
-  describe('when a very long list of words with utf8 responses', () => {
-    beforeEach(async () => {
+  describe('when a very long list of words with utf8 responses', function() {
+    beforeEach(async function () {
       let words = []
       for (let i = 0; i <= 20000; i++) {
         words.push('anéther')
@@ -89,12 +89,12 @@ describe('checking words', () => {
       response = await checkWord(words, 'bg') // use Bulgarian to generate utf8 response
     })
 
-    it('should return misspellings for the first 10K results only', async () => {
+    it('should return misspellings for the first 10K results only', async function () {
       const body = JSON.parse(response.body)
       expect(body.misspellings.length).to.equal(10000)
     })
 
-    it('should have misspelling suggestions with consecutive indexes', async () => {
+    it('should have misspelling suggestions with consecutive indexes', async function () {
       const body = JSON.parse(response.body)
       const indexList = body.misspellings.map(mspl => mspl.index)
       expect(indexList.length).to.equal(10000) // avoid testing over an incorrect array
@@ -104,23 +104,23 @@ describe('checking words', () => {
     })
   })
 
-  describe('when multiple words with utf8 are submitted', () => {
-    beforeEach(async () => {
+  describe('when multiple words with utf8 are submitted', function() {
+    beforeEach(async function () {
       response = await checkWord(['mneá', 'meniésn', 'meônoi', 'mneá'], 'pt_BR')
     })
 
-    it('should return the misspellings for all the words', async () => {
+    it('should return the misspellings for all the words', async function () {
       const body = JSON.parse(response.body)
       expect(body.misspellings.length).to.equal(4)
     })
 
-    it('should have misspelling suggestions with consecutive indexes', async () => {
+    it('should have misspelling suggestions with consecutive indexes', async function () {
       const body = JSON.parse(response.body)
       const indexes = body.misspellings.map(mspl => mspl.index)
       expect(indexes).to.deep.equal([0, 1, 2, 3])
     })
 
-    it('should return identical suggestions for the same entry', async () => {
+    it('should return identical suggestions for the same entry', async function () {
       const body = JSON.parse(response.body)
       expect(body.misspellings[0].suggestions).to.deep.equal(
         body.misspellings[3].suggestions

services/spelling/test/acceptance/js/HealthCheckTest.js

@@ -1,8 +1,8 @@
 const { expect } = require('chai')
 const request = require('./helpers/request')
 
-describe('/health_check', () => {
-  it('should return 200', async () => {
+describe('/health_check', function() {
+  it('should return 200', async function () {
     const response = await request.get('/health_check')
     expect(response.statusCode).to.equal(200)
   })

services/spelling/test/acceptance/js/Init.js

@@ -1,4 +1,4 @@
 const App = require('../../../app.js')
 const { PORT } = require('./helpers/request')
 
-before(done => App.listen(PORT, 'localhost', done))
+before(function(done) { return App.listen(PORT, 'localhost', done); })

services/spelling/test/acceptance/js/LearnTest.js

@@ -32,13 +32,13 @@ const deleteDict = () =>
     url: `/user/${USER_ID}`
   })
 
-describe('learning words', () => {
-  it('should return status 204 when posting a word successfully', async () => {
+describe('learning words', function() {
+  it('should return status 204 when posting a word successfully', async function () {
     const response = await learnWord('abcd')
     expect(response.statusCode).to.equal(204)
   })
 
-  it('should return no misspellings after a word is learnt', async () => {
+  it('should return no misspellings after a word is learnt', async function () {
     const response = await checkWord(['abv'])
     const responseBody = JSON.parse(response.body)
     expect(responseBody.misspellings.length).to.equals(1)
@@ -50,7 +50,7 @@ describe('learning words', () => {
     expect(responseBody2.misspellings.length).to.equals(0)
   })
 
-  it('should return misspellings again after a personal dictionary is deleted', async () => {
+  it('should return misspellings again after a personal dictionary is deleted', async function () {
     await learnWord('bvc')
     await deleteDict()
 
@@ -60,13 +60,13 @@ describe('learning words', () => {
   })
 })
 
-describe('unlearning words', () => {
-  it('should return status 204 when posting a word successfully', async () => {
+describe('unlearning words', function() {
+  it('should return status 204 when posting a word successfully', async function () {
     const response = await unlearnWord('anything')
     expect(response.statusCode).to.equal(204)
   })
 
-  it('should return misspellings after a word is unlearnt', async () => {
+  it('should return misspellings after a word is unlearnt', async function () {
     await learnWord('abv')
 
     const response = await checkWord(['abv'])

services/spelling/test/acceptance/js/StatusTest.js

@@ -1,8 +1,8 @@
 const { expect } = require('chai')
 const request = require('./helpers/request')
 
-describe('/status', () => {
-  it('should return 200', async () => {
+describe('/status', function() {
+  it('should return 200', async function () {
     const response = await request.get('/health_check')
     expect(response.statusCode).to.equal(200)
   })

services/spelling/test/unit/js/ASpellTests.js

@@ -30,6 +30,9 @@ describe('ASpell', function() {
       }
     }))
   })
+  afterEach(function () {
+    clearInterval(this.ASpell.cacheDump)
+  })
 
   describe('a correctly spelled word', function() {
     beforeEach(function(done) {