Update build script to use build-target docker

services/filestore/Dockerfile

@@ -1,24 +1,21 @@
-FROM node:8.9.1 as app
-
-COPY ./ /app
+FROM node:6.9.5 as app
 
 WORKDIR /app
 
-RUN npm install
+COPY package.json package-lock.json /app/
+
+RUN npm install --quiet
+
+COPY . /app
 
 RUN npm run compile:all
 
-FROM node:8.9.1
+FROM node:6.9.5
 
 COPY --from=app /app /app
 
 WORKDIR /app
-
-# All app and node_modules will be owned by root.
-# The app will run as the 'node' user, and so not have write permissions
-# on any files it doesn't need.
 RUN ./install_deps.sh
-
 USER node
 
 CMD ["node","app.js"]

services/filestore/Jenkinsfile

@@ -1,79 +1,67 @@
-pipeline {
-  
-  agent {
-    docker {
-      image 'node:6.9.5'
-      args "-v /var/lib/jenkins/.npm:/tmp/.npm"
-    }
-  }
+String cron_string = BRANCH_NAME == "master" ? "@daily" : ""
 
-  environment  {
-      HOME = "/tmp"
-  }
+pipeline {
+  agent any
 
   triggers {
     pollSCM('* * * * *')
-    cron('@daily')
+    cron(cron_string)
   }
 
   stages {
-    stage('Set up') {
+    stage('Build') {
       steps {
-        // we need to disable logallrefupdates, else git clones during the npm install will require git to lookup the user id
-        // which does not exist in the container's /etc/passwd file, causing the clone to fail.
-        sh 'git config --global core.logallrefupdates false'
+        sh 'make build'
       }
     }
-    stage('Install') {
-      steps {
-        sh 'rm -fr node_modules'
-        sh 'npm install'
-        sh 'npm rebuild'
-        sh 'npm install --quiet grunt-cli'
-      }
-    }
-    stage('Compile') {
+
+    stage('Unit Tests') {
       steps {
-        sh 'node_modules/.bin/grunt compile'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_unit'
       }
     }
-    stage('Test') {
+
+    stage('Acceptance Tests') {
       steps {
-        sh 'NODE_ENV=development node_modules/.bin/grunt test:unit'
+        sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_acceptance'
       }
     }
-    stage('Package') {
+
+    stage('Package and publish build') {
       steps {
-        sh 'echo ${BUILD_NUMBER} > build_number.txt'
-        sh 'touch build.tar.gz' // Avoid tar warning about files changing during read
-        sh 'tar -czf build.tar.gz --exclude=build.tar.gz --exclude-vcs .'
+        sh 'make publish'
       }
     }
-    stage('Publish') {
+
+    stage('Publish build number') {
       steps {
+        sh 'echo ${BRANCH_NAME}-${BUILD_NUMBER} > build_number.txt'
         withAWS(credentials:'S3_CI_BUILDS_AWS_KEYS', region:"${S3_REGION_BUILD_ARTEFACTS}") {
-            s3Upload(file:'build.tar.gz', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/${BUILD_NUMBER}.tar.gz")
             // The deployment process uses this file to figure out the latest build
             s3Upload(file:'build_number.txt', bucket:"${S3_BUCKET_BUILD_ARTEFACTS}", path:"${JOB_NAME}/latest")
         }
       }
     }
   }
-  
+
   post {
+    always {
+      sh 'DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" make test_clean'
+    }
+
     failure {
-      mail(from: "${EMAIL_ALERT_FROM}", 
-           to: "${EMAIL_ALERT_TO}", 
+      mail(from: "${EMAIL_ALERT_FROM}",
+           to: "${EMAIL_ALERT_TO}",
            subject: "Jenkins build failed: ${JOB_NAME}:${BUILD_NUMBER}",
            body: "Build: ${BUILD_URL}")
     }
   }
-  
+
   // The options directive is for configuration that applies to the whole job.
   options {
     // we'd like to make sure remove old builds, so we don't fill up our storage!
     buildDiscarder(logRotator(numToKeepStr:'50'))
-    
+
     // And we'd really like to be sure that this build doesn't hang forever, so let's time it out after:
     timeout(time: 30, unit: 'MINUTES')
   }

services/filestore/Makefile

@@ -1,62 +1,44 @@
-NPM := docker-compose -f docker-compose.yml ${DOCKER_COMPOSE_FLAGS} run --rm npm npm
+# This file was auto-generated, do not edit it directly.
+# Instead run bin/update_build_scripts from
+# https://github.com/sharelatex/sharelatex-dev-environment
+# Version: 1.1.1
+
 BUILD_NUMBER ?= local
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 PROJECT_NAME = filestore
 DOCKER_COMPOSE_FLAGS ?= -f docker-compose.yml
-DOCKER_COMPOSE := docker-compose ${DOCKER_COMPOSE_FLAGS}
-
-all: install test
-	@echo "Run:"
-	@echo "	make install		to set up the project dependencies (in docker)"
-	@echo "	make test       to run all the tests for the project (in docker)"
-	@echo " make run        to run the app (natively)"
-
-install:
-	$(NPM) install
+DOCKER_COMPOSE := BUILD_NUMBER=$(BUILD_NUMBER) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	PROJECT_NAME=$(PROJECT_NAME) \
+	MOCHA_GREP=${MOCHA_GREP} \
+	docker-compose ${DOCKER_COMPOSE_FLAGS}
 
-run: 
-	$(NPM) run start
 
 clean:
 	rm -f app.js
 	rm -rf app/js
 	rm -rf test/unit/js
 	rm -rf test/acceptance/js
-	# Deletes node_modules volume
-	docker-compose down --volumes
 
-test: test_unit test_acceptance
+test:  test_unit test_acceptance
 
 test_unit:
-	$(DOCKER_COMPOSE) run --rm test_unit -- ${MOCHA_ARGS}
+	@[ ! -d test/unit ] && echo "filestore has no unit tests" || $(DOCKER_COMPOSE) run --rm test_unit npm run test:unit -- ${MOCHA_ARGS}
+
+test_acceptance: test_clean test_acceptance_pre_run # clear the database before each acceptance test run
+	@[ ! -d test/acceptance ] && echo "filestore has no acceptance tests" || $(DOCKER_COMPOSE) run --rm test_acceptance npm run test:acceptance -- ${MOCHA_ARGS}
 
-test_acceptance: ci_clean # clear the database before each acceptance test run
-	$(DOCKER_COMPOSE) run --rm test_acceptance -- ${MOCHA_ARGS}
+test_clean:
+	$(DOCKER_COMPOSE) down -v -t 0
 
+test_acceptance_pre_run:
+	@[ ! -f test/acceptance/scripts/pre-run ] && echo "filestore has no pre acceptance tests task" || $(DOCKER_COMPOSE) run --rm test_acceptance test/acceptance/scripts/pre-run
 build:
+	docker build --pull --tag quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
 	docker build --pull --tag gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) .
 
 publish:
+	docker push quay.io/sharelatex/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	docker push gcr.io/csh-gcdm-test/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 
-ci:
-	# On the CI server, we want to run our tests in the image that we
-	# have built for deployment, which is what the docker-compose.ci.yml
-	# override does.
-	PROJECT_NAME=$(PROJECT_NAME) \
-	BRANCH_NAME=$(BRANCH_NAME) \
-	BUILD_NUMBER=$(BUILD_NUMBER) \
-	DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" \
-	$(MAKE) build test publish
-
-ci_clean:
-	PROJECT_NAME=$(PROJECT_NAME) \
-	BRANCH_NAME=$(BRANCH_NAME) \
-	BUILD_NUMBER=$(BUILD_NUMBER) \
-	DOCKER_COMPOSE_FLAGS="-f docker-compose.ci.yml" \
-	$(DOCKER_COMPOSE) down
-
-.PHONY:
-	all install compile clean test test_unit test_acceptance \
-	test_acceptance_start_service test_acceptance_stop_service \
-	test_acceptance_run build publish ci ci_clean
+.PHONY: clean test test_unit test_acceptance test_clean build publish

services/filestore/package.json

@@ -8,15 +8,17 @@
   },
   "scripts": {
     "test:acceptance:run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
-    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:run -- $@",
+    "test:acceptance": "npm run compile:app && npm run compile:acceptance_tests && npm run test:acceptance:_run -- $@",
     "test:unit:run": "mocha --recursive --reporter spec $@ test/unit/js",
-    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:run -- $@",
-    "compile:unit_tests": "coffee -o test/unit/js -c test/unit/coffee",
-    "compile:acceptance_tests": "coffee -o test/acceptance/js -c test/acceptance/coffee",
-    "compile:app": "coffee -o app/js -c app/coffee && coffee -c app.coffee",
+    "test:unit": "npm run compile:app && npm run compile:unit_tests && npm run test:unit:_run -- $@",
+    "compile:unit_tests": "[ ! -e test/unit/coffee ] &&  echo 'No unit tests to compile' || coffee -o test/unit/js -c test/unit/coffee",
+    "compile:acceptance_tests": "[ ! -e test/acceptance/coffee ] && echo 'No acceptance tests to compile' || coffee -o test/acceptance/js -c test/acceptance/coffee",
+    "compile:app": "([ -e app/coffee ] && coffee $COFFEE_OPTIONS -o app/js -c app/coffee || echo 'No CoffeeScript folder to compile') && ( [ -e app.coffee ] && coffee $COFFEE_OPTIONS -c app.coffee || echo 'No CoffeeScript app to compile')",
     "compile:all": "npm run compile:app && npm run compile:unit_tests && npm run compile:acceptance_tests",
-    "start": "npm run compile:app && node app.js",
-    "nodemon": "nodemon --config nodemon.json"
+    "start": "npm run compile:app && node $NODE_APP_OPTIONS app.js",
+    "nodemon": "nodemon --config nodemon.json",
+    "test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+    "test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js"
   },
   "dependencies": {
     "async": "~0.2.10",