Merge pull request #2845 from overleaf/jpa-faster-web-ci

[misc] faster web ci

GitOrigin-RevId: bbd6b12040956f6b397c6ea4d8144e733299400b

services/web/Dockerfile

@@ -5,11 +5,17 @@ WORKDIR /app
 # install_deps changes app files and installs npm packages
 # as such it has to run at a later stage
 
-FROM base as app
+RUN apt-get update \
+&&  apt-get install -y parallel \
+&&  rm -rf /var/lib/apt/lists/*
+
+FROM base as deps
 
 COPY package.json package-lock.json /app/
 
-RUN npm install --quiet
+RUN npm ci --quiet
+
+FROM deps as app
 
 COPY . /app
 

services/web/Makefile

@@ -48,6 +48,13 @@ test_unit_app:
 	COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) run --name unit_test_$(BUILD_DIR_NAME) --rm test_unit
 	COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down -v -t 0
 
+test_unit_app_parallel: export COMPOSE_PROJECT_NAME = \
+	unit_test_parallel_$(BUILD_DIR_NAME)
+test_unit_app_parallel:
+	$(DOCKER_COMPOSE) down -v -t 0
+	$(DOCKER_COMPOSE) run --rm test_unit npm run test:unit:app:parallel
+	$(DOCKER_COMPOSE) down -v -t 0
+
 TEST_UNIT_MODULES = $(MODULE_DIRS:=/test_unit)
 $(TEST_UNIT_MODULES): %/test_unit: %/Makefile
 test_unit_modules: $(TEST_UNIT_MODULES)
@@ -102,23 +109,46 @@ ci:
 #
 # Lint & format
 #
+ORG_PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+RUN_LINT_FORMAT ?= \
+	docker run --rm \
+		--volume $(PWD):/src \
+		--workdir /src \
+		--env NODE_PATH=/app/node_modules \
+		--env PATH=$(ORG_PATH):/app/node_modules/.bin \
+		gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps
 
 format:
 	npm -q run format
 
+format_in_docker:
+	$(RUN_LINT_FORMAT) make format
+
 format_fix:
 	npm -q run format:fix
 
 lint:
 	npm -q run lint
 
+lint_in_docker:
+	$(RUN_LINT_FORMAT) make lint
+
 #
 # Build & publish
 #
 
-build:
+build_deps:
+	docker build --pull \
+		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps \
+		--cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps \
+		--cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):master-deps \
+		--target deps \
+		.
+
+build: build_deps
 	docker build --pull --tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
 		--tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+		--cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-deps \
 		--build-arg SENTRY_RELEASE=${COMMIT_SHA} \
 		--build-arg BRANCH_NAME=$(BRANCH_NAME) \
 		.

services/web/Makefile.module

@@ -21,10 +21,16 @@ DOCKER_COMPOSE_TEST_UNIT := \
 	export COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME)_$(MODULE_NAME) \
 	&& $(DOCKER_COMPOSE)
 
+ifeq (,$(wildcard test/unit))
+test_unit:
+
+else
 test_unit:
 	${DOCKER_COMPOSE_TEST_UNIT} run --rm test_unit npm -q run test:unit:run_dir -- ${MOCHA_ARGS} $(MODULE_DIR)/test/unit/src
 	${DOCKER_COMPOSE_TEST_UNIT} down
 
+endif
+
 ifeq (,$(wildcard test/acceptance))
 test_acceptance:
 clean_test_acceptance:

services/web/bin/cdn_upload

@@ -3,9 +3,9 @@ set -e
 
 # Upload to staging CDN if branch is either 'master' or 'staging-master'
 if [[ "$BRANCH_NAME" == "master" || "$BRANCH_NAME" == "staging-master" ]]; then
-  tar --directory=/tmp/ -xzf build.tar.gz ./public/
+  tar --directory=/tmp/ -xf build.tar
   gsutil -h "Cache-Control:public, max-age=31536000" -m cp -r /tmp/public $CDN_STAG
-  # Only upload to production CDN if branch is 
+  # Only upload to production CDN if branch is
   if [[ "$BRANCH_NAME" == "master" ]]; then
     gsutil -h "Cache-Control:public, max-age=31536000" -m cp -r /tmp/public $CDN_PROD
   fi

services/web/docker-compose.ci.yml

@@ -48,7 +48,7 @@ services:
     image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
     volumes:
       - ./:/tmp/build/
-    command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
+    command: tar -cf /tmp/build/build.tar public/
     user: root
 
   redis:

services/web/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "2"
+version: "2.3"
 
 volumes:
   data:
@@ -6,7 +6,9 @@ volumes:
 services:
 
   test_unit:
-    image: node:10.19.0
+    build:
+      context: .
+      target: base
     volumes:
       - .:/app
     working_dir: /app

services/web/install_deps.sh

@@ -1,15 +1,11 @@
 #!/bin/bash
 
-make --no-print-directory format & FORMAT=$!
-make --no-print-directory lint & LINT=$!
 npm install git+https://github.com/sharelatex/translations-sharelatex.git#master & TRANSLATIONS=$!
 
 npm run webpack:production & WEBPACK=$!
 
-echo "Waiting for lint, format, translations and minify to finish"
+echo "Waiting for translations and minify to finish"
 
-wait $LINT && echo "Lint complete" || exit 1
-wait $FORMAT && echo "Format complete" || exit 1
 wait $TRANSLATIONS && echo "Translations install complete" || exit 1
 wait $WEBPACK && echo "Webpack complete" || exit 1
 

services/web/package.json

@@ -16,6 +16,7 @@
     "test:acceptance:app": "npm run test:acceptance:run_dir -- test/acceptance/src",
     "test:unit:run_dir": "mocha --recursive --reporter spec --timeout 25000 --exit --grep=$MOCHA_GREP --file test/unit/bootstrap.js",
     "test:unit:app": "npm run test:unit:run_dir -- test/unit/src",
+    "test:unit:app:parallel": "parallel --plain --keep-order --halt now,fail=1 npm run test:unit:run_dir -- {} ::: test/unit/src/*",
     "test:frontend": "karma start",
     "test:frontend:single": "karma start --single-run",
     "start": "node $NODE_APP_OPTIONS app.js",

services/web/test/unit/bootstrap.js

@@ -1,6 +1,9 @@
 const chai = require('chai')
 const sinon = require('sinon')
 
+// add chai.should()
+chai.should()
+
 // Load sinon-chai assertions so expect(stubFn).to.have.been.calledWith('abc')
 // has a nicer failure messages
 chai.use(require('sinon-chai'))