[monorepo] extend cleanup in Jenkins VMs (#27793)

* [monorepo] extend docker compose cleanup

* [web] cleanup docker images

Only retain the branch builds for dependencies.

* [server-ce] jenkins: remove build specific image

Retain the branch image as cache.

* [monorepo] jenkins: prune untagged docker images and docker build cache

* [monorepo] jenkins: remove temporary files from monorepo

* [monorepo] jenkins: remove containers from sandboxed compiles

GitOrigin-RevId: c847015986a060c5c84c39cd06236fd73bbf6f3a

server-ce/Makefile

@@ -13,6 +13,9 @@ export OVERLEAF_TAG ?= sharelatex/sharelatex:$(BRANCH_NAME)-$(MONOREPO_REVISION)
 
 all: build-base build-community
 
+clean:
+	-docker rmi $(OVERLEAF_BASE_TAG) $(OVERLEAF_TAG)
+
 build-base:
 	cp .dockerignore $(MONOREPO_ROOT)
 	docker build \

server-ce/test/Jenkinsfile

@@ -223,6 +223,10 @@ pipeline {
       dir('server-ce/test') {
         sh 'make clean'
       }
+      dir('server-ce') {
+        sh 'make clean'
+      }
+      sh 'make clean_jenkins'
     }
   }
 }

server-ce/test/Makefile

@@ -44,10 +44,10 @@ SHARD_PROJECT_NAMES = \
 CLEAN_SHARDS=$(addprefix clean/,$(SHARD_PROJECT_NAMES))
 clean: $(CLEAN_SHARDS)
 	-docker compose run --no-deps --rm --entrypoint rm host-admin -rf docker-compose.override.yml docker-compose.*_*.yml data/
-	-docker compose down --volumes --timeout 0
+	-docker compose down --remove-orphans --rmi local --timeout 0 --volumes
 
 $(CLEAN_SHARDS): clean/%:
-	-COMPOSE_PROJECT_NAME=$* docker compose down --timeout=0 --volumes
+	-COMPOSE_PROJECT_NAME=$* docker compose down --remove-orphans --rmi local --timeout 0 --volumes
 
 prefetch: prefetch_default
 prefetch_default: prefetch_default_compose

services/chat/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/clsi/Makefile

@@ -25,8 +25,9 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
+	-git clean -dfX cache compiles output
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/contacts/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/docstore/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/document-updater/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/filestore/Makefile

@@ -25,8 +25,9 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
+	-git clean -dfX uploads user_files template_files
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/history-v1/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/notifications/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/project-history/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/real-time/Makefile

@@ -25,8 +25,8 @@ DOCKER_COMPOSE_TEST_UNIT = \
 clean:
 	-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
 	-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-	-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
+	-$(DOCKER_COMPOSE_TEST_UNIT) down --remove-orphans --rmi local --timeout 0 --volumes
+	-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --remove-orphans --rmi local --timeout 0 --volumes
 
 HERE=$(shell pwd)
 MONOREPO=$(shell cd ../../ && pwd)

services/web/Jenkinsfile

@@ -479,6 +479,7 @@ pipeline {
       dir('services/web') {
         sh 'make clean'
       }
+      sh 'make clean_jenkins'
     }
   }
 }

services/web/Makefile

@@ -30,24 +30,32 @@ $(MODULE_MAKEFILES): Makefile.module
 # Clean
 #
 
-clean:
-	-$(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=unit_test_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=unit_test_all_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=unit_test_parallel_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=unit_test_parallel_make_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_test_saas_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_test_server_ce_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_test_server_pro_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_saas_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_saas_1_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_saas_2_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_saas_3_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_saas_4_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_server_ce_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=acceptance_modules_merged_server_pro_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=test_frontend_ct_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
-	-COMPOSE_PROJECT_NAME=tar_$(BUILD_DIR_NAME) $(DOCKER_COMPOSE) down --rmi local
+SHARD_PROJECT_NAMES = \
+	unit_test_$(BUILD_DIR_NAME) \
+	unit_test_all_$(BUILD_DIR_NAME) \
+	unit_test_parallel_$(BUILD_DIR_NAME) \
+	unit_test_parallel_make_$(BUILD_DIR_NAME) \
+	acceptance_test_saas_$(BUILD_DIR_NAME) \
+	acceptance_test_server_ce_$(BUILD_DIR_NAME) \
+	acceptance_test_server_pro_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_saas_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_saas_1_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_saas_2_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_saas_3_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_saas_4_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_server_ce_$(BUILD_DIR_NAME) \
+	acceptance_modules_merged_server_pro_$(BUILD_DIR_NAME) \
+	test_frontend_ct_$(BUILD_DIR_NAME) \
+	tar_$(BUILD_DIR_NAME) \
+
+CLEAN_SHARDS=$(addprefix clean/,$(SHARD_PROJECT_NAMES))
+clean: $(CLEAN_SHARDS)
+	-docker rmi --force $(IMAGE_CI) $(IMAGE_CI)-dev $(IMAGE_CI)-pug $(IMAGE_CI)-webpack $(IMAGE_REPO_FINAL)
+	-docker compose down --remove-orphans --rmi local --timeout 0 --volumes
+	-git clean -dfX data/
+
+$(CLEAN_SHARDS): clean/%:
+	-COMPOSE_PROJECT_NAME=$* docker compose down --remove-orphans --rmi local --timeout 0 --volumes
 
 clean_ci:
 	$(DOCKER_COMPOSE) down -v -t 0