[monorepo] refactor retries in Jenkins to step level (#29008)

* [monorepo] refactor retries in Jenkins to step level

Change the junit reports to use step specific file names. The [hash]
template option was neat in getting unique file names, but results in
duplicate test reports on retry.

* [patches] add support for .cjs config files for mocha-multi-reporters

GitOrigin-RevId: 3a749441470b5ba633e71319589606cfbe860952

patches/cypress-multi-reporters+2.0.5.patch

@@ -0,0 +1,13 @@
+diff --git a/node_modules/cypress-multi-reporters/lib/MultiReporters.js b/node_modules/cypress-multi-reporters/lib/MultiReporters.js
+index 98dc4ef..b2a97bf 100644
+--- a/node_modules/cypress-multi-reporters/lib/MultiReporters.js
++++ b/node_modules/cypress-multi-reporters/lib/MultiReporters.js
+@@ -160,7 +160,7 @@ MultiReporters.prototype.getCustomOptions = function (options) {
+         debug('options file (custom)', customOptionsFile);
+ 
+         try {
+-            if ('.js' === path.extname(customOptionsFile)) {
++            if (['.js', '.cjs'].includes(path.extname(customOptionsFile))) {
+                 customOptions = require(customOptionsFile);
+             }
+             else {

patches/mocha-multi-reporters+1.5.1.patch

@@ -0,0 +1,13 @@
+diff --git a/node_modules/mocha-multi-reporters/lib/MultiReporters.js b/node_modules/mocha-multi-reporters/lib/MultiReporters.js
+index d61e019..e7a9515 100644
+--- a/node_modules/mocha-multi-reporters/lib/MultiReporters.js
++++ b/node_modules/mocha-multi-reporters/lib/MultiReporters.js
+@@ -153,7 +153,7 @@ MultiReporters.prototype.getCustomOptions = function (options) {
+         debug('options file (custom)', customOptionsFile);
+ 
+         try {
+-            if ('.js' === path.extname(customOptionsFile)) {
++            if (['.js', '.cjs'].includes(path.extname(customOptionsFile))) {
+                 customOptions = require(customOptionsFile);
+             }
+             else {

server-ce/test/Jenkinsfile

@@ -23,7 +23,6 @@ pipeline {
     timestamps()
     // Abort build after hitting first failure.
     parallelsAlwaysFailFast()
-    retry(3)
     timeout(time: 15, unit: 'MINUTES')
   }
   environment {
@@ -64,7 +63,9 @@ pipeline {
       parallel {
         stage('Install deps') {
           steps {
-            sh 'make monorepo_setup'
+            retry(count: 3) {
+              sh 'make monorepo_setup'
+            }
             script {
               job_npm_install_done = true
             }
@@ -103,8 +104,12 @@ pipeline {
               }
             }
             dir('copybara/public/repo/server-ce') {
-              sh 'make build-base'
-              sh 'make build-community'
+              retry(count: 3) {
+                sh 'make build-base'
+              }
+              retry(count: 3) {
+                sh 'make build-community'
+              }
             }
             script {
               job_server_ce_build_done = true
@@ -135,7 +140,9 @@ pipeline {
               }
             }
             dir('server-pro') {
-              sh 'make build-ci'
+              retry(count: 3) {
+                sh 'make build-ci'
+              }
             }
             script {
               job_server_pro_build_done = true
@@ -186,7 +193,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -202,7 +211,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -218,7 +229,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -234,7 +247,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -250,7 +265,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -266,7 +283,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -282,7 +301,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -298,7 +319,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }
@@ -314,7 +337,9 @@ pipeline {
               }
             }
             dir('server-ce/test') {
-              sh 'make test-e2e'
+              retry(count: 3) {
+                sh 'make test-e2e'
+              }
             }
           }
         }

server-ce/test/cypress.config.ts

@@ -38,7 +38,7 @@ if (process.env.CI) {
   reporterOptions = {
     reporter: `${process.env.MONOREPO}/node_modules/cypress-multi-reporters`,
     reporterOptions: {
-      configFile: 'cypress/cypress-multi-reporters.json',
+      configFile: 'cypress/cypress-multi-reporters.cjs',
     },
   }
 }

server-ce/test/cypress/cypress-multi-reporters.cjs

@@ -0,0 +1,10 @@
+module.exports = {
+  reporterEnabled: 'spec, mocha-junit-reporter',
+  mochaJunitReporterReporterOptions: {
+    mochaFile: `cypress-reports/junit-${process.env.CYPRESS_SHARD}-[suiteFilename].xml`,
+    includePending: true,
+    jenkinsMode: true,
+    jenkinsClassnamePrefix: 'Server Pro E2E tests',
+    rootSuiteTitle: 'Server Pro E2E tests',
+  },
+}

server-ce/test/cypress/cypress-multi-reporters.json

@@ -1,10 +0,0 @@
-{
-  "reporterEnabled": "spec, mocha-junit-reporter",
-  "mochaJunitReporterReporterOptions": {
-    "mochaFile": "cypress-reports/junit-[hash]-[suiteFilename].xml",
-    "includePending": true,
-    "useFullSuiteTitle": true,
-    "jenkinsMode": true,
-    "rootSuiteTitle": "Server Pro E2E tests"
-  }
-}

services/git-bridge/Jenkinsfile

@@ -7,7 +7,6 @@ pipeline {
   }
   options {
     timestamps()
-    parallelsAlwaysFailFast()
     timeout(time: 15, unit: 'MINUTES')
   }
   environment {
@@ -23,15 +22,12 @@ pipeline {
         stage('Build') {
           steps {
             dir('services/git-bridge') {
-              sh 'make docker_build_base'
+              retry(count: 3) {
+                sh 'make docker_build_base'
+              }
             }
           }
         }
-        stage('Install monorepo') {
-          steps {
-            sh 'make monorepo_setup'
-          }
-        }
       }
     }
     stage('Stage 2') {
@@ -39,7 +35,9 @@ pipeline {
         stage('Build production and push') {
           steps {
             dir('services/git-bridge') {
-              sh 'make docker_build'
+              retry(count: 3) {
+                sh 'make docker_build'
+              }
               sh 'make push_branch'
             }
           }
@@ -51,15 +49,12 @@ pipeline {
             }
           }
         }
-        stage('Format Jenkinsfile') {
-          steps {
-            sh 'bin/run monorepo npm run format:jenkins'
-          }
-        }
         stage('Test') {
           steps {
             dir('services/git-bridge') {
-              sh 'make docker_test'
+              retry(count: 3) {
+                sh 'make docker_test'
+              }
             }
           }
         }

services/web/.storybook/Jenkinsfile

@@ -10,7 +10,6 @@ pipeline {
   options {
     // Print timestamp next to each log line.
     timestamps()
-    retry(3)
     timeout(time: 15, unit: 'MINUTES')
   }
   environment {
@@ -20,7 +19,9 @@ pipeline {
   stages {
     stage('Install') {
       steps {
-        sh 'make install'
+        retry(count: 3) {
+          sh 'make monorepo_setup'
+        }
       }
     }
     stage('Build') {

services/web/Jenkinsfile

@@ -10,7 +10,6 @@ pipeline {
   options {
     // Print timestamp next to each log line.
     timestamps()
-    retry(3)
     timeout(time: 15, unit: 'MINUTES')
   }
   environment {
@@ -42,8 +41,12 @@ pipeline {
           steps {
             dir('services/web') {
               sh 'bin/copy_external_pages'
-              sh 'make build_deps'
-              sh 'make build_dev'
+              retry(count: 3) {
+                sh 'make build_deps'
+              }
+              retry(count: 3) {
+                sh 'make build_dev'
+              }
               sh 'make build_test_frontend_ct'
             }
           }
@@ -89,77 +92,99 @@ pipeline {
         stage('Acceptance SaaS') {
           steps {
             dir('services/web') {
-              sh 'make test_acceptance_app_saas'
+              retry(count: 3) {
+                sh 'make test_acceptance_app_saas'
+              }
             }
           }
         }
         stage('Acceptance Server CE') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_app_server_ce"
+              retry(count: 3) {
+                sh "make test_acceptance_app_server_ce"
+              }
             }
           }
         }
         stage('Acceptance Server Pro') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_app_server_pro"
+              retry(count: 3) {
+                sh "make test_acceptance_app_server_pro"
+              }
             }
           }
         }
         stage('test_acceptance_modules_merged_saas_1') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_modules_merged_saas_1"
+              retry(count: 3) {
+                sh "make test_acceptance_modules_merged_saas_1"
+              }
             }
           }
         }
         stage('test_acceptance_modules_merged_saas_2') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_modules_merged_saas_2"
+              retry(count: 3) {
+                sh "make test_acceptance_modules_merged_saas_2"
+              }
             }
           }
         }
         stage('test_acceptance_modules_merged_saas_3') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_modules_merged_saas_3"
+              retry(count: 3) {
+                sh "make test_acceptance_modules_merged_saas_3"
+              }
             }
           }
         }
         stage('test_acceptance_modules_merged_saas_4') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_modules_merged_saas_4"
+              retry(count: 3) {
+                sh "make test_acceptance_modules_merged_saas_4"
+              }
             }
           }
         }
         stage('test_acceptance_modules_merged_server_ce') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_modules_merged_server_ce"
+              retry(count: 3) {
+                sh "make test_acceptance_modules_merged_server_ce"
+              }
             }
           }
         }
         stage('test_acceptance_modules_merged_server_pro') {
           steps {
             dir('services/web') {
-              sh "make test_acceptance_modules_merged_server_pro"
+              retry(count: 3) {
+                sh "make test_acceptance_modules_merged_server_pro"
+              }
             }
           }
         }
         stage('test_frontend') {
           steps {
             dir('services/web') {
-              sh "make test_frontend"
+              retry(count: 3) {
+                sh "make test_frontend"
+              }
             }
           }
         }
         stage('test_writefull') {
           steps {
             dir('services/web') {
-              sh "make test_writefull"
+              retry(count: 3) {
+                sh "make test_writefull"
+              }
             }
           }
         }
@@ -169,7 +194,9 @@ pipeline {
           }
           steps {
             dir('services/web') {
-              sh "make test_frontend_ct_core_other"
+              retry(count: 3) {
+                sh "make test_frontend_ct_core_other"
+              }
             }
           }
         }
@@ -179,7 +206,9 @@ pipeline {
           }
           steps {
             dir('services/web') {
-              sh "make test_frontend_ct_core_features"
+              retry(count: 3) {
+                sh "make test_frontend_ct_core_features"
+              }
             }
           }
         }
@@ -189,35 +218,45 @@ pipeline {
           }
           steps {
             dir('services/web') {
-              sh "make test_frontend_ct_modules"
+              retry(count: 3) {
+                sh "make test_frontend_ct_modules"
+              }
             }
           }
         }
         stage('test_frontend_ct_editor_visual') {
           steps {
             dir('services/web') {
-              sh "make test_frontend_ct_editor_visual"
+              retry(count: 3) {
+                sh "make test_frontend_ct_editor_visual"
+              }
             }
           }
         }
         stage('test_frontend_ct_editor_other') {
           steps {
             dir('services/web') {
-              sh "make test_frontend_ct_editor_other"
+              retry(count: 3) {
+                sh "make test_frontend_ct_editor_other"
+              }
             }
           }
         }
         stage('Test Unit ESM') {
           steps {
             dir('services/web') {
-              sh "make test_unit_esm"
+              retry(count: 3) {
+                sh "make test_unit_esm"
+              }
             }
           }
         }
         stage('Test Unit Mocha') {
           steps {
             dir('services/web') {
-              sh "make test_unit_mocha"
+              retry(count: 3) {
+                sh "make test_unit_mocha"
+              }
             }
           }
         }
@@ -225,7 +264,7 @@ pipeline {
           stages {
             stage('Wait a bit to give tests all the CPU capacity') {
               steps {
-                sh 'sleep 120'
+                sh 'sleep 90'
               }
             }
             stage('Build Webpack') {
@@ -246,7 +285,9 @@ pipeline {
               steps {
                 dir('services/web') {
                   sh 'make tar'
-                  sh 'bin/cdn_upload'
+                  retry(count: 3) {
+                    sh 'bin/cdn_upload'
+                  }
                 }
               }
             }
@@ -261,7 +302,9 @@ pipeline {
               steps {
                 dir('services/web') {
                   sh 'gcloud secrets versions access latest --secret=web-sentryclirc > .sentryclirc'
-                  sh 'make sentry_upload'
+                  retry(count: 3) {
+                    sh 'make sentry_upload'
+                  }
                 }
               }
               post {
@@ -286,7 +329,7 @@ pipeline {
   }
   post {
     always {
-      junit checksName: 'Web test results', testResults: 'services/web/data/reports/junit-*.xml'
+      junit checksName: 'Web test results', testResults: 'services/web/data/reports/junit-*.xml,services/web/data/reports/junit-*/**/*.xml'
     }
     // Ensure tear down of test containers, then run general Jenkins VM cleanup.
     cleanup {

services/web/Makefile.module

@@ -1,5 +1,5 @@
 BUILD_DIR_NAME ?= web
-MODULE_NAME := $(notdir $(shell pwd))
+export MODULE_NAME := $(notdir $(shell pwd))
 MODULE_DIR := modules/$(MODULE_NAME)
 PROJECT_NAME = web
 

services/web/cypress/cypress-multi-reporters.json

@@ -1,7 +1,7 @@
 {
   "reporterEnabled": "spec, mocha-junit-reporter",
   "mochaJunitReporterReporterOptions": {
-    "mochaFile": "data/reports/junit-cypress-[hash].xml",
+    "mochaFile": "data/reports/junit-cypress-[suiteFilename].xml",
     "includePending": true,
     "jenkinsMode": true,
     "jenkinsClassnamePrefix": "Web Cypress tests",

services/web/docker-compose.ci.yml

@@ -20,6 +20,7 @@ services:
     env_file: docker-compose.common.env
     environment:
       CI:
+      MODULE_NAME:
       MOCHA_ROOT_SUITE_NAME:
       BASE_CONFIG:
       OVERLEAF_CONFIG:
@@ -42,6 +43,7 @@ services:
     env_file: docker-compose.common.env
     environment:
       CI:
+      MODULE_NAME:
       MOCHA_ROOT_SUITE_NAME:
       BASE_CONFIG:
       OVERLEAF_CONFIG:

services/web/docker-compose.yml

@@ -16,6 +16,7 @@ services:
       BASE_CONFIG:
       OVERLEAF_CONFIG:
       CI:
+      MODULE_NAME:
       MOCHA_ROOT_SUITE_NAME:
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}
@@ -43,6 +44,7 @@ services:
       BASE_CONFIG:
       OVERLEAF_CONFIG:
       CI:
+      MODULE_NAME:
       MOCHA_ROOT_SUITE_NAME:
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/web/test/mocha-multi-reporters.js

@@ -1,7 +1,7 @@
 module.exports = {
   reporterEnabled: 'spec, mocha-junit-reporter',
   mochaJunitReporterReporterOptions: {
-    mochaFile: 'data/reports/junit-mocha-[hash]-[suiteFilename].xml',
+    mochaFile: `data/reports/junit-mocha-${process.env.MOCHA_ROOT_SUITE_NAME}-${process.env.MODULE_NAME}.xml`,
     includePending: true,
     jenkinsMode: true,
     jenkinsClassnamePrefix: process.env.MOCHA_ROOT_SUITE_NAME,