Overleaf/server-ce/test/Jenkinsfile

// Initialize variables to signal that a given stage finished.
// We use them to build a graph of interconnected steps/dependencies.
// - Incoming edges use "waitUntil" and reference the given variables of dependencies.
// - Outgoing edges set the given variable to true.
def job_copybara_done = false
def job_npm_install_done = false
def job_prefetch_custom_done = false
def job_prefetch_default_done = false
def job_server_ce_build_done = false
def job_server_pro_build_done = false

pipeline {
  agent {
    node {
      // Select a VM with the given tabel.
      label 'jenkins-agent-web'
      // Use the monorepo checkout in /workspace.
      customWorkspace '/workspace'
    }
  }
  options {
    // Print timestamp next to each log line.
    timestamps()
    // Abort build after hitting first failure.
    parallelsAlwaysFailFast()
    timeout(time: 20, unit: 'MINUTES')
  }
  environment {
    BRANCH_NAME = "${GIT_BRANCH.replace('origin/', '')}"
    COMMIT_SHA = "${GIT_COMMIT}"
    SHORT_SHA = "${GIT_COMMIT.take(7)}"
    OVERLEAF_BASE_BRANCH = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base-internal:${BRANCH_NAME}"
    OVERLEAF_BASE_LATEST = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base-internal:main"
    OVERLEAF_BASE_TAG = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-base-internal:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}"
    OVERLEAF_BRANCH = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-internal:${BRANCH_NAME}"
    OVERLEAF_LATEST = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-internal:main"
    OVERLEAF_TAG = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/overleaf-internal:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}"
    IMAGE_TAG_CE = "${OVERLEAF_TAG}"
    IMAGE_TAG_PRO = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro:${BRANCH_NAME}-${SHORT_SHA}_${BUILD_ID}"
    OVERLEAF_PRO_TAG_BRANCH = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro-internal:${BRANCH_NAME}"
    OVERLEAF_PRO_TAG_LATEST = "us-east1-docker.pkg.dev/overleaf-ops/ol-docker/pro-internal:main"
  }
  stages {
    // Retries will use the same pipeline instance. Reset the vars.
    stage('Reset vars') {
      steps {
        script {
          job_copybara_done = false
          job_npm_install_done = false
          job_prefetch_custom_done = false
          job_prefetch_default_done = false
          job_server_ce_build_done = false
          job_server_pro_build_done = false
        }
        // Reset the results folder.
        // Use a folder that is not managed by cypress, as cypress will clear its results folder at the start of each individual run.
        // I.e. we would loose the test results from finished/running test suites when the last test suite starts.
        sh 'rm -rf server-ce/test/cypress-reports/'
        sh 'mkdir -p server-ce/test/cypress-reports/'
      }
    }
    stage('Parallel') {
      parallel {
        stage('Install deps') {
          steps {
            retry(count: 3) {
              sh 'make monorepo_setup'
            }
            script {
              job_npm_install_done = true
            }
          }
        }
        stage('shellcheck') {
          steps {
            dir('server-ce') {
              sh 'make shellcheck'
            }
          }
        }
        stage('Format') {
          steps {
            script {
              waitUntil {
                return job_npm_install_done
              }
            }
            sh 'bin/run -w /overleaf/server-ce/test monorepo npm run format'
          }
        }
        stage('Copybara') {
          steps {
            sh 'copybara/bin/sync'
            script {
              job_copybara_done = true
            }
          }
        }
        stage('Build CE image') {
          steps {
            script {
              waitUntil {
                return job_copybara_done
              }
            }
            dir('copybara/public/repo/server-ce') {
              sh 'make refresh-cache -j2'
              retry(count: 3) {
                sh 'make build-base'
              }
              retry(count: 3) {
                sh 'make build-community'
              }
            }
            script {
              job_server_ce_build_done = true
            }
          }
        }
        stage('Push CE to internal') {
          steps {
            script {
              waitUntil {
                return job_server_ce_build_done
              }
            }
            dir('copybara/public/repo/server-ce') {
              sh 'make push'
            }
          }
        }
        stage('Build Pro image') {
          environment {
            OVERLEAF_CE_TAG = "${OVERLEAF_TAG}"
            OVERLEAF_PRO_TAG = "${IMAGE_TAG_PRO}"
          }
          steps {
            script {
              waitUntil {
                return job_server_ce_build_done
              }
            }
            dir('server-pro') {
              retry(count: 3) {
                sh 'make build-ci'
              }
            }
            script {
              job_server_pro_build_done = true
            }
          }
        }
        stage('Push Pro to internal') {
          steps {
            script {
              waitUntil {
                return job_server_pro_build_done
              }
            }
            dir('server-pro') {
              sh 'make push_branch'
            }
          }
        }
        stage('Prefetch default') {
          steps {
            dir('server-ce/test') {
              sh 'make prefetch_default -j4'
            }
            script {
              job_prefetch_default_done = true
            }
          }
        }
        stage('Prefetch custom') {
          steps {
            dir('server-ce/test') {
              sh 'make prefetch_custom -j4'
            }
            script {
              job_prefetch_custom_done = true
            }
          }
        }
        stage('CE default') {
          environment {
            CYPRESS_SHARD = "CE_DEFAULT"
            COMPOSE_PROJECT_NAME = "test-ce-default"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_ce_build_done && job_prefetch_default_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('CE custom 1') {
          environment {
            CYPRESS_SHARD = "CE_CUSTOM_1"
            COMPOSE_PROJECT_NAME = "test-ce-custom-1"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_ce_build_done && job_prefetch_default_done && job_prefetch_custom_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO default 1') {
          environment {
            CYPRESS_SHARD = "PRO_DEFAULT_1"
            COMPOSE_PROJECT_NAME = "test-pro-default-1"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO default 2') {
          environment {
            CYPRESS_SHARD = "PRO_DEFAULT_2"
            COMPOSE_PROJECT_NAME = "test-pro-default-2"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO custom 1') {
          environment {
            CYPRESS_SHARD = "PRO_CUSTOM_1"
            COMPOSE_PROJECT_NAME = "test-pro-custom-1"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO custom 2') {
          environment {
            CYPRESS_SHARD = "PRO_CUSTOM_2"
            COMPOSE_PROJECT_NAME = "test-pro-custom-2"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO custom 3') {
          environment {
            CYPRESS_SHARD = "PRO_CUSTOM_3"
            COMPOSE_PROJECT_NAME = "test-pro-custom-3"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO custom 4') {
          environment {
            CYPRESS_SHARD = "PRO_CUSTOM_4"
            COMPOSE_PROJECT_NAME = "test-pro-custom-4"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
        stage('PRO custom 5') {
          environment {
            CYPRESS_SHARD = "PRO_CUSTOM_5"
            COMPOSE_PROJECT_NAME = "test-pro-custom-5"
          }
          steps {
            script {
              waitUntil {
                return job_npm_install_done && job_server_pro_build_done && job_prefetch_default_done && job_prefetch_custom_done
              }
            }
            dir('server-ce/test') {
              retry(count: 3) {
                sh 'make test-e2e'
              }
            }
          }
        }
      }
    }
  }
  post {
    // Collect junit test results for both success and failure case.
    always {
      junit checksName: 'Server Pro E2E test results', testResults: 'server-ce/test/cypress-reports/junit-*.xml'
    }
    // Ensure tear down of test containers, remove CE docker images, then run general Jenkins VM cleanup.
    cleanup {
      dir('server-ce/test') {
        sh 'make clean -j10'
      }
      dir('server-ce') {
        sh 'make clean'
      }
      dir('server-pro') {
        sh 'make clean'
      }
      sh 'make clean_jenkins -j10'
    }
  }
}

// vim: set ft=groovy :