move migrations to shared location (#28306)

* fix: correct typedef for Document in helpers.mjs

* add move-migrations codemod

* update migration paths to use shared migrations directory

* move migrations to shared location

* fix: update Dockerfile and docker-compose.ci.yml to include migrations directory

* feat: add migrations tool to workspaces in package.json

* [monorepo] Fix order of docker ignore rules

* [web] remove unused docker ignore file

* [monorepo] replace old references to migrations folder

* [server-ce] copy migrations from new place

* [migrations] Inline web scripts

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] move three web scripts over

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] add missing collection

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] remove lodash dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] avoid mongodb-legacy dependency

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [monorepo] run migrations from tools/migrations

Co-authored-by: Brian Gough <brian.gough@overleaf.com>

* [migrations] simplify migration for adding gitBridge feature to users

* [monorepo] run migrations from tests in all the services

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] fixup running web migrations everywhere

* [monorepo] trigger Jenkins builds on changes to mongo migrations

* [migrations] add Jenkins pipeline for linting/formatting

* [monorepo] build scripts: update devDependencies before deps scanning

* [monorepo] build scripts: formerly depend on tools/migrations

* [monorepo] run eslint on .mjs files

* [migrations] enable more eslint rules and fix all the errors

* [rake] fix migrations:list task

---------

Co-authored-by: Jakob Ackermann <jakob.ackermann@overleaf.com>
GitOrigin-RevId: 14cf69cc1b9405bbc75adbb9a000e555500e0614

libraries/access-token-encryptor/package.json

@@ -5,8 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",

libraries/fetch-utils/package.json

@@ -5,8 +5,8 @@
   "main": "index.js",
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",

libraries/logger/package.json

@@ -13,8 +13,8 @@
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "test:ci": "npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
     "types:check": "tsc --noEmit"

libraries/metrics/package.json

@@ -31,8 +31,8 @@
     "typescript": "^5.0.4"
   },
   "scripts": {
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
     "test:acceptance": "mocha --recursive --exit --grep=$MOCHA_GREP test/acceptance",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

libraries/mongo-utils/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",

libraries/o-error/package.json

@@ -18,8 +18,8 @@
   ],
   "scripts": {
     "build": "npm run --silent test",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",

libraries/object-persistor/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",

libraries/overleaf-editor-core/package.json

@@ -7,8 +7,8 @@
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "test:ci": "npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
     "types:check": "tsc --noEmit"

libraries/promise-utils/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",

libraries/ranges-tracker/package.json

@@ -9,8 +9,8 @@
   "author": "Overleaf (https://www.overleaf.com)",
   "private": true,
   "scripts": {
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

libraries/redis-wrapper/package.json

@@ -13,8 +13,8 @@
   "repository": "github:overleaf/redis-wrapper",
   "license": "ISC",
   "scripts": {
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

libraries/settings/package.json

@@ -5,8 +5,8 @@
   "repository": "overleaf/settings-module",
   "main": "index.js",
   "scripts": {
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",

libraries/stream-utils/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "test:unit": "mocha --exit test/**/*.{js,cjs}",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
     "test:ci": "npm run test:unit",

libraries/validation-tools/package.json

@@ -13,8 +13,8 @@
     "test": "npm run lint && npm run format && npm run types:check && npm run test:unit",
     "format": "prettier --list-different $PWD/'**/*.{js,cjs,ts}'",
     "format:fix": "prettier --write $PWD/'**/*.{js,cjs,ts}'",
-    "lint": "eslint --ext .js --ext .cjs --ext .ts --max-warnings 0 --format unix .",
-    "lint:fix": "eslint --fix --ext .js --ext .cjs --ext .ts .",
+    "lint": "eslint --ext .cjs,.js,.jsx,.mjs,.ts --max-warnings 0 --format unix .",
+    "lint:fix": "eslint --fix --ext .cjs,.js,.jsx,.mjs,.ts .",
     "test:ci": "npm run test:unit",
     "test:unit": "vitest --config vitest.config.ts",
     "types:check": "tsc --noEmit"

package-lock.json

@@ -33,7 +33,8 @@
         "services/third-party-references",
         "services/tpdsworker",
         "services/web",
-        "tools/saas-e2e"
+        "tools/saas-e2e",
+        "tools/migrations"
       ],
       "dependencies": {
         "patch-package": "^8.0.0"
@@ -14867,6 +14868,10 @@
       "resolved": "libraries/metrics",
       "link": true
     },
+    "node_modules/@overleaf/migrations": {
+      "resolved": "tools/migrations",
+      "link": true
+    },
     "node_modules/@overleaf/mongo-utils": {
       "resolved": "libraries/mongo-utils",
       "link": true
@@ -15248,7 +15253,6 @@
       "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
       "dev": true,
       "optional": true,
-      "peer": true,
       "engines": {
         "node": ">=14"
       }
@@ -51368,6 +51372,7 @@
         "mongodb": "6.12.0"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "acorn": "^7.1.1",
         "ajv": "^6.12.0",
         "chai": "^4.3.6",
@@ -51531,6 +51536,7 @@
         "underscore": "~1.13.1"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "esmock": "^2.6.3",
@@ -51605,6 +51611,7 @@
       },
       "devDependencies": {
         "@google-cloud/storage": "^6.10.1",
+        "@overleaf/migrations": "*",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "mocha": "^11.1.0",
@@ -51682,6 +51689,7 @@
         "requestretry": "^7.1.0"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "cluster-key-slot": "^1.0.5",
@@ -51815,6 +51823,7 @@
       },
       "devDependencies": {
         "@overleaf/fetch-utils": "*",
+        "@overleaf/migrations": "*",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "mocha": "^11.1.0",
@@ -51949,6 +51958,7 @@
         "utf-8-validate": "^5.0.4"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "benny": "^3.7.1",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
@@ -52689,6 +52699,7 @@
         "zod-validation-error": "^4.0.1"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "@types/method-override": "^3.0.0",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
@@ -52724,6 +52735,7 @@
         "request": "^2.88.2"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "mocha": "^11.1.0",
@@ -52896,6 +52908,7 @@
         "request": "^2.88.2"
       },
       "devDependencies": {
+        "@overleaf/migrations": "*",
         "chai": "^4.3.6",
         "chai-as-promised": "^7.1.1",
         "mocha": "^11.1.0",
@@ -53150,6 +53163,7 @@
         "@overleaf/fetch-utils": "*",
         "@overleaf/logger": "*",
         "@overleaf/metrics": "*",
+        "@overleaf/migrations": "*",
         "@overleaf/mongo-utils": "*",
         "@overleaf/o-error": "*",
         "@overleaf/object-persistor": "*",
@@ -53185,7 +53199,6 @@
         "csurf": "^1.11.0",
         "csv": "^6.2.5",
         "dateformat": "1.0.4-1.2.3",
-        "east": "^2.0.2",
         "ejs": "^3.1.10",
         "email-addresses": "^5.0.0",
         "eventsource-parser": "^1.1.2",
@@ -54545,6 +54558,18 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
     },
+    "tools/migrations": {
+      "name": "@overleaf/migrations",
+      "dependencies": {
+        "@overleaf/logger": "*",
+        "@overleaf/mongo-utils": "*",
+        "@overleaf/o-error": "*",
+        "@overleaf/promise-utils": "*",
+        "@overleaf/settings": "*",
+        "east": "2.0.3",
+        "mongodb": "6.12.0"
+      }
+    },
     "tools/saas-e2e": {
       "name": "@overleaf/saas-e2e",
       "devDependencies": {

package.json

@@ -76,6 +76,7 @@
     "services/third-party-references",
     "services/tpdsworker",
     "services/web",
-    "tools/saas-e2e"
+    "tools/saas-e2e",
+    "tools/migrations"
   ]
 }

server-ce/Dockerfile

@@ -14,6 +14,7 @@ ADD server-ce/services.js /overleaf/services.js
 ADD package.json package-lock.json /overleaf/
 ADD libraries/ /overleaf/libraries/
 ADD services/ /overleaf/services/
+ADD tools/migrations/ /overleaf/tools/migrations/
 
 # Add npm patches
 # -----------------------

server-ce/init_scripts/900_run_web_migrations.sh

@@ -8,6 +8,6 @@ else
 fi
 
 echo "Running migrations for $environment"
-cd /overleaf/services/web
+cd /overleaf/tools/migrations
 /sbin/setuser www-data npm run migrations -- migrate -t "$environment"
 echo "Finished migrations"

server-ce/test/.jenkinsIncludeFile

@@ -68,7 +68,6 @@ services/web/config/settings.defaults.js
 services/web/config/settings.overrides.server-pro.js
 services/web/frontend/**
 services/web/locales/**
-services/web/migrations/**
 services/web/modules/*/*
 services/web/modules/*/app/**
 services/web/modules/*/frontend/**
@@ -78,6 +77,8 @@ services/web/public/**
 services/web/types/**
 services/web/webpack-plugins/**
 
+tools/migrations/**
+
 .dockerignore
 .eslint*
 .pretter*

services/chat/.jenkinsIncludeFile

@@ -6,8 +6,10 @@ libraries/logger/**
 libraries/metrics/**
 libraries/mongo-utils/**
 libraries/o-error/**
+libraries/promise-utils/**
 libraries/settings/**
 package-lock.json
 package.json
 patches/**
 services/chat/**
+tools/migrations/**

services/chat/Dockerfile

@@ -18,9 +18,12 @@ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
 COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
 COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
 COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
+COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY services/chat/package.json /overleaf/services/chat/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -28,8 +31,10 @@ COPY libraries/logger/ /overleaf/libraries/logger/
 COPY libraries/metrics/ /overleaf/libraries/metrics/
 COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
 COPY libraries/o-error/ /overleaf/libraries/o-error/
+COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY services/chat/ /overleaf/services/chat/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/chat/Makefile

@@ -19,8 +19,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/metrics/package.json \
 	$(MONOREPO)/libraries/mongo-utils/package.json \
 	$(MONOREPO)/libraries/o-error/package.json \
+	$(MONOREPO)/libraries/promise-utils/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/services/chat/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/chat/docker-compose.yml

@@ -31,10 +31,11 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/chat
     environment:
       ELASTIC_SEARCH_DSN: es:9200
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/chat/package.json

@@ -29,6 +29,7 @@
     "mongodb": "6.12.0"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "acorn": "^7.1.1",
     "ajv": "^6.12.0",
     "chai": "^4.3.6",

services/chat/test/acceptance/js/helpers/ChatApp.js

@@ -1,5 +1,6 @@
 import { createServer } from '../../../../app/js/server.js'
 import { promisify } from 'node:util'
+import './MongoHelper.js'
 
 export { db } from '../../../../app/js/mongodb.js'
 

services/chat/test/acceptance/js/helpers/MongoHelper.js

@@ -0,0 +1,10 @@
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+
+before('run migrations', async function () {
+  this.timeout(60_000)
+
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
+})

services/clsi/docker-compose.yml

@@ -34,7 +34,7 @@ services:
     working_dir: /overleaf/services/clsi
     environment:
       ELASTIC_SEARCH_DSN: es:9200
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/contacts/.jenkinsIncludeFile

@@ -6,8 +6,10 @@ libraries/logger/**
 libraries/metrics/**
 libraries/mongo-utils/**
 libraries/o-error/**
+libraries/promise-utils/**
 libraries/settings/**
 package-lock.json
 package.json
 patches/**
 services/contacts/**
+tools/migrations/**

services/contacts/Dockerfile

@@ -18,9 +18,12 @@ COPY libraries/logger/package.json /overleaf/libraries/logger/package.json
 COPY libraries/metrics/package.json /overleaf/libraries/metrics/package.json
 COPY libraries/mongo-utils/package.json /overleaf/libraries/mongo-utils/package.json
 COPY libraries/o-error/package.json /overleaf/libraries/o-error/package.json
+COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY services/contacts/package.json /overleaf/services/contacts/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -28,8 +31,10 @@ COPY libraries/logger/ /overleaf/libraries/logger/
 COPY libraries/metrics/ /overleaf/libraries/metrics/
 COPY libraries/mongo-utils/ /overleaf/libraries/mongo-utils/
 COPY libraries/o-error/ /overleaf/libraries/o-error/
+COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY services/contacts/ /overleaf/services/contacts/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/contacts/Makefile

@@ -19,8 +19,10 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/metrics/package.json \
 	$(MONOREPO)/libraries/mongo-utils/package.json \
 	$(MONOREPO)/libraries/o-error/package.json \
+	$(MONOREPO)/libraries/promise-utils/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/services/contacts/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/contacts/docker-compose.yml

@@ -31,10 +31,11 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/contacts
     environment:
       ELASTIC_SEARCH_DSN: es:9200
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/contacts/package.json

@@ -31,6 +31,7 @@
     "underscore": "~1.13.1"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "esmock": "^2.6.3",

services/contacts/test/acceptance/js/MongoHelper.js

@@ -0,0 +1,10 @@
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+
+before('run migrations', async function () {
+  this.timeout(60_000)
+
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
+})

services/docstore/.jenkinsIncludeFile

@@ -14,3 +14,4 @@ package-lock.json
 package.json
 patches/**
 services/docstore/**
+tools/migrations/**

services/docstore/Dockerfile

@@ -23,7 +23,9 @@ COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/pack
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
 COPY services/docstore/package.json /overleaf/services/docstore/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -36,6 +38,7 @@ COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
 COPY services/docstore/ /overleaf/services/docstore/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/docstore/Makefile

@@ -24,6 +24,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
 	$(MONOREPO)/services/docstore/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/docstore/docker-compose.yml

@@ -31,10 +31,11 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/docstore
     environment:
       ELASTIC_SEARCH_DSN: es:9200
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       GCS_API_ENDPOINT: http://gcs:9090
       GCS_PROJECT_ID: fake

services/docstore/package.json

@@ -37,6 +37,7 @@
   },
   "devDependencies": {
     "@google-cloud/storage": "^6.10.1",
+    "@overleaf/migrations": "*",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "mocha": "^11.1.0",

services/docstore/test/acceptance/js/helpers/DocstoreApp.js

@@ -1,5 +1,6 @@
 const app = require('../../../../app')
 const Settings = require('@overleaf/settings')
+require('./MongoHelper')
 
 function startApp() {
   return new Promise((resolve, reject) => {

services/docstore/test/acceptance/js/helpers/MongoHelper.js

@@ -0,0 +1,10 @@
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+
+before('run migrations', async function () {
+  this.timeout(60_000)
+
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
+})

services/document-updater/.jenkinsIncludeFile

@@ -15,3 +15,4 @@ package-lock.json
 package.json
 patches/**
 services/document-updater/**
+tools/migrations/**

services/document-updater/Dockerfile

@@ -24,7 +24,9 @@ COPY libraries/ranges-tracker/package.json /overleaf/libraries/ranges-tracker/pa
 COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/package.json
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY services/document-updater/package.json /overleaf/services/document-updater/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -38,6 +40,7 @@ COPY libraries/ranges-tracker/ /overleaf/libraries/ranges-tracker/
 COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY services/document-updater/ /overleaf/services/document-updater/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/document-updater/Makefile

@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/redis-wrapper/package.json \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/services/document-updater/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/document-updater/docker-compose.yml

@@ -37,6 +37,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/document-updater
     environment:
       ELASTIC_SEARCH_DSN: es:9200
@@ -44,7 +45,7 @@ services:
       HISTORY_REDIS_HOST: redis_test
       QUEUES_REDIS_HOST: redis_test
       ANALYTICS_QUEUES_REDIS_HOST: redis_test
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/document-updater/package.json

@@ -41,6 +41,7 @@
     "requestretry": "^7.1.0"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "cluster-key-slot": "^1.0.5",

services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js

@@ -1,4 +1,5 @@
 const app = require('../../../../app')
+require('./MongoHelper')
 
 function startApp() {
   return new Promise((resolve, reject) => {

services/document-updater/test/acceptance/js/helpers/MongoHelper.js

@@ -0,0 +1,10 @@
+const { exec } = require('node:child_process')
+const { promisify } = require('node:util')
+
+before('run migrations', async function () {
+  this.timeout(60_000)
+
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
+})

services/filestore/docker-compose.yml

@@ -38,7 +38,7 @@ services:
     working_dir: /overleaf/services/filestore
     environment:
       ELASTIC_SEARCH_DSN: es:9200
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: https://minio:9000
       AWS_S3_PATH_STYLE: 'true'

services/history-v1/.jenkinsIncludeFile

@@ -16,3 +16,4 @@ package-lock.json
 package.json
 patches/**
 services/history-v1/**
+tools/migrations/**

services/history-v1/Dockerfile

@@ -30,7 +30,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
 COPY services/history-v1/package.json /overleaf/services/history-v1/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -45,6 +47,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
 COPY services/history-v1/ /overleaf/services/history-v1/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/history-v1/Makefile

@@ -26,6 +26,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
 	$(MONOREPO)/services/history-v1/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/history-v1/docker-compose.yml

@@ -47,6 +47,7 @@ services:
       - ../../libraries:/overleaf/libraries
       - minio-certs:/certs
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/history-v1
     environment:
       ELASTIC_SEARCH_DSN: es:9200
@@ -54,7 +55,7 @@ services:
       HISTORY_REDIS_HOST: redis_test
       QUEUES_REDIS_HOST: redis_test
       ANALYTICS_QUEUES_REDIS_HOST: redis_test
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: https://minio:9000
       AWS_S3_PATH_STYLE: 'true'

services/history-v1/package.json

@@ -47,6 +47,7 @@
     "utf-8-validate": "^5.0.4"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "benny": "^3.7.1",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",

services/history-v1/test/setup.js

@@ -2,7 +2,9 @@ const chai = require('chai')
 const chaiAsPromised = require('chai-as-promised')
 const config = require('config')
 const fetch = require('node-fetch')
-const { knex, mongodb, redis } = require('../storage')
+const { knex, redis } = require('../storage')
+const { exec } = require('node:child_process')
+const { promisify } = require('node:util')
 
 // ensure every ObjectId has the id string as a property for correct comparisons
 require('mongodb').ObjectId.cacheHexString = true
@@ -17,19 +19,9 @@ async function setupPostgresDatabase() {
 
 async function setupMongoDatabase() {
   this.timeout(60_000)
-  await mongodb.db.collection('projectHistoryChunks').createIndexes([
-    {
-      key: { projectId: 1, startVersion: 1 },
-      name: 'projectId_1_startVersion_1',
-      partialFilterExpression: { state: { $in: ['active', 'closed'] } },
-      unique: true,
-    },
-    {
-      key: { state: 1 },
-      name: 'state_1',
-      partialFilterExpression: { state: 'deleted' },
-    },
-  ])
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
 }
 
 async function createGcsBuckets() {

services/notifications/.jenkinsIncludeFile

@@ -13,3 +13,4 @@ package-lock.json
 package.json
 patches/**
 services/notifications/**
+tools/migrations/**

services/notifications/Dockerfile

@@ -22,7 +22,9 @@ COPY libraries/promise-utils/package.json /overleaf/libraries/promise-utils/pack
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
 COPY services/notifications/package.json /overleaf/services/notifications/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -34,6 +36,7 @@ COPY libraries/promise-utils/ /overleaf/libraries/promise-utils/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/notifications/ /overleaf/services/notifications/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/notifications/Makefile

@@ -23,6 +23,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/validation-tools/package.json \
 	$(MONOREPO)/services/notifications/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/notifications/docker-compose.yml

@@ -32,11 +32,12 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
       - ../../tsconfig.backend.json:/overleaf/tsconfig.backend.json
     working_dir: /overleaf/services/notifications
     environment:
       ELASTIC_SEARCH_DSN: es:9200
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/notifications/package.json

@@ -38,6 +38,7 @@
     "zod-validation-error": "^4.0.1"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "@types/method-override": "^3.0.0",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",

services/notifications/test/acceptance/js/HealthCheck.test.ts

@@ -2,6 +2,7 @@ import { beforeAll, describe, it, expect } from 'vitest'
 import { fetchStringWithResponse } from '@overleaf/fetch-utils'
 import app from '../../../app.ts'
 import logger from '@overleaf/logger'
+import './MongoHelper.ts'
 
 let runAppPromise: Promise<void> | null = null
 

services/notifications/test/acceptance/js/MongoHelper.ts

@@ -0,0 +1,9 @@
+import { beforeAll } from 'vitest'
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+
+beforeAll(async function () {
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
+}, 60_000)

services/project-history/.jenkinsIncludeFile

@@ -15,3 +15,4 @@ package-lock.json
 package.json
 patches/**
 services/project-history/**
+tools/migrations/**

services/project-history/Dockerfile

@@ -24,7 +24,9 @@ COPY libraries/redis-wrapper/package.json /overleaf/libraries/redis-wrapper/pack
 COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
 COPY services/project-history/package.json /overleaf/services/project-history/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 RUN cd /overleaf && npm ci --quiet
 COPY libraries/fetch-utils/ /overleaf/libraries/fetch-utils/
@@ -38,6 +40,7 @@ COPY libraries/redis-wrapper/ /overleaf/libraries/redis-wrapper/
 COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
 COPY services/project-history/ /overleaf/services/project-history/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 FROM app
 USER node

services/project-history/Makefile

@@ -25,6 +25,7 @@ IMAGE_CACHE ?= $(IMAGE_REPO):cache-$(shell cat \
 	$(MONOREPO)/libraries/settings/package.json \
 	$(MONOREPO)/libraries/stream-utils/package.json \
 	$(MONOREPO)/services/project-history/package.json \
+	$(MONOREPO)/tools/migrations/package.json \
 	$(MONOREPO)/patches/* \
 | sha256sum | cut -d '-' -f1)
 

services/project-history/docker-compose.yml

@@ -37,6 +37,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/project-history
     environment:
       ELASTIC_SEARCH_DSN: es:9200
@@ -44,7 +45,7 @@ services:
       HISTORY_REDIS_HOST: redis_test
       QUEUES_REDIS_HOST: redis_test
       ANALYTICS_QUEUES_REDIS_HOST: redis_test
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/project-history/package.json

@@ -42,6 +42,7 @@
     "request": "^2.88.2"
   },
   "devDependencies": {
+    "@overleaf/migrations": "*",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "mocha": "^11.1.0",

services/project-history/test/acceptance/js/helpers/MongoHelper.js

@@ -0,0 +1,10 @@
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+
+before('run migrations', async function () {
+  this.timeout(60_000)
+
+  await promisify(exec)(
+    `cd ../../tools/migrations && npm run migrations -- migrate -t server-ce`
+  )
+})

services/project-history/test/acceptance/js/helpers/ProjectHistoryApp.js

@@ -1,5 +1,6 @@
 import { app } from '../../../../app/js/server.js'
 import { mongoClient } from '../../../../app/js/mongodb.js'
+import './MongoHelper.js'
 
 let running = false
 let initPromise = null

services/real-time/docker-compose.yml

@@ -39,7 +39,7 @@ services:
       HISTORY_REDIS_HOST: redis_test
       QUEUES_REDIS_HOST: redis_test
       ANALYTICS_QUEUES_REDIS_HOST: redis_test
-      MONGO_HOST: mongo
+      MONGO_CONNECTION_STRING: mongodb://mongo/test-overleaf
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}
       LOG_LEVEL: ${LOG_LEVEL:-}

services/web/.dockerignore

@@ -1 +0,0 @@
-.gitignore

services/web/.eslintrc.js

@@ -286,10 +286,8 @@ module.exports = {
         'scripts/add_subscription_members_csv.mjs',
         'scripts/analytics/helpers/GoogleBigQueryHelper.mjs',
         'scripts/attach_dangling_comments_to_doc.mjs',
-        'scripts/back_fill_doc_rev.mjs',
         'scripts/backfill_mixpanel_user_properties.mjs',
         'scripts/backfill_project_image_name.mjs',
-        'scripts/backfill_project_invites_token_hmac.mjs',
         'scripts/backfill_user_properties.mjs',
         'scripts/backfill_users_sso_attribute.mjs',
         'scripts/bench_bcrypt.mjs',

services/web/.jenkinsIncludeFile

@@ -20,3 +20,4 @@ package-lock.json
 package.json
 patches/**
 services/web/**
+tools/migrations/**

services/web/Dockerfile

@@ -39,6 +39,7 @@ COPY libraries/settings/package.json /overleaf/libraries/settings/package.json
 COPY libraries/stream-utils/package.json /overleaf/libraries/stream-utils/package.json
 COPY libraries/validation-tools/package.json /overleaf/libraries/validation-tools/package.json
 COPY services/web/package.json /overleaf/services/web/package.json
+COPY tools/migrations/package.json /overleaf/tools/migrations/package.json
 COPY patches/ /overleaf/patches/
 
 RUN cd /overleaf && NODE_ENV=production npm ci --quiet
@@ -72,6 +73,7 @@ COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
 COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/web/ /overleaf/services/web/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 # Build the latex parser
 RUN cd /overleaf/services/web && npm run 'lezer-latex:generate'
@@ -108,6 +110,7 @@ COPY libraries/settings/ /overleaf/libraries/settings/
 COPY libraries/stream-utils/ /overleaf/libraries/stream-utils/
 COPY libraries/validation-tools/ /overleaf/libraries/validation-tools/
 COPY services/web/ /overleaf/services/web/
+COPY tools/migrations/ /overleaf/tools/migrations/
 
 # Omit Server Pro/CE specific scripts from SaaS image
 RUN rm /overleaf/services/web/modules/server-ce-scripts -rf

services/web/Makefile

@@ -146,7 +146,7 @@ test_unit_module: mongo_migrations_for_tests
 	$(MAKE) modules/$(MODULE_NAME)/test_unit
 
 mongo_migrations_for_tests:
-	$(DOCKER_COMPOSE) run --rm test_unit npm run migrations -- migrate -t saas
+	$(DOCKER_COMPOSE) run --rm --workdir /overleaf/tools/migrations test_unit npm run migrations -- migrate -t saas
 
 #
 # Frontend tests

services/web/buildscript.txt

@@ -1,5 +1,5 @@
 web
---dependencies=
+--dependencies=mongo
 --docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
 --env-add=
 --env-pass-through=

services/web/docker-compose.yml

@@ -10,6 +10,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     working_dir: /overleaf/services/web
     env_file: docker-compose.common.env
     environment:
@@ -37,6 +38,7 @@ services:
       - ../../node_modules:/overleaf/node_modules
       - ../../libraries:/overleaf/libraries
       - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
+      - ../../tools/migrations:/overleaf/tools/migrations
     user: node
     working_dir: /overleaf/services/web
     env_file: docker-compose.common.env

services/web/migrations/20190730093801_script_example.mjs

@@ -1,28 +0,0 @@
-/* eslint-disable no-unused-vars */
-
-/*
- * Example migration for a script:
- *
- * This migration demonstrates how to run a script. In this case, the example
- * script will print "hello world" if there are no users in the users collection
- * or "hello <name>", when User.findOne() finds something.
- */
-
-import runScript from '../scripts/example/script_for_migration.mjs'
-
-const tags = []
-
-const migrate = async client => {
-  const { db } = client
-  await runScript()
-}
-
-const rollback = async client => {
-  const { db } = client
-}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}

services/web/migrations/20210727150530_ce_sp_backfill_deleted_docs.mjs

@@ -1,19 +0,0 @@
-import runScript from '../scripts/back_fill_doc_name_for_deleted_docs.mjs'
-
-const tags = ['server-ce', 'server-pro', 'saas']
-
-const migrate = async client => {
-  const options = {
-    performCleanup: true,
-    letUserDoubleCheckInputsFor: 10,
-  }
-  await runScript(options)
-}
-
-const rollback = async client => {}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}

services/web/migrations/20220913125500_migrate_auditLog_to_collections.mjs

@@ -1,20 +0,0 @@
-import runScript from '../scripts/migrate_audit_logs.mjs'
-
-const tags = ['server-ce', 'server-pro', 'saas']
-
-const migrate = async () => {
-  const options = {
-    letUserDoubleCheckInputsFor: 10,
-    writeConcurrency: 5,
-    dryRun: false,
-  }
-  await runScript(options)
-}
-
-const rollback = async () => {}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}

services/web/migrations/20230817081910_back_fill_gitBridge_feature_server_pro.mjs

@@ -1,19 +0,0 @@
-const tags = ['server-ce', 'server-pro']
-
-const migrate = async () => {
-  // Run-time import as SaaS does not ship with the server-ce-scripts module
-  const { default: runScript } = await import(
-    '../modules/server-ce-scripts/scripts/upgrade-user-features.mjs'
-  )
-  await runScript(false, {
-    gitBridge: 1,
-  })
-}
-
-const rollback = async () => {}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}

services/web/migrations/20240524135408_add_token_hmac_project_invite_tokens.mjs

@@ -1,30 +0,0 @@
-/* eslint-disable no-unused-vars */
-
-import Helpers from './lib/helpers.mjs'
-import runScript from '../scripts/backfill_project_invites_token_hmac.mjs'
-
-const tags = ['server-ce', 'server-pro', 'saas']
-
-const index = {
-  key: {
-    tokenHmac: 1,
-  },
-  name: 'tokenHmac_1',
-}
-
-const migrate = async client => {
-  const { db } = client
-  await Helpers.addIndexesToCollection(db.projectInvites, [index])
-  await runScript(false)
-}
-
-const rollback = async client => {
-  const { db } = client
-  await Helpers.dropIndexesFromCollection(db.projectInvites, [index])
-}
-
-export default {
-  tags,
-  migrate,
-  rollback,
-}

services/web/package.json

@@ -40,7 +40,6 @@
     "type-check": "tsc --noEmit",
     "type-check:backend": "tsc -p tsconfig.backend.json --noEmit",
     "extract-translations": "i18next-scanner",
-    "migrations": "MONGO_SOCKET_TIMEOUT=0 east --es-modules",
     "convert-themes": "node frontend/js/features/source-editor/themes/scripts/convert.js",
     "cypress:open-ct": "OVERLEAF_CONFIG=$PWD/config/settings.webpack.js cypress open --component",
     "cypress:run-ct": "OVERLEAF_CONFIG=$PWD/config/settings.webpack.js cypress run --component --browser chrome",
@@ -91,6 +90,7 @@
     "@overleaf/fetch-utils": "*",
     "@overleaf/logger": "*",
     "@overleaf/metrics": "*",
+    "@overleaf/migrations": "*",
     "@overleaf/mongo-utils": "*",
     "@overleaf/o-error": "*",
     "@overleaf/object-persistor": "*",
@@ -126,7 +126,6 @@
     "csurf": "^1.11.0",
     "csv": "^6.2.5",
     "dateformat": "1.0.4-1.2.3",
-    "east": "^2.0.2",
     "ejs": "^3.1.10",
     "email-addresses": "^5.0.0",
     "eventsource-parser": "^1.1.2",
@@ -194,6 +193,7 @@
     "zod-validation-error": "^4.0.1"
   },
   "devDependencies": {
+    "5to6-codemod": "^1.8.0",
     "@babel/cli": "^7.27.0",
     "@babel/core": "^7.26.10",
     "@babel/plugin-proposal-decorators": "^7.27.0",
@@ -278,7 +278,6 @@
     "@writefull/core": "^1.27.27",
     "@writefull/ui": "^1.27.27",
     "@writefull/utils": "^1.27.27",
-    "5to6-codemod": "^1.8.0",
     "abort-controller": "^3.0.0",
     "acorn": "^7.1.1",
     "acorn-walk": "^7.1.1",
@@ -354,8 +353,8 @@
     "mock-fs": "^5.1.2",
     "nock": "^13.5.6",
     "nvd3": "^1.8.6",
-    "path-browserify": "^1.0.1",
     "p-reflect": "^3.1.0",
+    "path-browserify": "^1.0.1",
     "pdfjs-dist": "5.1.91",
     "pirates": "^4.0.1",
     "postcss": "^8.4.31",

services/web/scripts/back_fill_doc_name_for_deleted_docs.mjs

@@ -1,97 +0,0 @@
-import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
-import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
-import { db } from '../app/src/infrastructure/mongodb.js'
-import { fileURLToPath } from 'node:url'
-import _ from 'lodash'
-import { scriptRunner } from './lib/ScriptRunner.mjs'
-
-const sleep = promisify(setTimeout)
-
-async function main(options, trackProgress) {
-  if (!options) {
-    options = {}
-  }
-  _.defaults(options, {
-    writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
-    performCleanup: process.argv.pop() === '--perform-cleanup',
-    letUserDoubleCheckInputsFor: parseInt(
-      process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
-      10
-    ),
-  })
-
-  await letUserDoubleCheckInputs(options)
-
-  await batchedUpdate(
-    db.projects,
-    // array is not empty ~ array has one item
-    { 'deletedDocs.0': { $exists: true } },
-    async projects => {
-      await processBatch(projects, options)
-    },
-    { _id: 1, deletedDocs: 1 },
-    undefined,
-    { trackProgress }
-  )
-}
-
-async function processBatch(projects, options) {
-  await promiseMapWithLimit(
-    options.writeConcurrency,
-    projects,
-    async project => {
-      await processProject(project, options)
-    }
-  )
-}
-
-async function processProject(project, options) {
-  for (const doc of project.deletedDocs) {
-    await backFillDoc(doc)
-  }
-  if (options.performCleanup) {
-    await cleanupProject(project)
-  }
-}
-
-async function backFillDoc(doc) {
-  const { name, deletedAt } = doc
-  await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } })
-}
-
-async function cleanupProject(project) {
-  await db.projects.updateOne(
-    { _id: project._id },
-    { $set: { deletedDocs: [] } }
-  )
-}
-
-async function letUserDoubleCheckInputs(options) {
-  if (options.performCleanup) {
-    console.error('BACK FILLING AND PERFORMING CLEANUP')
-  } else {
-    console.error(
-      'BACK FILLING ONLY - You will need to rerun with --perform-cleanup'
-    )
-  }
-  console.error(
-    'Waiting for you to double check inputs for',
-    options.letUserDoubleCheckInputsFor,
-    'ms'
-  )
-  await sleep(options.letUserDoubleCheckInputsFor)
-}
-
-export default main
-
-if (fileURLToPath(import.meta.url) === process.argv[1]) {
-  try {
-    await scriptRunner(
-      async trackProgress => await main(undefined, trackProgress)
-    )
-    process.exit(0)
-  } catch (error) {
-    console.error({ error })
-    process.exit(1)
-  }
-}

services/web/scripts/backfill_project_invites_token_hmac.mjs

@@ -1,79 +0,0 @@
-import { db } from '../app/src/infrastructure/mongodb.js'
-import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
-import minimist from 'minimist'
-import CollaboratorsInviteHelper from '../app/src/Features/Collaborators/CollaboratorsInviteHelper.js'
-import { fileURLToPath } from 'node:url'
-
-const argv = minimist(process.argv.slice(2), {
-  boolean: ['dry-run', 'help'],
-  default: {
-    'dry-run': true,
-  },
-})
-
-const DRY_RUN = argv['dry-run']
-
-async function addTokenHmacField(DRY_RUN) {
-  const query = { tokenHmac: { $exists: false } }
-
-  await batchedUpdate(
-    db.projectInvites,
-    query,
-    async invites => {
-      for (const invite of invites) {
-        console.log(
-          `=> Missing "tokenHmac" token in invitation: ${invite._id.toString()}`
-        )
-
-        if (DRY_RUN) {
-          console.log(
-            `=> DRY RUN - would add "tokenHmac" token to invitation ${invite._id.toString()}`
-          )
-          continue
-        }
-
-        const tokenHmac = CollaboratorsInviteHelper.hashInviteToken(
-          invite.token
-        )
-
-        await db.projectInvites.updateOne(
-          { _id: invite._id },
-          { $set: { tokenHmac } }
-        )
-
-        console.log(
-          `=> Added "tokenHmac" token to invitation ${invite._id.toString()}`
-        )
-      }
-    },
-    { token: 1 }
-  )
-}
-
-async function main(DRY_RUN) {
-  await addTokenHmacField(DRY_RUN)
-}
-
-export default main
-
-if (fileURLToPath(import.meta.url) === process.argv[1]) {
-  if (argv.help || argv._.length > 1) {
-    console.error(`Usage: node scripts/backfill_project_invites_token_hmac.mjs
-      Adds a "tokenHmac" field (which is a hashed version of the token) to each project invite record.
-
-      Options:
-          --dry-run         finds invitations without HMAC token but does not do any updates
-    `)
-
-    process.exit(1)
-  }
-
-  try {
-    await main(DRY_RUN)
-    console.error('Done')
-    process.exit(0)
-  } catch (error) {
-    console.error(error)
-    process.exit(1)
-  }
-}

services/web/scripts/mark_migration.mjs

@@ -1,4 +1,4 @@
-import Adapter from '../migrations/lib/adapter.mjs'
+import Adapter from '../../../tools/migrations/lib/adapter.mjs'
 import { promises as fs } from 'node:fs'
 import { join, dirname } from 'node:path'
 import { fileURLToPath } from 'node:url'

services/web/scripts/migrate_audit_logs.mjs

@@ -1,169 +0,0 @@
-import { batchedUpdate } from '@overleaf/mongo-utils/batchedUpdate.js'
-import { promiseMapWithLimit, promisify } from '@overleaf/promise-utils'
-import { db, ObjectId } from '../app/src/infrastructure/mongodb.js'
-import _ from 'lodash'
-import { fileURLToPath } from 'node:url'
-import { scriptRunner } from './lib/ScriptRunner.mjs'
-
-const sleep = promisify(setTimeout)
-
-async function main(options, trackProgress) {
-  if (!options) {
-    options = {}
-  }
-  _.defaults(options, {
-    dryRun: process.env.DRY_RUN !== 'false',
-    projectId: process.env.PROJECT_ID,
-    userId: process.env.USER_ID,
-    skipUsersMigration: process.env.SKIP_USERS_MIGRATION === 'true',
-    writeConcurrency: parseInt(process.env.WRITE_CONCURRENCY, 10) || 10,
-    letUserDoubleCheckInputsFor: parseInt(
-      process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
-      10
-    ),
-  })
-
-  await letUserDoubleCheckInputs(options)
-
-  if (options.projectId) {
-    console.log('migrating projectId=' + options.projectId)
-    const project = await db.projects.findOne(
-      { _id: new ObjectId(options.projectId) },
-      { _id: 1, auditLog: 1 }
-    )
-    if (!project || !project.auditLog) {
-      console.error('unable to process project', project)
-      return
-    }
-    await processProjectsBatch([project], options)
-  } else if (options.userId) {
-    console.log('migrating userId=' + options.userId)
-    const user = await db.users.findOne(
-      { _id: new ObjectId(options.userId) },
-      { _id: 1, auditLog: 1 }
-    )
-    if (!user || !user.auditLog) {
-      console.error('unable to process user', user)
-      return
-    }
-    await processUsersBatch([user], options)
-  } else {
-    if (!options.skipUsersMigration) {
-      await batchedUpdate(
-        db.users,
-        { auditLog: { $exists: true } },
-        async users => {
-          await processUsersBatch(users, options)
-        },
-        { _id: 1, auditLog: 1 },
-        undefined,
-        { trackProgress }
-      )
-    }
-
-    // most projects are processed after its owner has been processed, but only those
-    // users with an existing `auditLog` have been taken into consideration, leaving
-    // some projects orphan. This batched update processes all remaining projects.
-    await batchedUpdate(
-      db.projects,
-      { auditLog: { $exists: true } },
-      async projects => {
-        await processProjectsBatch(projects, options)
-      },
-      { _id: 1, auditLog: 1 },
-      undefined,
-      { trackProgress }
-    )
-  }
-}
-
-async function processUsersBatch(users, options) {
-  if (!users || users.length <= 0) {
-    return
-  }
-
-  const entries = users
-    .map(user => user.auditLog.map(log => ({ ...log, userId: user._id })))
-    .flat()
-
-  if (!options.dryRun && entries?.length > 0) {
-    await db.userAuditLogEntries.insertMany(entries)
-  }
-
-  if (!options.dryRun) {
-    const userIds = users.map(user => user._id)
-    await db.users.updateMany(
-      { _id: { $in: userIds } },
-      { $unset: { auditLog: 1 } }
-    )
-  }
-
-  await promiseMapWithLimit(options.writeConcurrency, users, async user => {
-    const projects = await db.projects
-      .find(
-        { owner_ref: user._id, auditLog: { $exists: true } },
-        { _id: 1, auditLog: 1 }
-      )
-      .toArray()
-    await processProjectsBatch(projects, options)
-  })
-}
-
-async function processProjectsBatch(projects, options) {
-  if (!projects || projects.length <= 0) {
-    return
-  }
-
-  const entries = projects
-    .map(project =>
-      project.auditLog.map(log => ({ ...log, projectId: project._id }))
-    )
-    .flat()
-
-  if (!options.dryRun && entries?.length > 0) {
-    await db.projectAuditLogEntries.insertMany(entries)
-  }
-
-  if (!options.dryRun) {
-    const projectIds = projects.map(project => project._id)
-    await db.projects.updateMany(
-      { _id: { $in: projectIds } },
-      { $unset: { auditLog: 1 } }
-    )
-  }
-}
-
-async function letUserDoubleCheckInputs(options) {
-  const allOptions = {
-    ...options,
-    // batchedUpdate() environment variables
-    BATCH_DESCENDING: process.env.BATCH_DESCENDING,
-    BATCH_SIZE: process.env.BATCH_SIZE,
-    VERBOSE_LOGGING: process.env.VERBOSE_LOGGING,
-    BATCH_LAST_ID: process.env.BATCH_LAST_ID,
-    BATCH_RANGE_END: process.env.BATCH_RANGE_END,
-    SKIP_USERS_MIGRATION: process.env.SKIP_USERS_MIGRATION,
-  }
-  console.error('Options:', JSON.stringify(allOptions, null, 2))
-  console.error(
-    'Waiting for you to double check inputs for',
-    options.letUserDoubleCheckInputsFor,
-    'ms'
-  )
-  await sleep(options.letUserDoubleCheckInputsFor)
-}
-
-export default main
-
-if (fileURLToPath(import.meta.url) === process.argv[1]) {
-  try {
-    await scriptRunner(
-      async trackProgress => await main(undefined, trackProgress)
-    )
-    console.log('Done.')
-    process.exit(0)
-  } catch (error) {
-    console.error({ error })
-    process.exit(1)
-  }
-}

services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.mjs

@@ -63,14 +63,11 @@ describe('BackFillDocNameForDeletedDocs', function () {
     await setDeletedDocs(projectId2, deletedDocs2)
   })
 
-  async function runScript(args = []) {
+  async function runScript() {
     let result
     try {
       result = await promisify(exec)(
-        ['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1']
-          .concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs.mjs'])
-          .concat(args)
-          .join(' ')
+        'cd ../../tools/migrations && east migrate -t saas --force 20210727150530_ce_sp_backfill_deleted_docs'
       )
     } catch (error) {
       // dump details like exit code, stdErr and stdOut
@@ -95,20 +92,9 @@ describe('BackFillDocNameForDeletedDocs', function () {
     })
   }
 
-  describe('back fill only', function () {
-    beforeEach('run script', runScript)
-
-    checkDocsBackFilled()
-
-    it('should leave the deletedDocs as is', async function () {
-      expect(await getDeletedDocs(projectId1)).to.deep.equal(deletedDocs1)
-      expect(await getDeletedDocs(projectId2)).to.deep.equal(deletedDocs2)
-    })
-  })
-
   describe('back fill and cleanup', function () {
     beforeEach('run script with cleanup flag', async function () {
-      await runScript(['--perform-cleanup'])
+      await runScript()
     })
 
     checkDocsBackFilled()

services/web/test/acceptance/src/BackFillDocRevTests.mjs

@@ -17,16 +17,11 @@ describe('BackFillDocRevTests', function () {
     ])
   })
 
-  async function runScript(dryRun) {
+  async function runScript() {
     let result
     try {
       result = await promisify(exec)(
-        [
-          'VERBOSE_LOGGING=true',
-          'node',
-          'scripts/back_fill_doc_rev.mjs',
-          dryRun,
-        ].join(' ')
+        'cd ../../tools/migrations && VERBOSE_LOGGING=true east migrate -t server-ce --force 20230315170739_back_fill_doc_rev'
       )
     } catch (error) {
       // dump details like exit code, stdErr and stdOut
@@ -47,24 +42,9 @@ describe('BackFillDocRevTests', function () {
     )
   }
 
-  describe('dry-run=true', function () {
-    beforeEach('run script', async function () {
-      await runScript('--dry-run=true')
-    })
-
-    it('should not back fill the rev', async function () {
-      const docs = await db.docs.find({}, { $sort: { _id: 1 } }).toArray()
-      expect(docs).to.deep.equal([
-        { _id: docId1, deleted: true },
-        { _id: docId2 },
-        { _id: docId3, rev: 42 },
-      ])
-    })
-  })
-
   describe('dry-run=false', function () {
     beforeEach('run script', async function () {
-      await runScript('--dry-run=false')
+      await runScript()
     })
 
     it('should back fill the rev', async function () {

services/web/test/acceptance/src/BackFillDummyDocMetaTests.mjs

@@ -7,8 +7,6 @@ import { db, ObjectId } from '../../../app/src/infrastructure/mongodb.js'
 
 const DUMMY_NAME = 'unknown.tex'
 const DUMMY_TIME = new Date('2021-04-12T00:00:00.000Z')
-const ONE_DAY_IN_S = 60 * 60 * 24
-const BATCH_SIZE = 3
 
 function getObjectIdFromDate(date) {
   const seconds = new Date(date).getTime() / 1000
@@ -18,7 +16,6 @@ function getObjectIdFromDate(date) {
 describe('BackFillDummyDocMeta', function () {
   let docIds
   let projectIds
-  let stopAtSeconds
   beforeEach('create docs', async function () {
     docIds = []
     docIds[0] = getObjectIdFromDate('2021-04-01T00:00:00.000Z')
@@ -48,11 +45,15 @@ describe('BackFillDummyDocMeta', function () {
     // two docs in the same project
     projectIds[10] = projectIds[9]
     projectIds[11] = projectIds[4]
-
-    stopAtSeconds = new Date('2021-04-17T00:00:00.000Z').getTime() / 1000
   })
   const now = new Date()
   beforeEach('insert doc stubs into docs collection', async function () {
+    // don't look here, just drop duplicates from the list of projectIds :)
+    await db.projects.insertMany(
+      Array.from(new Set(projectIds.map(id => id.toString()))).map(_id => ({
+        _id: new ObjectId(_id),
+      }))
+    )
     await db.docs.insertMany([
       // incomplete, without deletedDocs context
       { _id: docIds[0], project_id: projectIds[0], deleted: true },
@@ -104,33 +105,18 @@ describe('BackFillDummyDocMeta', function () {
     ])
   })
 
-  let options
-  async function runScript(dryRun) {
-    options = {
-      BATCH_SIZE,
-      CACHE_SIZE: 100,
-      DRY_RUN: dryRun,
-      FIRST_PROJECT_ID: projectIds[0].toString(),
-      INCREMENT_BY_S: ONE_DAY_IN_S,
-      STOP_AT_S: stopAtSeconds,
-      // start right away
-      LET_USER_DOUBLE_CHECK_INPUTS_FOR: 1,
-    }
+  async function runScript() {
     let result
     try {
       result = await promisify(exec)(
-        Object.entries(options)
-          .map(([key, value]) => `${key}=${value}`)
-          .concat(['node', 'scripts/back_fill_dummy_doc_meta.mjs'])
-          .join(' ')
+        'cd ../../tools/migrations && east migrate -t saas --force 20210728115327_ce_sp_backfill_dummy_doc_meta'
       )
     } catch (error) {
       // dump details like exit code, stdErr and stdOut
       logger.error({ error }, 'script failed')
       throw error
     }
-    let { stderr: stdErr, stdout: stdOut } = result
-    stdErr = stdErr.split('\n')
+    let { stdout: stdOut } = result
     stdOut = stdOut.split('\n').filter(filterOutput)
 
     expect(stdOut.filter(filterOutput)).to.include.members([
@@ -145,42 +131,8 @@ describe('BackFillDummyDocMeta', function () {
       `Orphaned deleted doc ${docIds[9]} (no deletedProjects entry)`,
       `Orphaned deleted doc ${docIds[10]} (no deletedProjects entry)`,
     ])
-    expect(stdErr.filter(filterOutput)).to.include.members([
-      `Processed 9 until ${projectIds[9]}`,
-      'Done.',
-    ])
   }
 
-  describe('DRY_RUN=true', function () {
-    beforeEach('run script', async function () {
-      await runScript(true)
-    })
-
-    it('should leave docs as is', async function () {
-      const docs = await db.docs.find({}).toArray()
-      expect(docs).to.deep.equal([
-        { _id: docIds[0], project_id: projectIds[0], deleted: true },
-        { _id: docIds[1], project_id: projectIds[1], deleted: true },
-        { _id: docIds[2], project_id: projectIds[2], deleted: true },
-        { _id: docIds[3], project_id: projectIds[3], deleted: true },
-        { _id: docIds[4], project_id: projectIds[4], deleted: true },
-        {
-          _id: docIds[5],
-          project_id: projectIds[5],
-          deleted: true,
-          name: 'foo.tex',
-          deletedAt: now,
-        },
-        { _id: docIds[6], project_id: projectIds[6] },
-        { _id: docIds[7], project_id: projectIds[7], deleted: true },
-        { _id: docIds[8], project_id: projectIds[8], deleted: true },
-        { _id: docIds[9], project_id: projectIds[9], deleted: true },
-        { _id: docIds[10], project_id: projectIds[10], deleted: true },
-        { _id: docIds[11], project_id: projectIds[11], deleted: true },
-      ])
-    })
-  })
-
   describe('DRY_RUN=false', function () {
     beforeEach('run script', async function () {
       await runScript(false)

services/web/test/acceptance/src/ConvertArchivedState.mjs

@@ -120,7 +120,7 @@ describe('ConvertArchivedState', function () {
 
   beforeEach(function (done) {
     exec(
-      'east migrate --tag server-ce --force 20221111111111_ce_sp_convert_archived_state',
+      'cd ../../tools/migrations && east migrate --tag server-ce --force 20221111111111_ce_sp_convert_archived_state',
       error => {
         if (error) {
           return done(error)

services/web/test/acceptance/src/ConvertEmailConfirmedAtToDates.js

@@ -28,7 +28,7 @@ describe('ConvertEmailConfirmedAtToDates', function () {
 
   beforeEach('run migration', function (done) {
     exec(
-      'east migrate -t saas --force 20210726083523_convert_confirmedAt_strings_to_dates',
+      'cd ../../tools/migrations && east migrate -t saas --force 20210726083523_convert_confirmedAt_strings_to_dates',
       done
     )
   })

services/web/test/acceptance/src/ConvertSplitTestAssignedAtToDates.js

@@ -65,7 +65,7 @@ describe('ConvertSplitTestAssignedAtToDates', function () {
 
   beforeEach('run migration', function (done) {
     exec(
-      'east migrate -t saas --force 20210726083523_convert_split_tests_assigned_at_strings_to_dates',
+      'cd ../../tools/migrations && east migrate -t saas --force 20210726083523_convert_split_tests_assigned_at_strings_to_dates',
       done
     )
   })

services/web/test/acceptance/src/RemoveDeletedUsersFromTokenAccessRefsTests.mjs

@@ -22,9 +22,8 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
   const projectId3 = new ObjectId('65d726e807c024c8db43be24')
   const projectId4 = new ObjectId('65d726e807c024c8db43be25')
 
-  let insertedProjects
   beforeEach('insert projects', async function () {
-    insertedProjects = await db.projects.insertMany([
+    await db.projects.insertMany([
       {
         _id: projectId1,
         tokenAccessReadAndWrite_refs: [userId1],
@@ -47,17 +46,11 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
 
   let stdOut
 
-  const runScript = async (dryRun, projectsList) => {
+  const runScript = async () => {
     let result
     try {
       result = await promisify(exec)(
-        [
-          'VERBOSE_LOGGING=true',
-          'node',
-          'scripts/remove_deleted_users_from_token_access_refs.mjs',
-          dryRun,
-          projectsList,
-        ].join(' ')
+        'cd ../../tools/migrations && east migrate -t saas --force 20240220130452_remove_deleted_users_from_token_access_refs'
       )
     } catch (error) {
       // dump details like exit code, stdErr and stdOut
@@ -70,91 +63,6 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
     expect(stdOut).to.match(new RegExp(`User ids count: ${insertedUsersCount}`))
   }
 
-  describe('dry-run=true', function () {
-    beforeEach('run script', async function () {
-      await runScript('--dry-run=true')
-      expect(stdOut).to.match(/doing dry run/i)
-    })
-
-    it('should show current user id to be removed', function () {
-      expect(stdOut).to.match(
-        new RegExp(
-          `Found deleted user id: ${userId2.toString()} in project: ${projectId2.toString()}`
-        )
-      )
-      expect(stdOut).to.match(
-        new RegExp(
-          `DRY RUN - would remove deleted ${userId2.toString()} from all projects \\(found in project ${projectId2.toString()}\\)`
-        )
-      )
-      expect(stdOut).to.match(
-        new RegExp(
-          `Found deleted user id: ${userId3.toString()} in project: ${projectId3.toString()}`
-        )
-      )
-      expect(stdOut).to.match(
-        new RegExp(
-          `DRY RUN - would remove deleted ${userId3.toString()} from all projects \\(found in project ${projectId3.toString()}\\)`
-        )
-      )
-    })
-
-    it('should show projects with non-existing token access fields', function () {
-      expect(stdOut)
-        .to.match(
-          new RegExp(
-            `DRY RUN - would fix non-existing token access fields in project ${projectId3.toString()}`
-          )
-        )
-        .and.match(
-          new RegExp(
-            `DRY RUN - would fix non-existing token access fields in project ${projectId4.toString()}`
-          )
-        )
-    })
-
-    it('should show the user ids (and their count) to be deleted', function () {
-      expect(stdOut).to.match(
-        new RegExp(
-          `DRY RUN - would delete user ids \\(2\\)\\n${userId2.toString()}\\n${userId3.toString()}`
-        )
-      )
-    })
-
-    it('should show the project ids (and their count) that needs fixing', function () {
-      expect(stdOut).to.match(
-        new RegExp(
-          `Projects with deleted user ids \\(2\\)\\n${projectId2.toString()}\\n${projectId3.toString()}`
-        )
-      )
-    })
-
-    it('should not fix the token access fields of projects', async function () {
-      const projects = await db.projects
-        .find({}, { $sort: { _id: 1 } })
-        .toArray()
-      expect(projects).to.deep.equal([
-        {
-          _id: projectId1,
-          tokenAccessReadAndWrite_refs: [userId1],
-          tokenAccessReadOnly_refs: [],
-        },
-        {
-          _id: projectId2,
-          tokenAccessReadAndWrite_refs: [userId2],
-          tokenAccessReadOnly_refs: [],
-        },
-        {
-          _id: projectId3,
-          tokenAccessReadAndWrite_refs: [userId3],
-        },
-        {
-          _id: projectId4,
-        },
-      ])
-    })
-  })
-
   describe('dry-run=false', function () {
     beforeEach('run script', async function () {
       await runScript('--dry-run=false')
@@ -237,20 +145,4 @@ describe('RemoveDeletedUsersFromTokenAccessRefsTests', function () {
       ])
     })
   })
-
-  describe('projects=projectId2', function () {
-    beforeEach('run script', async function () {
-      const projectId2 = insertedProjects.insertedIds[1]
-      await runScript('--dry-run=false', `--projects=${projectId2.toString()}`)
-    })
-
-    it('should fix only the projects provided', async function () {
-      const [project1, project2, project3] = await db.projects
-        .find({}, { $sort: { _id: 1 } })
-        .toArray()
-      expect(project1.tokenAccessReadAndWrite_refs.length).to.be.gt(0)
-      expect(project2.tokenAccessReadAndWrite_refs.length).to.eq(0) // deleted user removed
-      expect(project3.tokenAccessReadAndWrite_refs.length).to.be.gt(0)
-    })
-  })
 })

services/web/test/acceptance/src/helpers/MongoHelper.mjs

@@ -1,10 +1,11 @@
-import { execFile } from 'node:child_process'
+import { exec } from 'node:child_process'
 import {
   connectionPromise,
   cleanupTestDatabase,
   dropTestDatabase,
 } from '../../../../app/src/infrastructure/mongodb.js'
 import Settings from '@overleaf/settings'
+import { promisify } from 'node:util'
 
 const DEFAULT_ENV = 'saas'
 
@@ -17,21 +18,12 @@ export default {
       before('drop test database', dropTestDatabase)
     }
 
-    before('run migrations', function (done) {
-      const args = [
-        'run',
-        'migrations',
-        '--',
-        'migrate',
-        '-t',
-        Settings.env || DEFAULT_ENV,
-      ]
-      execFile('npm', args, (error, stdout, stderr) => {
-        if (error) {
-          throw error
-        }
-        done()
-      })
+    before('run migrations', async function () {
+      this.timeout(60_000)
+
+      await promisify(exec)(
+        `cd ../../tools/migrations && npm run migrations -- migrate -t ${Settings.env || DEFAULT_ENV}`
+      )
     })
 
     afterEach('purge mongo data', cleanupTestDatabase)

services/web/.eastrc → tools/migrations/.eastrc

@@ -1,5 +1,6 @@
 {
-  "adapter": "./migrations/lib/adapter.mjs",
+  "adapter": "./lib/adapter.mjs",
+  "dir": ".",
   "migrationNumberFormat": "dateTime",
   "migrationExtension": "mjs"
 }

tools/migrations/.jenkinsIncludeFile

@@ -0,0 +1,6 @@
+tools/migrations/**
+
+package.json
+package-lock.json
+
+Makefile