[server-ce] test filestore migration with upgrade from version 1.x (#27342)

* [server-ce] test filestore migration with upgrade from version 1.x

* [server-ce] tests: drop verbose logs from host-admin in CI

* [server-ce] tests: fix flag following rebase

GitOrigin-RevId: dc00127fc76f87ee3eb5071fd430f4917e8123ff

server-ce/test/Makefile

@@ -26,7 +26,7 @@ test-e2e:
 
 test-e2e-open:
 	docker compose up -d host-admin
-	docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open
+	docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open host-admin
 
 clean:
 	docker compose down --volumes --timeout 0

server-ce/test/filestore-migration.spec.ts

@@ -1,39 +1,259 @@
-import { ensureUserExists, login } from './helpers/login'
+import { DEFAULT_PASSWORD, login } from './helpers/login'
 import {
-  createProject,
+  expectFileExists,
   openProjectById,
   prepareFileUploadTest,
 } from './helpers/project'
 import { isExcludedBySharding, startWith } from './helpers/config'
 import { prepareWaitForNextCompileSlot } from './helpers/compile'
-import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
 import { v4 as uuid } from 'uuid'
-import { purgeFilestoreData, runScript } from './helpers/hostAdminClient'
+import {
+  purgeFilestoreData,
+  runGruntTask,
+  runScript,
+  setMongoFeatureCompatibilityVersion,
+} from './helpers/hostAdminClient'
 
-describe('filestore migration', function () {
-  if (isExcludedBySharding('CE_CUSTOM_3')) return
-  startWith({ withDataDir: true, resetData: true, vars: {} })
-  ensureUserExists({ email: 'user@example.com' })
+function activateUserVersion1x(url: string, password = DEFAULT_PASSWORD) {
+  cy.session(url, () => {
+    cy.visit(url)
+    cy.url().then(url => {
+      if (url.includes('/login')) return
+      cy.url().should('contain', '/user/password/set')
+      cy.get('input[type="password"]').type(password)
+      cy.findByRole('button', { name: 'Set new password' }).click()
+    })
+  })
+}
 
+describe('filestore migration', function () {
+  if (isExcludedBySharding('LOCAL_ONLY')) return
+  const email = 'user@example.com'
+  // Branding of env vars changed in 5.x
+  const sharelatexBrandedVars = {
+    SHARELATEX_SITE_URL: 'http://sharelatex',
+    SHARELATEX_MONGO_URL: 'mongodb://mongo/sharelatex',
+    SHARELATEX_REDIS_HOST: 'redis',
+  }
   let projectName: string
   let projectId: string
   let waitForCompileRateLimitCoolOff: (fn: () => void) => void
   const previousBinaryFiles: (() => void)[] = []
-  beforeWithReRunOnTestRetry(function () {
+
+  function avoid502() {
+    // The next step will likely restart the instance and any following
+    // requests will fail with a 502/bad gateway. Avoid this by navigating
+    // away from the editor, which will reload upon receiving a
+    // 'forceDisconnect' socket.io message.
+    cy.visit('/project')
+  }
+
+  function addNewBinaryFileAndCheckPrevious(
+    universeSelector = 'img[alt="universe.jpg"]'
+  ) {
+    before(function () {
+      login(email)
+      waitForCompileRateLimitCoolOff(() => {
+        cy.visit(`/project/${projectId}`)
+      })
+      previousBinaryFiles.push(prepareFileUploadTest(true))
+      cy.log('check binary files')
+      for (const check of previousBinaryFiles) {
+        check()
+      }
+      cy.findByRole('treeitem', { name: 'universe.jpg' }).click()
+      cy.get(universeSelector)
+        .should('be.visible')
+        .and('have.prop', 'naturalWidth')
+        .should('be.greaterThan', 0)
+
+      avoid502()
+    })
+  }
+
+  // --------------
+  // Server Pro 1.x
+  startWith({
+    pro: true,
+    resetData: true,
+    withDataDir: true,
+    vars: sharelatexBrandedVars,
+    version: '1.2.4',
+    mongoVersion: '5.0',
+  })
+
+  let activateURL: string
+  before(async function () {
+    const { stdout } = await runGruntTask({
+      task: 'user:create-admin',
+      args: ['--email', email],
+    })
+    ;[activateURL] = stdout.match(
+      /http:\/\/.+\/user\/password\/set\?passwordResetToken=\S+/
+    )!
+  })
+  before(function () {
+    activateUserVersion1x(activateURL)
+    login(email)
     projectName = `project-${uuid()}`
-    login('user@example.com')
-    createProject(projectName, { type: 'Example project' }).then(
-      id => (projectId = id)
-    )
+    cy.visit('/project')
+
+    // Legacy angular based UI uses links instead of buttons
+    cy.findByRole('link', {
+      name: /Create First Project|New Project/,
+    }).click()
+    cy.findByRole('link', { name: 'Example Project' }).click()
+    cy.findByPlaceholderText('Project Name').type(projectName)
+    cy.findByRole('button', { name: 'Create' }).click()
+    cy.url()
+      .should('match', /\/project\/[a-fA-F0-9]{24}/)
+      .then(url => (projectId = url.split('/').pop()!))
     let queueReset
     ;({ waitForCompileRateLimitCoolOff, queueReset } =
       prepareWaitForNextCompileSlot())
     queueReset()
-    previousBinaryFiles.push(prepareFileUploadTest(true))
+
+    // Create a new binary file
+    cy.get(`a[tooltip="Upload"]`).click()
+    const name = `${uuid()}.txt`
+    // Binary file detection is not sophisticated in version 1.x
+    const binName = name.replace('.txt', '.bin')
+    const content = `Test File Content ${name} \x00`
+    cy.get('input[type=file]')
+      .first()
+      .selectFile(
+        {
+          contents: Cypress.Buffer.from(content),
+          fileName: binName,
+          lastModified: Date.now(),
+        },
+        { force: true }
+      )
+    // Rename back to .txt to enable preview
+    cy.findByText(binName).click()
+    cy.findByText(binName).dblclick()
+    cy.focused().type(name + '{del}'.repeat('.bin'.length) + '{enter}')
+    // Switch back and forth
+    cy.findByText('universe.jpg').click()
+    cy.findByText(name).click()
+    cy.findByText(content)
+      .parent()
+      .parent()
+      .should('have.class', 'text-preview')
+
+    previousBinaryFiles.push(() => expectFileExists(name, true, content))
+    avoid502()
+  })
+
+  // --------------
+  // Server Pro 2.x
+  startWith({
+    pro: true,
+    withDataDir: true,
+    vars: sharelatexBrandedVars,
+    version: '2.7.1',
+    mongoVersion: '5.0',
+  })
+  before(function () {
+    // Cypress strips the Content-Length header: https://github.com/cypress-io/cypress/issues/16469
+    // Server Pro 2.x does not gracefully handle a missing value.
+    cy.intercept(
+      {
+        method: 'HEAD',
+        url: `http://sharelatex/project/${projectId}/file/*`,
+        times: previousBinaryFiles.length + 1,
+      },
+      req => {
+        req.continue(res => {
+          res.headers['Content-Length'] = '60'
+        })
+      }
+    )
+  })
+  // Server Pro 2.x does not have alt tags on images.
+  addNewBinaryFileAndCheckPrevious('img')
+
+  // ----------------------------------
+  // Server Pro 3.x + history migration
+  startWith({
+    pro: true,
+    withDataDir: true,
+    vars: sharelatexBrandedVars,
+    version: '3.5.13',
+    mongoVersion: '5.0',
+  })
+  addNewBinaryFileAndCheckPrevious() // before history migration
+  before(async function () {
+    await runScript({
+      cwd: 'services/web',
+      script: 'scripts/history/migrate_history.js',
+      args: [
+        '--force-clean',
+        '--fix-invalid-characters',
+        '--convert-large-docs-to-file',
+      ],
+      hasOverleafEnv: false,
+    })
+  })
+  before(async function () {
+    await runScript({
+      cwd: 'services/web',
+      script: 'scripts/history/clean_sl_history_data.js',
+      hasOverleafEnv: false,
+    })
+  })
+  addNewBinaryFileAndCheckPrevious() // after history migration
+
+  // ------------------------------
+  // Server Pro 4.x + mongo upgrade
+  startWith({
+    pro: true,
+    withDataDir: true,
+    vars: sharelatexBrandedVars,
+    version: '4.2.9',
+    mongoVersion: '5.0',
+  })
+  startWith({
+    pro: true,
+    withDataDir: true,
+    vars: sharelatexBrandedVars,
+    version: '4.2.9',
+    mongoVersion: '6.0',
+  })
+  before(async function () {
+    await setMongoFeatureCompatibilityVersion('6.0')
+  })
+  addNewBinaryFileAndCheckPrevious()
+
+  // ------------------------------------------
+  // Server Pro 5.x + mongo upgrade 6 -> 7 -> 8
+  startWith({
+    pro: true,
+    withDataDir: true,
+    mongoVersion: '6.0',
+  })
+  startWith({
+    pro: true,
+    withDataDir: true,
+    mongoVersion: '7.0',
+  })
+  before(async function () {
+    await setMongoFeatureCompatibilityVersion('7.0')
+  })
+  startWith({
+    pro: true,
+    withDataDir: true,
+    // implicit mongo upgrade to 8.0
+  })
+  before(async function () {
+    await setMongoFeatureCompatibilityVersion('8.0')
   })
+  addNewBinaryFileAndCheckPrevious()
 
+  // -------------------
+  // filestore-migration
   beforeEach(() => {
-    login('user@example.com')
+    login(email)
     waitForCompileRateLimitCoolOff(() => {
       openProjectById(projectId)
     })
@@ -47,9 +267,9 @@ describe('filestore migration', function () {
       }
     })
 
-    it('renders frog jpg', () => {
-      cy.findByTestId('file-tree').findByText('frog.jpg').click()
-      cy.get('[alt="frog.jpg"]')
+    it('renders universe jpg', () => {
+      cy.findByTestId('file-tree').findByText('universe.jpg').click()
+      cy.get('[alt="universe.jpg"]')
         .should('be.visible')
         .and('have.prop', 'naturalWidth')
         .should('be.greaterThan', 0)
@@ -57,12 +277,13 @@ describe('filestore migration', function () {
   }
 
   describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () {
-    startWith({ withDataDir: true, vars: {} })
+    startWith({ pro: true, withDataDir: true, vars: {} })
     checkFilesAreAccessible()
   })
 
   describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=0', function () {
     startWith({
+      pro: true,
       withDataDir: true,
       vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '0' },
     })
@@ -70,6 +291,7 @@ describe('filestore migration', function () {
 
     describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=1', function () {
       startWith({
+        pro: true,
         withDataDir: true,
         vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' },
       })
@@ -77,6 +299,7 @@ describe('filestore migration', function () {
 
       describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=2', function () {
         startWith({
+          pro: true,
           withDataDir: true,
           vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' },
         })
@@ -84,9 +307,11 @@ describe('filestore migration', function () {
           await runScript({
             cwd: 'services/history-v1',
             script: 'storage/scripts/back_fill_file_hash.mjs',
+            args: ['--all'],
           })
         })
         startWith({
+          pro: true,
           withDataDir: true,
           vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' },
         })

server-ce/test/helpers/config.ts

@@ -6,6 +6,7 @@ export const STARTUP_TIMEOUT =
 
 export function isExcludedBySharding(
   shard:
+    | 'LOCAL_ONLY'
     | 'CE_DEFAULT'
     | 'CE_CUSTOM_1'
     | 'CE_CUSTOM_2'
@@ -29,6 +30,7 @@ export function startWith({
   varsFn = () => ({}),
   withDataDir = false,
   resetData = false,
+  mongoVersion = '',
 }) {
   before(async function () {
     Object.assign(vars, varsFn())
@@ -38,14 +40,18 @@ export function startWith({
       vars,
       withDataDir,
       resetData,
+      mongoVersion,
     })
     if (resetData) {
+      cy.log('resetting data and sessions')
       resetCreatedUsersCache()
       resetActivateUserRateLimit()
       // no return here, always reconfigure when resetting data
     } else if (previousConfigFrontend === cfg) {
+      cy.log(`already running with ${cfg}`)
       return
     }
+    cy.log(`starting with ${cfg}`)
 
     this.timeout(STARTUP_TIMEOUT)
     const { previousConfigServer } = await reconfigure({
@@ -54,6 +60,7 @@ export function startWith({
       vars,
       withDataDir,
       resetData,
+      mongoVersion,
     })
     if (previousConfigServer !== cfg) {
       await Cypress.session.clearAllSavedSessions()

server-ce/test/helpers/hostAdminClient.ts

@@ -15,6 +15,7 @@ export async function reconfigure({
   vars = {},
   withDataDir = false,
   resetData = false,
+  mongoVersion = '',
 }): Promise<{ previousConfigServer: string }> {
   return await fetchJSON(`${hostAdminURL}/reconfigure`, {
     method: 'POST',
@@ -24,6 +25,7 @@ export async function reconfigure({
       vars,
       withDataDir,
       resetData,
+      mongoVersion,
     }),
   })
 }
@@ -63,10 +65,12 @@ export async function runScript({
   cwd,
   script,
   args = [],
+  hasOverleafEnv = true,
 }: {
   cwd: string
   script: string
   args?: string[]
+  hasOverleafEnv?: boolean
 }) {
   return await fetchJSON(`${hostAdminURL}/run/script`, {
     method: 'POST',
@@ -74,6 +78,23 @@ export async function runScript({
       cwd,
       script,
       args,
+      hasOverleafEnv,
+    }),
+  })
+}
+
+export async function runGruntTask({
+  task,
+  args = [],
+}: {
+  task: string
+  args?: string[]
+}) {
+  return await fetchJSON(`${hostAdminURL}/run/gruntTask`, {
+    method: 'POST',
+    body: JSON.stringify({
+      task,
+      args,
     }),
   })
 }
@@ -85,6 +106,18 @@ export async function getRedisKeys() {
   return stdout.split('\n')
 }
 
+export async function setMongoFeatureCompatibilityVersion(
+  mongoVersion: string
+) {
+  cy.log(`advancing mongo featureCompatibilityVersion to ${mongoVersion}`)
+  await fetchJSON(`${hostAdminURL}/mongo/setFeatureCompatibilityVersion`, {
+    method: 'POST',
+    body: JSON.stringify({
+      mongoVersion,
+    }),
+  })
+}
+
 export async function purgeFilestoreData() {
   await fetchJSON(`${hostAdminURL}/data/user_files`, {
     method: 'DELETE',

server-ce/test/helpers/login.ts

@@ -1,6 +1,6 @@
 import { runScript } from './hostAdminClient'
 
-const DEFAULT_PASSWORD = 'Passw0rd!'
+export const DEFAULT_PASSWORD = 'Passw0rd!'
 
 const createdUsers = new Set<string>()
 

server-ce/test/helpers/project.ts

@@ -217,6 +217,19 @@ export function createNewFile() {
   return fileName
 }
 
+export function expectFileExists(
+  name: string,
+  binary: boolean,
+  content: string
+) {
+  cy.findByRole('treeitem', { name }).click()
+  if (binary) {
+    cy.findByText(content).should('not.have.class', 'cm-line')
+  } else {
+    cy.findByText(content).should('have.class', 'cm-line')
+  }
+}
+
 export function prepareFileUploadTest(binary = false) {
   const name = `${uuid()}.txt`
   const content = `Test File Content ${name}${binary ? ' \x00' : ''}`
@@ -235,14 +248,7 @@ export function prepareFileUploadTest(binary = false) {
   // wait for the upload to finish
   cy.findByRole('treeitem', { name })
 
-  return function check() {
-    cy.findByRole('treeitem', { name }).click()
-    if (binary) {
-      cy.findByText(content).should('not.have.class', 'cm-line')
-    } else {
-      cy.findByText(content).should('have.class', 'cm-line')
-    }
-  }
+  return () => expectFileExists(name, binary, content)
 }
 
 export function testNewFileUpload() {

server-ce/test/host-admin.js

@@ -107,20 +107,62 @@ app.post(
         cwd: Joi.string().required(),
         script: Joi.string().required(),
         args: Joi.array().items(Joi.string()),
+        hasOverleafEnv: Joi.boolean().required(),
       },
     },
     { allowUnknown: false }
   ),
   (req, res) => {
-    const { cwd, script, args } = req.body
+    const { cwd, script, args, hasOverleafEnv } = req.body
+
+    const env = hasOverleafEnv
+      ? 'source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh'
+      : 'true'
 
     runDockerCompose(
       'exec',
       [
+        '--workdir',
+        `/overleaf/${cwd}`,
         'sharelatex',
         'bash',
         '-c',
-        `source /etc/container_environment.sh && source /etc/overleaf/env.sh || source /etc/sharelatex/env.sh && cd ${JSON.stringify(cwd)} && node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
+        `source /etc/container_environment.sh && ${env} && node ${JSON.stringify(script)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
+      ],
+      (error, stdout, stderr) => {
+        res.json({
+          error,
+          stdout,
+          stderr,
+        })
+      }
+    )
+  }
+)
+
+app.post(
+  '/run/gruntTask',
+  validate(
+    {
+      body: {
+        task: Joi.string().required(),
+        args: Joi.array().items(Joi.string()),
+      },
+    },
+    { allowUnknown: false }
+  ),
+  (req, res) => {
+    const { task, args } = req.body
+
+    runDockerCompose(
+      'exec',
+      [
+        '--workdir',
+        '/var/www/sharelatex',
+        'sharelatex',
+        'bash',
+        '-c',
+        `source /etc/container_environment.sh && grunt ${JSON.stringify(task)} ${args.map(a => JSON.stringify(a)).join(' ')}`,
       ],
       (error, stdout, stderr) => {
         res.json({
@@ -178,7 +220,13 @@ const allowedVars = Joi.object(
   )
 )
 
-function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
+function setVarsDockerCompose({
+  pro,
+  vars,
+  version,
+  withDataDir,
+  mongoVersion,
+}) {
   const cfg = readDockerComposeOverride()
 
   cfg.services.sharelatex.image = `${pro ? IMAGES.PRO : IMAGES.CE}:${version}`
@@ -198,8 +246,8 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
 
   const dataDirInContainer =
     version === 'latest' || version >= '5.0'
-      ? '/var/lib/overleaf/data'
-      : '/var/lib/sharelatex/data'
+      ? '/var/lib/overleaf'
+      : '/var/lib/sharelatex'
 
   cfg.services.sharelatex.volumes = []
   if (withDataDir) {
@@ -220,11 +268,19 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
     )
     if (!withDataDir) {
       cfg.services.sharelatex.volumes.push(
-        `${PATHS.SANDBOXED_COMPILES_HOST_DIR}:${dataDirInContainer}/compiles`
+        `${PATHS.SANDBOXED_COMPILES_HOST_DIR}:${dataDirInContainer}/data/compiles`
       )
     }
   }
 
+  if (mongoVersion) {
+    cfg.services.mongo = {
+      image: `mongo:${mongoVersion}`,
+    }
+  } else {
+    delete cfg.services.mongo
+  }
+
   writeDockerComposeOverride(cfg)
 }
 
@@ -285,6 +341,7 @@ app.post(
     {
       body: {
         pro: Joi.boolean().required(),
+        mongoVersion: Joi.string().allow('').optional(),
         version: Joi.string().required(),
         vars: allowedVars,
         withDataDir: Joi.boolean().optional(),
@@ -294,7 +351,8 @@ app.post(
     { allowUnknown: false }
   ),
   (req, res) => {
-    const { pro, version, vars, withDataDir, resetData } = req.body
+    const { pro, version, vars, withDataDir, resetData, mongoVersion } =
+      req.body
     maybeResetData(resetData, (error, stdout, stderr) => {
       if (error) return res.json({ error, stdout, stderr })
 
@@ -305,7 +363,7 @@ app.post(
       }
 
       try {
-        setVarsDockerCompose({ pro, version, vars, withDataDir })
+        setVarsDockerCompose({ pro, version, vars, withDataDir, mongoVersion })
       } catch (error) {
         return res.json({ error })
       }
@@ -323,6 +381,43 @@ app.post(
   }
 )
 
+app.post(
+  '/mongo/setFeatureCompatibilityVersion',
+  validate(
+    {
+      body: {
+        mongoVersion: Joi.string().required(),
+      },
+    },
+    { allowUnknown: false }
+  ),
+  (req, res) => {
+    const { mongoVersion } = req.body
+    const mongosh = mongoVersion > '5' ? 'mongosh' : 'mongo'
+    const params = {
+      setFeatureCompatibilityVersion: mongoVersion,
+    }
+    if (mongoVersion >= '7.0') {
+      // MongoServerError: Once you have upgraded to 7.0, you will not be able to downgrade FCV and binary version without support assistance. Please re-run this command with 'confirm: true' to acknowledge this and continue with the FCV upgrade.
+      // NOTE: 6.0 does not know about this flag. So conditionally add it.
+      // MongoServerError: BSON field 'setFeatureCompatibilityVersion.confirm' is an unknown field.
+      params.confirm = true
+    }
+    runDockerCompose(
+      'exec',
+      [
+        'mongo',
+        mongosh,
+        '--eval',
+        `db.adminCommand(${JSON.stringify(params)})`,
+      ],
+      (error, stdout, stderr) => {
+        res.json({ error, stdout, stderr })
+      }
+    )
+  }
+)
+
 app.get('/redis/keys', (req, res) => {
   runDockerCompose(
     'exec',