Merge pull request #6661 from overleaf/spd-local-tests

Move acceptance test mocks to nonstandard ports and add options for running locally

GitOrigin-RevId: bd8f70ac8d80599daccc51cfe7b90a2ad8d8c3d8

services/web/app/src/infrastructure/mongodb.js

@@ -1,5 +1,6 @@
 const Settings = require('@overleaf/settings')
 const { MongoClient, ObjectId } = require('mongodb')
+const OError = require('@overleaf/o-error')
 const { addConnectionDrainer } = require('./GracefulShutdown')
 
 if (
@@ -86,6 +87,20 @@ async function getCollectionNames() {
   return collections.map(collection => collection.collectionName)
 }
 
+async function dropTestDatabase() {
+  const internalDb = (await clientPromise).db()
+  const dbName = internalDb.databaseName
+  const env = process.env.NODE_ENV
+
+  if (dbName !== 'test-sharelatex' || env !== 'test') {
+    throw new OError(
+      `Refusing to clear database '${dbName}' in environment '${env}'`
+    )
+  }
+
+  await internalDb.dropDatabase()
+}
+
 /**
  * WARNING: Consider using a pre-populated collection from `db` to avoid typos!
  */
@@ -99,5 +114,6 @@ module.exports = {
   ObjectId,
   getCollectionNames,
   getCollectionInternal,
+  dropTestDatabase,
   waitForDb,
 }

services/web/config/settings.defaults.js

@@ -1,3 +1,4 @@
+const Path = require('path')
 const { merge } = require('@overleaf/settings/merge')
 
 let defaultFeatures, siteUrl
@@ -105,6 +106,7 @@ module.exports = {
       host: process.env.REDIS_HOST || 'localhost',
       port: process.env.REDIS_PORT || '6379',
       password: process.env.REDIS_PASSWORD || '',
+      db: process.env.REDIS_DB,
       maxRetriesPerRequest: parseInt(
         process.env.REDIS_MAX_RETRIES_PER_REQUEST || '20'
       ),
@@ -565,8 +567,8 @@ module.exports = {
     // If we ever need to write something to disk (e.g. incoming requests
     // that need processing but may be too big for memory, then write
     // them to disk here).
-    dumpFolder: './data/dumpFolder',
-    uploadFolder: './data/uploads',
+    dumpFolder: Path.resolve(__dirname, '../data/dumpFolder'),
+    uploadFolder: Path.resolve(__dirname, '../data/uploads'),
   },
 
   // Automatic Snapshots

services/web/docker-compose.ci.yml

@@ -110,4 +110,4 @@ services:
       environment:
           SAML_BASE_URL_PATH: 'http://saml/simplesaml/'
           SAML_TEST_SP_ENTITY_ID: 'sharelatex-test-saml'
-          SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:3000/saml/callback'
+          SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:23000/saml/callback'

services/web/docker-compose.common.env

@@ -1,9 +1,10 @@
 ADMIN_PRIVILEGE_AVAILABLE=true
 BCRYPT_ROUNDS=1
+WEB_PORT=23000
 REDIS_HOST=redis
 QUEUES_REDIS_HOST=redis
 ANALYTICS_QUEUES_REDIS_HOST=redis
-MONGO_URL=mongodb://mongo/sharelatex
+MONGO_URL=mongodb://mongo/test-sharelatex
 SHARELATEX_ALLOW_PUBLIC_ACCESS=true
 LINKED_URL_PROXY=http://localhost:6543
 ENABLED_LINKED_FILE_TYPES=url,project_file,project_output_file,mendeley,zotero
@@ -12,15 +13,15 @@ NODE_OPTIONS=--unhandled-rejections=strict
 LOCK_MANAGER_MAX_LOCK_WAIT_TIME=30000
 COOKIE_DOMAIN=.overleaf.test
 ADMIN_URL=http://admin.overleaf.test
-PUBLIC_URL=http://www.overleaf.test:3000
+PUBLIC_URL=http://www.overleaf.test:23000
 HTTP_TEST_HOST=www.overleaf.test
-OT_JWT_AUTH_KEY=very secret key
+OT_JWT_AUTH_KEY="very secret key"
 EXTERNAL_AUTH=none
 RECAPTCHA_ENDPOINT=http://localhost:2222/recaptcha/api/siteverify
 # Server-Pro LDAP
 SHARELATEX_LDAP_URL=ldap://ldap:389
 SHARELATEX_LDAP_SEARCH_BASE=ou=people,dc=planetexpress,dc=com
-SHARELATEX_LDAP_SEARCH_FILTER=(uid={{username}})
+SHARELATEX_LDAP_SEARCH_FILTER="(uid={{username}})"
 SHARELATEX_LDAP_BIND_DN=cn=admin,dc=planetexpress,dc=com
 SHARELATEX_LDAP_BIND_CREDENTIALS=GoodNewsEveryone
 SHARELATEX_LDAP_EMAIL_ATT=mail
@@ -31,7 +32,7 @@ SHARELATEX_LDAP_UPDATE_USER_DETAILS_ON_LOGIN=true
 SHARELATEX_SAML_ENTRYPOINT=http://saml/simplesaml/saml2/idp/SSOService.php
 SHARELATEX_SAML_CALLBACK_URL=http://saml/saml/callback
 SHARELATEX_SAML_ISSUER=sharelatex-test-saml
-SHARELATEX_SAML_IDENTITY_SERVICE_NAME=SAML Test Server
+SHARELATEX_SAML_IDENTITY_SERVICE_NAME="SAML Test Server"
 SHARELATEX_SAML_EMAIL_FIELD=email
 SHARELATEX_SAML_FIRST_NAME_FIELD=givenName
 SHARELATEX_SAML_LAST_NAME_FIELD=sn

services/web/docker-compose.yml

@@ -108,4 +108,4 @@ services:
       environment:
           SAML_BASE_URL_PATH: 'http://saml/simplesaml/'
           SAML_TEST_SP_ENTITY_ID: 'sharelatex-test-saml'
-          SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:3000/saml/callback'
+          SAML_TEST_SP_LOCATION: 'http://www.overleaf.test:23000/saml/callback'

services/web/local-dev.env

@@ -0,0 +1,19 @@
+REDIS_HOST=localhost
+QUEUES_REDIS_HOST=localhost
+PUBSUB_REDIS_HOST=localhost
+GCLOUD_2_REDIS_HOST=localhost
+QUEUES_REDIS_HOST=localhost
+POSTGRES_HOST=localhost
+MONGO_URL=mongodb://localhost/sharelatex
+SHARELATEX_LDAP_URL=ldap://localhost:22389
+SAML_BASE_URL_PATH=http://localhost:22280/simplesaml/
+SHARELATEX_SAML_ENTRYPOINT=http://localhost:22280/simplesaml/saml2/idp/SSOService.php
+SHARELATEX_SAML_CALLBACK_URL=http://www.dev-overleaf.com/saml/callback
+COOKIE_DOMAIN=.dev-overleaf.com
+PUBLIC_URL=https://www.dev-overleaf.com/
+CURRENT_IMAGE_NAME=texlive-full:2021.1
+LINKED_URL_PROXY=http://localhost:8081/proxy/secret-proxy-token
+SHARELATEX_CONFIG=$(pwd)/../../config/settings.web.js
+ENABLE_PDF_CACHING=true
+HAVE_I_BEEN_PWNED_ENABLED=true
+NODE_ENV=development

services/web/local-test.env

@@ -0,0 +1,18 @@
+CLEANUP_MONGO=true
+
+REDIS_HOST=localhost
+REDIS_DB=5
+QUEUES_REDIS_HOST=localhost
+PUBSUB_REDIS_HOST=localhost
+GCLOUD_2_REDIS_HOST=localhost
+QUEUES_REDIS_HOST=localhost
+POSTGRES_HOST=localhost
+MONGO_URL=mongodb://localhost/test-sharelatex
+SHARELATEX_LDAP_URL=ldap://localhost:22389
+SAML_BASE_URL_PATH=http://localhost:22280/simplesaml/
+SHARELATEX_SAML_ENTRYPOINT=http://localhost:22280/simplesaml/saml2/idp/SSOService.php
+SHARELATEX_SAML_CALLBACK_URL=http://localhost:22280/saml/callback
+
+COOKIE_DOMAIN=
+PUBLIC_URL=http://localhost:23000
+HTTP_TEST_HOST=localhost

services/web/modules/server-ce-scripts/test/acceptance/src/ServerCEScriptsTests.js

@@ -50,9 +50,7 @@ describe('ServerCEScripts', function () {
 
     it('should exit with code 1 on error', function () {
       try {
-        run(
-          'REDIS_HOST=localhost node modules/server-ce-scripts/scripts/check-redis'
-        )
+        run('REDIS_PORT=42 node modules/server-ce-scripts/scripts/check-redis')
       } catch (e) {
         expect(e.status).to.equal(1)
         return

services/web/package.json

@@ -36,7 +36,26 @@
     "cypress:run-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress run --component",
     "lezer-latex:generate": "node modules/source-editor/scripts/lezer-latex/generate.js",
     "lezer-latex:run": "node modules/source-editor/scripts/lezer-latex/run.mjs",
-    "routes": "bin/routes"
+    "routes": "bin/routes",
+    "routes": "bin/routes",
+    "local:nodemon": "set -a;. ../../config/dev-environment.env;. ../../config/local.env;. ../../config/local-dev.env;. ./docker-compose.common.env;. ./local-dev.env; set +a; echo $SHARELATEX_CONFIG; WEB_PORT=13000 LISTEN_ADDRESS=0.0.0.0 npm run nodemon",
+    "local:webpack": "set -a;. ../../config/dev-environment.env;. ../../config/local.env;. ../../config/local-dev.env;. ./docker-compose.common.env;. ./local-dev.env; set +a; PORT=13808 npm run webpack",
+    "local:test:acceptance:run_dir": "set -a;. $(pwd)/docker-compose.common.env;. $(pwd)/local-test.env; set +a; npm run test:acceptance:run_dir",
+    "local:test:acceptance:run_app": "SHARELATEX_CONFIG=$(pwd)/test/acceptance/config/settings.test.${OVERLEAF_APP}.js npm run local:test:acceptance:run_dir -- $(pwd)/test/acceptance/src",
+    "local:test:acceptance:run_module": "if [ ! -d $(pwd)/modules/${MODULE}/test/acceptance ]; then echo \"Module '${MODULE}' does not have acceptance tests\"; exit 0; fi; SHARELATEX_CONFIG=$(pwd)/modules/${MODULE}/test/acceptance/config/settings.test.js BASE_CONFIG=$(pwd)/test/acceptance/config/settings.test.${OVERLEAF_APP}.js npm run local:test:acceptance:run_dir -- $(pwd)/modules/${MODULE}/test/acceptance",
+    "local:test:acceptance:run_modules": "SHARELATEX_CONFIG=$(pwd)/test/acceptance/config/settings.test.${OVERLEAF_APP}.js node $(pwd)/test/acceptance/getModuleTargets --name-only | xargs -n1 sh -c 'MODULE=$0 npm run local:test:acceptance:run_module || exit 255' $1",
+    "local:test:acceptance:app:saas": "OVERLEAF_APP=saas npm run local:test:acceptance:run_app",
+    "local:test:acceptance:app:server-pro": "OVERLEAF_APP=server-pro npm run local:test:acceptance:run_app",
+    "local:test:acceptance:app:server-ce": "OVERLEAF_APP=server-ce npm run local:test:acceptance:run_app",
+    "local:test:acceptance:app": "echo saas server-pro server-ce | xargs -n1 sh -c 'npm run local:test:acceptance:app:${0} || exit 255'",
+    "local:test:acceptance:modules:saas": "OVERLEAF_APP=saas npm run local:test:acceptance:run_modules",
+    "local:test:acceptance:modules:server-pro": "OVERLEAF_APP=server-pro npm run local:test:acceptance:run_modules",
+    "local:test:acceptance:modules:server-ce": "OVERLEAF_APP=server-ce npm run local:test:acceptance:run_modules",
+    "local:test:acceptance:modules": "echo saas server-ce server-pro | xargs -n1 sh -c 'npm run local:test:acceptance:modules:${0} || exit 255'",
+    "local:test:acceptance": "npm run local:test:acceptance:app && npm run local:test:acceptance:modules",
+    "local:test:unit": "npm run test:unit:all",
+    "local:test:frontend": "npm run test:frontend",
+    "local:test": "npm run local:test:unit && npm run local:test:frontend && npm run test:karma:single && npm run local:test:acceptance"
   },
   "browserslist": [
     "last 1 year",

services/web/test/acceptance/config/settings.test.defaults.js

@@ -30,6 +30,7 @@ module.exports = {
     },
 
     web: {
+      url: 'http://localhost:23000',
       user: httpAuthUser,
       pass: httpAuthPass,
     },
@@ -38,6 +39,41 @@ module.exports = {
       enabled: false,
       url: 'http://localhost:1337',
     },
+    documentupdater: {
+      url: 'http://localhost:23003',
+    },
+    spelling: {
+      url: 'http://localhost:23005',
+      host: 'localhost',
+    },
+    trackchanges: {
+      url: 'http://localhost:23015',
+    },
+    docstore: {
+      url: 'http://localhost:23016',
+      pubUrl: 'http://localhost:23016',
+    },
+    chat: {
+      internal_url: 'http://localhost:23010',
+    },
+    filestore: {
+      url: 'http://localhost:23009',
+    },
+    clsi: {
+      url: 'http://localhost:23013',
+    },
+    realTime: {
+      url: 'http://localhost:23026',
+    },
+    contacts: {
+      url: 'http://localhost:23036',
+    },
+    notifications: {
+      url: 'http://localhost:23042',
+    },
+    webpack: {
+      url: 'http://localhost:23808',
+    },
   },
 
   // for registration via SL, set enableLegacyRegistration to true

services/web/test/acceptance/config/settings.test.saas.js

@@ -11,15 +11,21 @@ const overrides = {
   enableSubscriptions: true,
 
   apis: {
+    thirdPartyDataStore: {
+      url: `http://localhost:23002`,
+    },
+    analytics: {
+      url: `http://localhost:23050`,
+    },
     project_history: {
       sendProjectStructureOps: true,
       initializeHistoryForNewProjects: true,
       displayHistoryForNewProjects: true,
-      url: `http://localhost:3054`,
+      url: `http://localhost:23054`,
     },
 
     recurly: {
-      url: 'http://localhost:6034',
+      url: 'http://localhost:26034',
       subdomain: 'test',
       apiKey: 'private-nonsense',
       webhookUser: 'recurly',
@@ -32,13 +38,13 @@ const overrides = {
     },
 
     v1: {
-      url: 'http://localhost:5000',
+      url: `http://localhost:25000`,
       user: 'overleaf',
       pass: 'password',
     },
 
     v1_history: {
-      url: `http://localhost:3100/api`,
+      url: `http://localhost:23100/api`,
       user: 'overleaf',
       pass: 'password',
     },

services/web/test/acceptance/getModuleTargets.js

@@ -5,4 +5,8 @@ const Settings = require('@overleaf/settings')
 const MODULES = Settings.moduleImportSequence
 const TARGET = process.argv.slice(2).pop() || 'test_acceptance'
 
-console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n'))
+if (TARGET === '--name-only') {
+  console.debug(MODULES.join('\n'))
+} else {
+  console.debug(MODULES.map(name => `modules/${name}/${TARGET}`).join('\n'))
+}

services/web/test/acceptance/src/Init.js

@@ -18,18 +18,18 @@ const mockOpts = {
   debug: ['1', 'true', 'TRUE'].includes(process.env.DEBUG_MOCKS),
 }
 
-MockChatApi.initialize(3010, mockOpts)
-MockClsiApi.initialize(3013, mockOpts)
-MockDocstoreApi.initialize(3016, mockOpts)
-MockDocUpdaterApi.initialize(3003, mockOpts)
-MockFilestoreApi.initialize(3009, mockOpts)
-MockNotificationsApi.initialize(3042, mockOpts)
-MockSpellingApi.initialize(3005, mockOpts)
+MockChatApi.initialize(23010, mockOpts)
+MockClsiApi.initialize(23013, mockOpts)
+MockDocstoreApi.initialize(23016, mockOpts)
+MockDocUpdaterApi.initialize(23003, mockOpts)
+MockFilestoreApi.initialize(23009, mockOpts)
+MockNotificationsApi.initialize(23042, mockOpts)
+MockSpellingApi.initialize(23005, mockOpts)
 MockHaveIBeenPwnedApi.initialize(1337, mockOpts)
 
 if (Features.hasFeature('saas')) {
-  MockAnalyticsApi.initialize(3050, mockOpts)
-  MockProjectHistoryApi.initialize(3054, mockOpts)
-  MockV1Api.initialize(5000, mockOpts)
-  MockV1HistoryApi.initialize(3100, mockOpts)
+  MockAnalyticsApi.initialize(23050, mockOpts)
+  MockProjectHistoryApi.initialize(23054, mockOpts)
+  MockV1Api.initialize(25000, mockOpts)
+  MockV1HistoryApi.initialize(23100, mockOpts)
 }

services/web/test/acceptance/src/PasswordResetTests.js

@@ -1,16 +1,9 @@
 const { expect } = require('chai')
-const RateLimiter = require('../../../app/src/infrastructure/RateLimiter')
 const UserHelper = require('./helpers/UserHelper')
 const { db } = require('../../../app/src/infrastructure/mongodb')
 
 describe('PasswordReset', function () {
   let email, response, user, userHelper, token, emailQuery
-  afterEach(async function () {
-    await RateLimiter.promises.clearRateLimit(
-      'password_reset_rate_limit',
-      '127.0.0.1'
-    )
-  })
   beforeEach(async function () {
     userHelper = new UserHelper()
     email = userHelper.getDefaultEmail()

services/web/test/acceptance/src/helpers/InitApp.js

@@ -18,7 +18,7 @@ MockReCAPTCHAApi.initialize(2222)
 let server
 
 before('start main app', function (done) {
-  server = App.listen(3000, 'localhost', done)
+  server = App.listen(23000, 'localhost', done)
 })
 
 before('start queue workers', function () {

services/web/test/acceptance/src/helpers/MongoHelper.js

@@ -1,5 +1,9 @@
 const { execFile } = require('child_process')
-const { waitForDb, db } = require('../../../../app/src/infrastructure/mongodb')
+const {
+  waitForDb,
+  db,
+  dropTestDatabase,
+} = require('../../../../app/src/infrastructure/mongodb')
 const Settings = require('@overleaf/settings')
 
 const DEFAULT_ENV = 'saas'
@@ -7,6 +11,9 @@ const DEFAULT_ENV = 'saas'
 module.exports = {
   initialize() {
     before(waitForDb)
+    if (process.env.CLEANUP_MONGO === 'true') {
+      before(dropTestDatabase)
+    }
 
     before(function (done) {
       const args = [

services/web/test/acceptance/src/helpers/RedisHelper.js

@@ -1,11 +1,10 @@
-const RateLimiter = require('../../../../app/src/infrastructure/RateLimiter')
-
-async function clearOverleafLoginRateLimit() {
-  await RateLimiter.promises.clearRateLimit('overleaf-login', '127.0.0.1')
-}
+const RedisWrapper = require('../../../../app/src/infrastructure/RedisWrapper')
+const client = RedisWrapper.client('ratelimiter')
 
 module.exports = {
   initialize() {
-    before(clearOverleafLoginRateLimit)
+    beforeEach('clear redis', function (done) {
+      client.flushdb(done)
+    })
   },
 }

services/web/test/acceptance/src/helpers/UserHelper.js

@@ -162,7 +162,7 @@ class UserHelper {
    * @returns {string} baseUrl
    */
   static baseUrl() {
-    return `http://${process.env.HTTP_TEST_HOST || 'localhost'}:3000`
+    return `http://${process.env.HTTP_TEST_HOST || 'localhost'}:23000`
   }
 
   /* static async instantiation methods */

services/web/test/acceptance/src/helpers/request.js

@@ -1,6 +1,6 @@
 // TODO: This file was created by bulk-decaffeinate.
 // Sanity-check the conversion and remove this comment.
-const BASE_URL = `http://${process.env.HTTP_TEST_HOST || 'localhost'}:3000`
+const BASE_URL = `http://${process.env.HTTP_TEST_HOST || 'localhost'}:23000`
 const request = require('request').defaults({
   baseUrl: BASE_URL,
   followRedirect: false,

services/web/test/acceptance/src/mocks/MockV1HistoryApi.js

@@ -69,7 +69,7 @@ class MockV1HistoryApi extends AbstractMockApi {
         this.requestedZipPacks++
         this.events.emit('v1-history-pack-zip')
         res.json({
-          zipUrl: `http://localhost:3100/fake-zip-download/${req.params.project_id}/version/${req.params.version}`,
+          zipUrl: `http://localhost:23100/fake-zip-download/${req.params.project_id}/version/${req.params.version}`,
         })
       }
     )

services/web/test/unit/src/Settings/SettingsTests.js

@@ -2,13 +2,14 @@ const chai = require('chai')
 const { expect } = chai
 
 function clearSettingsCache() {
-  delete require.cache[
-    require.resolve('../../../../config/settings.defaults.js')
-  ]
+  const monorepoPath = require
+    .resolve('../../../../config/settings.defaults.js')
+    .replace(/\/services\/web\/config\/settings\.defaults\.js$/, '')
   const settingsDeps = Object.keys(require.cache).filter(
     x =>
-      x.includes('/@overleaf/settings/') ||
-      x.includes('/overleaf/libraries/settings')
+      x.includes('/@overleaf/settings') ||
+      x.includes(`${monorepoPath}/libraries/settings`) ||
+      x.includes(`${monorepoPath}/services/web/config`)
   )
   settingsDeps.forEach(dep => delete require.cache[dep])
 }

services/web/webpack.config.dev.js

@@ -52,7 +52,7 @@ module.exports = merge(base, {
   devServer: {
     // Expose dev server at www.dev-overleaf.com
     host: '0.0.0.0',
-    port: 3808,
+    port: parseInt(process.env.PORT, 10) || 3808,
     client: {
       webSocketURL: 'auto://0.0.0.0:0/ws',
     },