Merge pull request #7304 from overleaf/em-split-queues

Split the config for web and analytics queues

GitOrigin-RevId: fa3fc86c874e653649aefad030607105d514a52b

services/chat/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/chat/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/clsi/docker-compose.ci.yml

@@ -23,6 +23,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/clsi/docker-compose.yml

@@ -38,6 +38,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/contacts/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/contacts/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/docstore/docker-compose.ci.yml

@@ -24,6 +24,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       GCS_API_ENDPOINT: gcs:9090

services/docstore/docker-compose.yml

@@ -33,6 +33,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       GCS_API_ENDPOINT: gcs:9090

services/document-updater/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/document-updater/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/filestore/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: http://s3:9090

services/filestore/docker-compose.yml

@@ -36,6 +36,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: http://s3:9090

services/notifications/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/notifications/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/real-time/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/real-time/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/spelling/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/spelling/docker-compose.yml

@@ -36,6 +36,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       MOCHA_GREP: ${MOCHA_GREP}

services/track-changes/docker-compose.ci.yml

@@ -21,6 +21,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: http://s3:9090

services/track-changes/docker-compose.yml

@@ -30,6 +30,7 @@ services:
       ELASTIC_SEARCH_DSN: es:9200
       REDIS_HOST: redis
       QUEUES_REDIS_HOST: redis
+      ANALYTICS_QUEUES_REDIS_HOST: redis
       MONGO_HOST: mongo
       POSTGRES_HOST: postgres
       AWS_S3_ENDPOINT: http://s3:9090

services/web/app/src/infrastructure/Queues.js

@@ -21,15 +21,25 @@ const QUEUES_JOB_OPTIONS = {
   },
 }
 
+const ANALYTICS_QUEUES = [
+  'analytics-events',
+  'analytics-editing-sessions',
+  'analytics-user-properties',
+  'post-registration-analytics',
+]
+
 const queues = {}
 
 function getQueue(queueName) {
   if (!queues[queueName]) {
+    const redisOptions = ANALYTICS_QUEUES.includes(queueName)
+      ? Settings.redis.analyticsQueues
+      : Settings.redis.queues
     const jobOptions = QUEUES_JOB_OPTIONS[queueName] || {}
     queues[queueName] = new Queue(queueName, {
       // this configuration is duplicated in /services/analytics/app/js/Queues.js
       // and needs to be manually kept in sync whenever modified
-      redis: Settings.redis.queues,
+      redis: redisOptions,
       defaultJobOptions: {
         removeOnComplete: MAX_COMPLETED_JOBS_RETAINED,
         removeOnFail: MAX_FAILED_JOBS_RETAINED,

services/web/docker-compose.common.env

@@ -2,6 +2,7 @@ ADMIN_PRIVILEGE_AVAILABLE=true
 BCRYPT_ROUNDS=1
 REDIS_HOST=redis
 QUEUES_REDIS_HOST=redis
+ANALYTICS_QUEUES_REDIS_HOST=redis
 MONGO_URL=mongodb://mongo/sharelatex
 SHARELATEX_ALLOW_PUBLIC_ACCESS=true
 LINKED_URL_PROXY=http://localhost:6543