Merge pull request #21303 from overleaf/revert-21300-revert-20641-ar-organisation-mapping-analytics-pipeline

Revert "Revert "[web] [v1] Send salesforce modification events to analytics queues""

GitOrigin-RevId: cfcd7a6e25ad0902c699e1a34e8654347ebf71fb

services/web/app/src/Features/Analytics/AccountMappingHelper.mjs

@@ -0,0 +1,71 @@
+export function extractAccountMappingsFromSubscription(
+  subscription,
+  updatedSubscription
+) {
+  const accountMappings = []
+  if (
+    updatedSubscription.salesforce_id ||
+    updatedSubscription.salesforce_id === ''
+  ) {
+    if (subscription.salesforce_id !== updatedSubscription.salesforce_id) {
+      accountMappings.push(
+        generateSubscriptionToSalesforceMapping(
+          subscription.id,
+          updatedSubscription.salesforce_id
+        )
+      )
+    }
+  }
+  if (updatedSubscription.v1_id || updatedSubscription.v1_id === '') {
+    if (subscription.v1_id !== updatedSubscription.v1_id) {
+      accountMappings.push(
+        generateSubscriptionToV1Mapping(
+          subscription.id,
+          updatedSubscription.v1_id
+        )
+      )
+    }
+  }
+  return accountMappings
+}
+
+export function generateV1Mapping(v1Id, salesforceId, createdAt) {
+  return {
+    source: 'salesforce',
+    sourceEntity: 'account',
+    sourceEntityId: salesforceId,
+    target: 'v1',
+    targetEntity: 'university',
+    targetEntityId: v1Id,
+    createdAt,
+  }
+}
+
+function generateSubscriptionToV1Mapping(subscriptionId, v1Id) {
+  return {
+    source: 'v1',
+    sourceEntity: 'university',
+    sourceEntityId: v1Id,
+    target: 'v2',
+    targetEntity: 'subscription',
+    targetEntityId: subscriptionId,
+    createdAt: new Date().toISOString(),
+  }
+}
+
+function generateSubscriptionToSalesforceMapping(subscriptionId, salesforceId) {
+  return {
+    source: 'salesforce',
+    sourceEntity: 'account',
+    sourceEntityId: salesforceId,
+    target: 'v2',
+    targetEntity: 'subscription',
+    targetEntityId: subscriptionId,
+    createdAt: new Date().toISOString(),
+  }
+}
+
+export default {
+  extractAccountMappingsFromSubscription,
+  generateV1Mapping,
+}

services/web/app/src/Features/Analytics/AnalyticsController.mjs

@@ -4,6 +4,18 @@ import SessionManager from '../Authentication/SessionManager.js'
 import GeoIpLookup from '../../infrastructure/GeoIpLookup.js'
 import Features from '../../infrastructure/Features.js'
 import { expressify } from '@overleaf/promise-utils'
+import { generateV1Mapping } from './AccountMappingHelper.mjs'
+
+async function registerSalesforceMapping(req, res, next) {
+  if (!Features.hasFeature('analytics')) {
+    return res.sendStatus(202)
+  }
+  const { createdAt, salesforceId, v1Id } = req.body
+  AnalyticsManager.registerAccountMapping(
+    generateV1Mapping(v1Id, salesforceId, createdAt)
+  )
+  res.sendStatus(202)
+}
 
 async function updateEditingSession(req, res, next) {
   if (!Features.hasFeature('analytics')) {
@@ -47,6 +59,7 @@ function recordEvent(req, res, next) {
 }
 
 export default {
+  registerSalesforceMapping: expressify(registerSalesforceMapping),
   updateEditingSession: expressify(updateEditingSession),
   recordEvent,
 }

services/web/app/src/Features/Analytics/AnalyticsManager.js

@@ -15,6 +15,9 @@ const analyticsEditingSessionsQueue = Queues.getQueue(
 const analyticsUserPropertiesQueue = Queues.getQueue(
   'analytics-user-properties'
 )
+const analyticsAccountMappingQueue = Queues.getQueue(
+  'analytics-account-mapping'
+)
 
 const ONE_MINUTE_MS = 60 * 1000
 
@@ -143,6 +146,55 @@ function setUserPropertyForSessionInBackground(session, property, value) {
   })
 }
 
+/**
+ * Register mapping between two accounts.
+ *
+ * @param {object} payload - The event payload to send to Analytics
+ * @param {string} payload.source - The type of account linked from
+ * @param {string} payload.sourceId - The ID of the account linked from
+ * @param {string} payload.target - The type of account linked to
+ * @param {string} payload.targetId - The ID of the account linked to
+ * @param {Date} payload.createdAt - The date the mapping was created
+ * @property
+ */
+function registerAccountMapping({
+  source,
+  sourceEntity,
+  sourceEntityId,
+  target,
+  targetEntity,
+  targetEntityId,
+  createdAt,
+}) {
+  Metrics.analyticsQueue.inc({
+    status: 'adding',
+    event_type: 'account-mapping',
+  })
+
+  analyticsAccountMappingQueue
+    .add('account-mapping', {
+      source,
+      sourceEntity,
+      sourceEntityId,
+      target,
+      targetEntity,
+      targetEntityId,
+      createdAt: createdAt ?? new Date(),
+    })
+    .then(() => {
+      Metrics.analyticsQueue.inc({
+        status: 'added',
+        event_type: 'account-mapping',
+      })
+    })
+    .catch(() => {
+      Metrics.analyticsQueue.inc({
+        status: 'error',
+        event_type: 'account-mapping',
+      })
+    })
+}
+
 function updateEditingSession(userId, projectId, countryCode, segmentation) {
   if (!userId) {
     return
@@ -349,5 +401,6 @@ module.exports = {
   setUserPropertyForAnalyticsId,
   updateEditingSession,
   getIdsFromSession,
+  registerAccountMapping,
   analyticsIdMiddleware: expressify(analyticsIdMiddleware),
 }

services/web/app/src/Features/Analytics/AnalyticsRouter.mjs

@@ -41,5 +41,11 @@ export default {
       RateLimiterMiddleware.rateLimit(rateLimiters.uniExternalCollabProxy),
       AnalyticsProxy.call('/uniExternalCollaboration')
     )
+
+    publicApiRouter.post(
+      '/analytics/register-v-1-salesforce-mapping',
+      AuthenticationController.requirePrivateApiAuth(),
+      AnalyticsController.registerSalesforceMapping
+    )
   },
 }

services/web/app/src/infrastructure/Queues.js

@@ -17,6 +17,9 @@ const QUEUES_JOB_OPTIONS = {
   'analytics-editing-sessions': {
     removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
   },
+  'analytics-account-mapping': {
+    removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
+  },
   'analytics-user-properties': {
     removeOnFail: MAX_FAILED_JOBS_RETAINED_ANALYTICS,
   },
@@ -38,6 +41,7 @@ const QUEUES_JOB_OPTIONS = {
     removeOnFail: MAX_FAILED_JOBS_RETAINED,
     attempts: 3,
   },
+
   'group-sso-reminder': {
     removeOnFail: MAX_FAILED_JOBS_RETAINED,
     attempts: 3,
@@ -54,6 +58,7 @@ const QUEUE_OPTIONS = {
 }
 
 const ANALYTICS_QUEUES = [
+  'analytics-account-mapping',
   'analytics-events',
   'analytics-editing-sessions',
   'analytics-user-properties',

services/web/test/unit/src/Analytics/AccountMappingHelperTests.mjs

@@ -0,0 +1,228 @@
+import path from 'node:path'
+import esmock from 'esmock'
+import { expect } from 'chai'
+import mongodb from 'mongodb-legacy'
+import { fileURLToPath } from 'node:url'
+
+const { ObjectId } = mongodb
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url))
+
+const MODULE_PATH = path.join(
+  __dirname,
+  '../../../../app/src/Features/Analytics/AccountMappingHelper'
+)
+
+describe('AccountMappingHelper', function () {
+  beforeEach(async function () {
+    this.AccountMappingHelper = await esmock.strict(MODULE_PATH)
+  })
+
+  describe('extractAccountMappingsFromSubscription', function () {
+    describe('when the v1 id is the same in the updated subscription and the subscription', function () {
+      describe('when the salesforce id is the same in the updated subscription and the subscription', function () {
+        beforeEach(function () {
+          this.subscription = {
+            id: new ObjectId('abc123abc123abc123abc123'),
+            salesforce_id: 'def456def456def456',
+          }
+          this.updatedSubscription = { salesforce_id: 'def456def456def456' }
+          this.result =
+            this.AccountMappingHelper.extractAccountMappingsFromSubscription(
+              this.subscription,
+              this.updatedSubscription
+            )
+        })
+
+        it('returns an empty array', function () {
+          expect(this.result).to.be.an('array')
+          expect(this.result).to.have.length(0)
+        })
+      })
+      describe('when the salesforce id has changed between the subscription and the updated subscription', function () {
+        beforeEach(function () {
+          this.subscription = {
+            id: new ObjectId('abc123abc123abc123abc123'),
+            salesforce_id: 'def456def456def456',
+          }
+          this.updatedSubscription = { salesforce_id: 'ghi789ghi789ghi789' }
+          this.result =
+            this.AccountMappingHelper.extractAccountMappingsFromSubscription(
+              this.subscription,
+              this.updatedSubscription
+            )
+        })
+
+        it('returns an array with a single item', function () {
+          expect(this.result).to.be.an('array')
+          expect(this.result).to.have.length(1)
+        })
+
+        it('uses "account" as sourceEntity', function () {
+          expect(this.result[0]).to.haveOwnProperty('sourceEntity', 'account')
+        })
+
+        it('uses the salesforceId from the updated subscription as sourceEntityId', function () {
+          expect(this.result[0]).to.haveOwnProperty(
+            'sourceEntityId',
+            this.updatedSubscription.salesforce_id
+          )
+        })
+
+        it('uses "subscription" as targetEntity', function () {
+          expect(this.result[0]).to.haveOwnProperty(
+            'targetEntity',
+            'subscription'
+          )
+        })
+
+        it('uses the subscriptionId as targetEntityId', function () {
+          expect(this.result[0]).to.haveOwnProperty(
+            'targetEntityId',
+            this.subscription.id
+          )
+        })
+      })
+      describe('when the update subscription has a salesforce id and the subscription has no salesforce_id', function () {
+        beforeEach(function () {
+          this.subscription = { id: new ObjectId('abc123abc123abc123abc123') }
+          this.updatedSubscription = { salesforce_id: 'def456def456def456' }
+          this.result =
+            this.AccountMappingHelper.extractAccountMappingsFromSubscription(
+              this.subscription,
+              this.updatedSubscription
+            )
+        })
+
+        it('returns an array with a single item', function () {
+          expect(this.result).to.be.an('array')
+          expect(this.result).to.have.length(1)
+        })
+
+        it('uses "account" as sourceEntity', function () {
+          expect(this.result[0]).to.haveOwnProperty('sourceEntity', 'account')
+        })
+
+        it('uses the salesforceId from the updated subscription as sourceEntityId', function () {
+          expect(this.result[0]).to.haveOwnProperty(
+            'sourceEntityId',
+            this.updatedSubscription.salesforce_id
+          )
+        })
+
+        it('uses "subscription" as targetEntity', function () {
+          expect(this.result[0]).to.haveOwnProperty(
+            'targetEntity',
+            'subscription'
+          )
+        })
+
+        it('uses the subscriptionId as targetEntityId', function () {
+          expect(this.result[0]).to.haveOwnProperty(
+            'targetEntityId',
+            this.subscription.id
+          )
+        })
+      })
+    })
+  })
+
+  describe('when the v1 id has changed between the subscription and the updated subscription', function () {
+    describe('when the salesforce id has not changed between the subscription and the updated subscription', function () {
+      beforeEach(function () {
+        this.subscription = {
+          id: new ObjectId('abc123abc123abc123abc123'),
+          v1_id: '1',
+          salesforce_id: '',
+        }
+        this.updatedSubscription = { v1_id: '2', salesforce_id: '' }
+        this.result =
+          this.AccountMappingHelper.extractAccountMappingsFromSubscription(
+            this.subscription,
+            this.updatedSubscription
+          )
+      })
+
+      it('returns an array with a single item', function () {
+        expect(this.result).to.be.an('array')
+        expect(this.result).to.have.length(1)
+      })
+
+      it('uses "university" as the sourceEntity', function () {
+        expect(this.result[0]).to.haveOwnProperty('sourceEntity', 'university')
+      })
+
+      it('uses the v1_id from the updated subscription as the sourceEntityId', function () {
+        expect(this.result[0]).to.haveOwnProperty(
+          'sourceEntityId',
+          this.updatedSubscription.v1_id
+        )
+      })
+
+      it('uses "subscription" as the targetEntity', function () {
+        expect(this.result[0]).to.haveOwnProperty(
+          'targetEntity',
+          'subscription'
+        )
+      })
+
+      it('uses the subscription id as the targetEntityId', function () {
+        expect(this.result[0]).to.haveOwnProperty(
+          'targetEntityId',
+          this.subscription.id
+        )
+      })
+    })
+    describe('when the salesforce id has changed between the subscription and the updated subscription', function () {
+      beforeEach(function () {
+        this.subscription = {
+          id: new ObjectId('abc123abc123abc123abc123'),
+          v1_id: '',
+          salesforce_id: 'def456def456def456',
+        }
+        this.updatedSubscription = {
+          v1_id: '2',
+          salesforce_id: '',
+        }
+        this.result =
+          this.AccountMappingHelper.extractAccountMappingsFromSubscription(
+            this.subscription,
+            this.updatedSubscription
+          )
+      })
+
+      it('returns an array with two items', function () {
+        expect(this.result).to.be.an('array')
+        expect(this.result).to.have.length(2)
+      })
+
+      it('uses the salesforce_id from the updated subscription as the sourceEntityId for the first item', function () {
+        expect(this.result[0]).to.haveOwnProperty(
+          'sourceEntityId',
+          this.updatedSubscription.salesforce_id
+        )
+      })
+
+      it('uses the subscription id as the targetEntityId for the first item', function () {
+        expect(this.result[0]).to.haveOwnProperty(
+          'targetEntityId',
+          this.subscription.id
+        )
+      })
+
+      it('uses the v1_id from the updated subscription as the sourceEntityId for the second item', function () {
+        expect(this.result[1]).to.haveOwnProperty(
+          'sourceEntityId',
+          this.updatedSubscription.v1_id
+        )
+      })
+
+      it('uses the subscription id as the targetEntityId for the second item', function () {
+        expect(this.result[1]).to.haveOwnProperty(
+          'targetEntityId',
+          this.subscription.id
+        )
+      })
+    })
+  })
+})

services/web/test/unit/src/Analytics/AnalyticsManagerTests.js

@@ -34,6 +34,10 @@ describe('AnalyticsManager', function () {
       add: sinon.stub().resolves(),
       process: sinon.stub().resolves(),
     }
+    this.analyticsAccountMappingQueue = {
+      add: sinon.stub().resolves(),
+      process: sinon.stub().resolves(),
+    }
     const self = this
     this.Queues = {
       getQueue: queueName => {
@@ -46,6 +50,8 @@ describe('AnalyticsManager', function () {
             return self.onboardingEmailsQueue
           case 'analytics-user-properties':
             return self.analyticsUserPropertiesQueue
+          case 'analytics-account-mapping':
+            return self.analyticsAccountMappingQueue
           default:
             throw new Error('Unexpected queue name')
         }
@@ -278,6 +284,24 @@ describe('AnalyticsManager', function () {
         isLoggedIn: true,
       })
     })
+
+    it('account mapping', async function () {
+      const message = {
+        source: 'salesforce',
+        sourceEntity: 'account',
+        sourceEntityId: 'abc123abc123abc123',
+        target: 'v1',
+        targetEntity: 'university',
+        targetEntityId: 1,
+        createdAt: '2021-01-01T00:00:00Z',
+      }
+      await this.AnalyticsManager.registerAccountMapping(message)
+      sinon.assert.calledWithMatch(
+        this.analyticsAccountMappingQueue.add,
+        'account-mapping',
+        message
+      )
+    })
   })
 
   describe('AnalyticsIdMiddleware', function () {
@@ -299,6 +323,8 @@ describe('AnalyticsManager', function () {
                   return self.onboardingEmailsQueue
                 case 'analytics-user-properties':
                   return self.analyticsUserPropertiesQueue
+                case 'analytics-account-mapping':
+                  return self.analyticsAccountMappingQueue
                 default:
                   throw new Error('Unexpected queue name')
               }