Merge pull request #26393 from overleaf/bg-history-redis-gradual-rollout

add gradual rollout mechanism for history-v1 rollout

GitOrigin-RevId: 5fa69f5c3874bd5df1f31fdd3115e4ba6a0dab51

services/history-v1/api/app/rollout.js

@@ -0,0 +1,76 @@
+const crypto = require('node:crypto')
+
+class Rollout {
+  constructor(config) {
+    // The history buffer level is used to determine whether to queue changes
+    // in Redis or persist them directly to the chunk store.
+    // If defaults to 0 (no queuing) if not set.
+    this.historyBufferLevel = config.has('historyBufferLevel')
+      ? parseInt(config.get('historyBufferLevel'), 10)
+      : 0
+    // The forcePersistBuffer flag will ensure the buffer is fully persisted before
+    // any persist operation. Set this to true if you want to make the persisted-version
+    // in Redis match the endVersion of the latest chunk. This should be set to true
+    // when downgrading from a history buffer level that queues changes in Redis
+    // without persisting them immediately.
+    this.forcePersistBuffer = config.has('forcePersistBuffer')
+      ? config.get('forcePersistBuffer') === 'true'
+      : false
+
+    // Support gradual rollout of the next history buffer level
+    // with a percentage of projects using it.
+    this.nextHistoryBufferLevel = config.has('nextHistoryBufferLevel')
+      ? parseInt(config.get('nextHistoryBufferLevel'), 10)
+      : null
+    this.nextHistoryBufferLevelRolloutPercentage = config.has(
+      'nextHistoryBufferLevelRolloutPercentage'
+    )
+      ? parseInt(config.get('nextHistoryBufferLevelRolloutPercentage'), 10)
+      : 0
+  }
+
+  report(logger) {
+    logger.info(
+      {
+        historyBufferLevel: this.historyBufferLevel,
+        forcePersistBuffer: this.forcePersistBuffer,
+        nextHistoryBufferLevel: this.nextHistoryBufferLevel,
+        nextHistoryBufferLevelRolloutPercentage:
+          this.nextHistoryBufferLevelRolloutPercentage,
+      },
+      this.historyBufferLevel > 0 || this.forcePersistBuffer
+        ? 'using history buffer'
+        : 'history buffer disabled'
+    )
+  }
+
+  /**
+   * Get the history buffer level for a project.
+   * @param {string} projectId
+   * @returns {Object} - An object containing the history buffer level and force persist buffer flag.
+   * @property {number} historyBufferLevel - The history buffer level to use for processing changes.
+   * @property {boolean} forcePersistBuffer - If true, forces the buffer to be persisted before any operation.
+   */
+  getHistoryBufferLevelOptions(projectId) {
+    if (
+      this.nextHistoryBufferLevel > this.historyBufferLevel &&
+      this.nextHistoryBufferLevelRolloutPercentage > 0
+    ) {
+      const hash = crypto.createHash('sha1').update(projectId).digest('hex')
+      const percentage = parseInt(hash.slice(0, 8), 16) % 100
+      // If the project is in the rollout percentage, we use the next history buffer level.
+      if (percentage < this.nextHistoryBufferLevelRolloutPercentage) {
+        return {
+          historyBufferLevel: this.nextHistoryBufferLevel,
+          forcePersistBuffer: this.forcePersistBuffer,
+        }
+      }
+    }
+    return {
+      historyBufferLevel: this.historyBufferLevel,
+      forcePersistBuffer: this.forcePersistBuffer,
+    }
+  }
+}
+
+module.exports = Rollout

services/history-v1/api/controllers/project_import.js

@@ -2,6 +2,7 @@
 
 'use strict'
 
+const config = require('config')
 const { expressify } = require('@overleaf/promise-utils')
 
 const HTTPStatus = require('http-status')
@@ -26,29 +27,10 @@ const persistBuffer = storage.persistBuffer
 const InvalidChangeError = storage.InvalidChangeError
 
 const render = require('./render')
+const Rollout = require('../app/rollout')
 
-const config = require('config')
-// The history buffer level is used to determine whether to queue changes
-// in Redis or persist them directly to the chunk store.
-// If defaults to 0 (no queuing) if not set.
-const historyBufferLevel = config.has('historyBufferLevel')
-  ? parseInt(config.historyBufferLevel, 10)
-  : 0
-// The forcePersistBuffer flag will ensure the buffer is fully persisted before
-// any persist operation. Set this to true if you want to make the persisted-version
-// in Redis match the endVersion of the latest chunk. This should be set to true
-// when downgrading from a history buffer level that queues changes in Redis
-// without persisting them immediately.
-const forcePersistBuffer = config.has('forcePersistBuffer')
-  ? config.get('forcePersistBuffer') === 'true'
-  : false
-
-logger.info(
-  { historyBufferLevel, forcePersistBuffer },
-  historyBufferLevel > 0 || forcePersistBuffer
-    ? 'using history buffer'
-    : 'history buffer disabled'
-)
+const rollout = new Rollout(config)
+rollout.report(logger) // display the rollout configuration in the logs
 
 async function importSnapshot(req, res) {
   const projectId = req.swagger.params.project_id.value
@@ -134,6 +116,8 @@ async function importChanges(req, res, next) {
 
   let result
   try {
+    const { historyBufferLevel, forcePersistBuffer } =
+      rollout.getHistoryBufferLevelOptions(projectId)
     result = await commitChanges(projectId, changes, limits, endVersion, {
       historyBufferLevel,
       forcePersistBuffer,

services/history-v1/config/custom-environment-variables.json

@@ -86,6 +86,8 @@
   "httpRequestTimeout": "HTTP_REQUEST_TIMEOUT",
   "historyBufferLevel": "HISTORY_BUFFER_LEVEL",
   "forcePersistBuffer": "FORCE_PERSIST_BUFFER",
+  "nextHistoryBufferLevel": "NEXT_HISTORY_BUFFER_LEVEL",
+  "nextHistoryBufferLevelRolloutPercentage": "NEXT_HISTORY_BUFFER_LEVEL_ROLLOUT_PERCENTAGE",
   "redis": {
     "queue": {
       "host": "QUEUES_REDIS_HOST",

services/history-v1/test/acceptance/js/api/rollout.test.js

@@ -0,0 +1,115 @@
+const config = require('config')
+const sinon = require('sinon')
+const { expect } = require('chai')
+
+const cleanup = require('../storage/support/cleanup')
+const Rollout = require('../../../../api/app/rollout')
+
+describe('rollout', function () {
+  beforeEach(cleanup.everything)
+  beforeEach('Set up stubs', function () {
+    sinon.stub(config, 'has').callThrough()
+    sinon.stub(config, 'get').callThrough()
+  })
+  afterEach(sinon.restore)
+
+  it('should return a valid history buffer level', function () {
+    setMockConfig('historyBufferLevel', '2')
+    setMockConfig('forcePersistBuffer', 'false')
+
+    const rollout = new Rollout(config)
+    const { historyBufferLevel, forcePersistBuffer } =
+      rollout.getHistoryBufferLevelOptions('test-project-id')
+    expect(historyBufferLevel).to.equal(2)
+    expect(forcePersistBuffer).to.be.false
+  })
+
+  it('should return a valid history buffer level and force persist buffer options', function () {
+    setMockConfig('historyBufferLevel', '1')
+    setMockConfig('forcePersistBuffer', 'true')
+    const rollout = new Rollout(config)
+    const { historyBufferLevel, forcePersistBuffer } =
+      rollout.getHistoryBufferLevelOptions('test-project-id')
+    expect(historyBufferLevel).to.equal(1)
+    expect(forcePersistBuffer).to.be.true
+  })
+
+  describe('with a higher next history buffer level rollout', function () {
+    beforeEach(function () {
+      setMockConfig('historyBufferLevel', '2')
+      setMockConfig('forcePersistBuffer', 'false')
+      setMockConfig('nextHistoryBufferLevel', '3')
+    })
+    it('should return the expected history buffer level when the rollout percentage is zero', function () {
+      setMockConfig('nextHistoryBufferLevelRolloutPercentage', '0')
+      const rollout = new Rollout(config)
+      for (let i = 0; i < 1000; i++) {
+        const { historyBufferLevel, forcePersistBuffer } =
+          rollout.getHistoryBufferLevelOptions(`test-project-id-${i}`)
+        expect(historyBufferLevel).to.equal(2)
+        expect(forcePersistBuffer).to.be.false
+      }
+    })
+
+    it('should return the expected distribution of levels when the rollout percentage is 10%', function () {
+      setMockConfig('nextHistoryBufferLevelRolloutPercentage', '10')
+      const rollout = new Rollout(config)
+      let currentLevel = 0
+      let nextLevel = 0
+      for (let i = 0; i < 1000; i++) {
+        const { historyBufferLevel } = rollout.getHistoryBufferLevelOptions(
+          `test-project-id-${i}`
+        )
+        switch (historyBufferLevel) {
+          case 2:
+            currentLevel++
+            break
+          case 3:
+            nextLevel++
+            break
+          default:
+            expect.fail(
+              `Unexpected history buffer level: ${historyBufferLevel}`
+            )
+        }
+      }
+      const twoPercentage = (currentLevel / 1000) * 100
+      const threePercentage = (nextLevel / 1000) * 100
+      expect(twoPercentage).to.be.closeTo(90, 5) // 90% for level 2
+      expect(threePercentage).to.be.closeTo(10, 5) // 10% for level 3
+    })
+  })
+  describe('with a next history buffer level lower than the current level', function () {
+    beforeEach(function () {
+      setMockConfig('historyBufferLevel', '3')
+      setMockConfig('forcePersistBuffer', 'false')
+      setMockConfig('nextHistoryBufferLevel', '2')
+    })
+    it('should always return the current level when the rollout percentage is zero', function () {
+      setMockConfig('nextHistoryBufferLevelRolloutPercentage', '0')
+      const rollout = new Rollout(config)
+      for (let i = 0; i < 1000; i++) {
+        const { historyBufferLevel, forcePersistBuffer } =
+          rollout.getHistoryBufferLevelOptions(`test-project-id-${i}`)
+        expect(historyBufferLevel).to.equal(3)
+        expect(forcePersistBuffer).to.be.false
+      }
+    })
+
+    it('should always return the current level regardless of the rollout percentage', function () {
+      setMockConfig('nextHistoryBufferLevelRolloutPercentage', '10')
+      const rollout = new Rollout(config)
+      for (let i = 0; i < 1000; i++) {
+        const { historyBufferLevel } = rollout.getHistoryBufferLevelOptions(
+          `test-project-id-${i}`
+        )
+        expect(historyBufferLevel).to.equal(3)
+      }
+    })
+  })
+})
+
+function setMockConfig(path, value) {
+  config.has.withArgs(path).returns(true)
+  config.get.withArgs(path).returns(value)
+}