Overleaf/services/project-history/test/unit/js/RedisManager/RedisManagerTests.js

import { expect } from 'chai'
import sinon from 'sinon'
import { strict as esmock } from 'esmock'

const MODULE_PATH = '../../../../app/js/RedisManager.js'

describe('RedisManager', function () {
  beforeEach(async function () {
    this.rclient = new FakeRedis()
    this.RedisWrapper = {
      createClient: sinon.stub().returns(this.rclient),
    }
    this.Settings = {
      redis: {
        project_history: {
          key_schema: {
            projectHistoryOps({ project_id: projectId }) {
              return `Project:HistoryOps:{${projectId}}`
            },
            projectHistoryFirstOpTimestamp({ project_id: projectId }) {
              return `ProjectHistory:FirstOpTimestamp:{${projectId}}`
            },
          },
        },
      },
    }

    this.Metrics = {
      timing: sinon.stub(),
      summary: sinon.stub(),
      globalGauge: sinon.stub(),
    }
    this.RedisManager = await esmock(MODULE_PATH, {
      '@overleaf/redis-wrapper': this.RedisWrapper,
      '@overleaf/settings': this.Settings,
      '@overleaf/metrics': this.Metrics,
    })

    this.projectId = 'project-id-123'
    this.batchSize = 100
    this.historyOpsKey = `Project:HistoryOps:{${this.projectId}}`
    this.firstOpTimestampKey = `ProjectHistory:FirstOpTimestamp:{${this.projectId}}`

    this.updates = [
      { v: 42, op: ['a', 'b', 'c', 'd'] },
      { v: 45, op: ['e', 'f', 'g', 'h'] },
    ]
    this.extraUpdates = [{ v: 100, op: ['i', 'j', 'k'] }]
    this.rawUpdates = this.updates.map(update => JSON.stringify(update))
    this.extraRawUpdates = this.extraUpdates.map(update =>
      JSON.stringify(update)
    )
  })

  describe('getRawUpdatesBatch', function () {
    it('gets a small number of updates in one batch', async function () {
      const updates = makeUpdates(2)
      const rawUpdates = makeRawUpdates(updates)
      this.rclient.setList(this.historyOpsKey, rawUpdates)
      const result = await this.RedisManager.promises.getRawUpdatesBatch(
        this.projectId,
        100
      )
      expect(result).to.deep.equal({ rawUpdates, hasMore: false })
    })

    it('gets a larger number of updates in several batches', async function () {
      const updates = makeUpdates(
        this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2 + 12
      )
      const rawUpdates = makeRawUpdates(updates)
      this.rclient.setList(this.historyOpsKey, rawUpdates)
      const result = await this.RedisManager.promises.getRawUpdatesBatch(
        this.projectId,
        5000
      )
      expect(result).to.deep.equal({ rawUpdates, hasMore: false })
    })

    it("doesn't return more than the number of updates requested", async function () {
      const updates = makeUpdates(100)
      const rawUpdates = makeRawUpdates(updates)
      this.rclient.setList(this.historyOpsKey, rawUpdates)
      const result = await this.RedisManager.promises.getRawUpdatesBatch(
        this.projectId,
        75
      )
      expect(result).to.deep.equal({
        rawUpdates: rawUpdates.slice(0, 75),
        hasMore: true,
      })
    })
  })

  describe('parseDocUpdates', function () {
    it('should return the parsed ops', function () {
      const updates = makeUpdates(12)
      const rawUpdates = makeRawUpdates(updates)
      this.RedisManager.parseDocUpdates(rawUpdates).should.deep.equal(updates)
    })
  })

  describe('getUpdatesInBatches', function () {
    beforeEach(function () {
      this.runner = sinon.stub().resolves()
    })

    describe('single batch smaller than batch size', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(2)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          3,
          this.runner
        )
      })

      it('calls the runner once', function () {
        this.runner.callCount.should.equal(1)
      })

      it('calls the runner with the updates', function () {
        this.runner.should.have.been.calledWith(this.updates)
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })

      it('deletes the first op timestamp', function () {
        expect(this.rclient.del).to.have.been.calledWith(
          this.firstOpTimestampKey
        )
      })
    })

    describe('single batch at batch size', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(123)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          123,
          this.runner
        )
      })

      it('calls the runner once', function () {
        this.runner.callCount.should.equal(1)
      })

      it('calls the runner with the updates', function () {
        this.runner.should.have.been.calledWith(this.updates)
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })

      it('deletes the first op timestamp', function () {
        expect(this.rclient.del).to.have.been.calledWith(
          this.firstOpTimestampKey
        )
      })
    })

    describe('single batch exceeding size limit on updates', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(2, [
          'x'.repeat(this.RedisManager.RAW_UPDATE_SIZE_THRESHOLD),
        ])
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          123,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('calls the runner with the second update', function () {
        this.runner
          .getCall(1)
          .should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('two batches with first update below and second update above the size limit on updates', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(2, [
          'x'.repeat(this.RedisManager.RAW_UPDATE_SIZE_THRESHOLD / 2),
        ])
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          123,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('calls the runner with the second update', function () {
        this.runner
          .getCall(1)
          .should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('single batch exceeding op count limit on updates', function () {
      beforeEach(async function () {
        const ops = Array(this.RedisManager.MAX_UPDATE_OP_LENGTH + 1).fill('op')
        this.updates = makeUpdates(2, { op: ops })
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          123,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('calls the runner with the second update', function () {
        this.runner
          .getCall(1)
          .should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('single batch exceeding doc content count', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(
          this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT + 3,
          { resyncDocContent: 123 }
        )
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          123,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first batch of updates', function () {
        this.runner.should.have.been.calledWith(
          this.updates.slice(0, this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT)
        )
      })

      it('calls the runner with the second batch of updates', function () {
        this.runner.should.have.been.calledWith(
          this.updates.slice(this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT)
        )
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('two batches with first update below and second update above the ops length limit on updates', function () {
      beforeEach(async function () {
        // set the threshold below the size of the first update
        this.updates = makeUpdates(2, { op: ['op1', 'op2'] })
        this.updates[1].op = Array(
          this.RedisManager.MAX_UPDATE_OP_LENGTH + 2
        ).fill('op')
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          123,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('calls the runner with the second update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('two batches, one partial', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(15)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          10,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the updates', function () {
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, 10))
        this.runner
          .getCall(1)
          .should.have.been.calledWith(this.updates.slice(10))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('two full batches', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(20)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          10,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the updates', function () {
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, 10))
        this.runner
          .getCall(1)
          .should.have.been.calledWith(this.updates.slice(10))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('three full bathches, bigger than the Redis read batch size', function () {
      beforeEach(async function () {
        this.batchSize = this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2
        this.updates = makeUpdates(this.batchSize * 3)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.projectId,
          this.batchSize,
          this.runner
        )
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(3)
      })

      it('calls the runner with the updates', function () {
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, this.batchSize))
        this.runner
          .getCall(1)
          .should.have.been.calledWith(
            this.updates.slice(this.batchSize, this.batchSize * 2)
          )
        this.runner
          .getCall(2)
          .should.have.been.calledWith(this.updates.slice(this.batchSize * 2))
      })

      it('deletes the applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
      })
    })

    describe('error when first reading updates', function () {
      beforeEach(async function () {
        this.updates = makeUpdates(10)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        this.rclient.throwErrorOnLrangeCall(0)
        await expect(
          this.RedisManager.promises.getUpdatesInBatches(
            this.projectId,
            2,
            this.runner
          )
        ).to.be.rejected
      })

      it('does not delete any updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal(
          this.rawUpdates
        )
      })
    })

    describe('error when reading updates for a second batch', function () {
      beforeEach(async function () {
        this.batchSize = this.RedisManager.RAW_UPDATES_BATCH_SIZE - 1
        this.updates = makeUpdates(this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2)
        this.rawUpdates = makeRawUpdates(this.updates)
        this.rclient.setList(this.historyOpsKey, this.rawUpdates)
        this.rclient.throwErrorOnLrangeCall(1)
        await expect(
          this.RedisManager.promises.getUpdatesInBatches(
            this.projectId,
            this.batchSize,
            this.runner
          )
        ).to.be.rejected
      })

      it('calls the runner with the first batch of updates', function () {
        this.runner.should.have.been.calledOnce
        this.runner
          .getCall(0)
          .should.have.been.calledWith(this.updates.slice(0, this.batchSize))
      })

      it('deletes only the first batch of applied updates', function () {
        expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal(
          this.rawUpdates.slice(this.batchSize)
        )
      })
    })
  })
})

class FakeRedis {
  constructor() {
    this.data = new Map()
    this.del = sinon.stub()
    this.lrangeCallCount = -1
  }

  setList(key, list) {
    this.data.set(key, list)
  }

  getList(key) {
    return this.data.get(key)
  }

  throwErrorOnLrangeCall(callNum) {
    this.lrangeCallThrowingError = callNum
  }

  async lrange(key, start, stop) {
    this.lrangeCallCount += 1
    if (
      this.lrangeCallThrowingError != null &&
      this.lrangeCallThrowingError === this.lrangeCallCount
    ) {
      throw new Error('LRANGE failed!')
    }
    const list = this.data.get(key) ?? []
    return list.slice(start, stop + 1)
  }

  async lrem(key, count, elementToRemove) {
    expect(count).to.be.greaterThan(0)
    const original = this.data.get(key) ?? []
    const filtered = original.filter(element => {
      if (count > 0 && element === elementToRemove) {
        count--
        return false
      }
      return true
    })
    this.data.set(key, filtered)
  }

  async exec() {
    // Nothing to do
  }

  multi() {
    return this
  }
}

function makeUpdates(updateCount, extraFields = {}) {
  const updates = []
  for (let i = 0; i < updateCount; i++) {
    updates.push({ v: i, ...extraFields })
  }
  return updates
}

function makeRawUpdates(updates) {
  return updates.map(JSON.stringify)
}