overleaf/services/project-history/test/unit/js/RedisManager/RedisManagerTests.js

import { expect } from 'chai'
import sinon from 'sinon'
import { strict as esmock } from 'esmock'

const MODULE_PATH = '../../../../app/js/RedisManager.js'

describe('RedisManager', function () {
  beforeEach(async function () {
    this.rclient = {
      auth: sinon.stub(),
      exec: sinon.stub().resolves(),
      lrange: sinon.stub(),
      lrem: sinon.stub(),
      srem: sinon.stub(),
      del: sinon.stub(),
    }
    this.rclient.multi = sinon.stub().returns(this.rclient)
    this.RedisWrapper = {
      createClient: sinon.stub().returns(this.rclient),
    }
    this.Settings = {
      redis: {
        project_history: {
          key_schema: {
            projectHistoryOps({ project_id: projectId }) {
              return `Project:HistoryOps:{${projectId}}`
            },
            projectHistoryFirstOpTimestamp({ project_id: projectId }) {
              return `ProjectHistory:FirstOpTimestamp:{${projectId}}`
            },
          },
        },
      },
    }

    this.Metrics = {
      timing: sinon.stub(),
      summary: sinon.stub(),
      globalGauge: sinon.stub(),
    }
    this.RedisManager = await esmock(MODULE_PATH, {
      '@overleaf/redis-wrapper': this.RedisWrapper,
      '@overleaf/settings': this.Settings,
      '@overleaf/metrics': this.Metrics,
    })

    this.project_id = 'project-id-123'
    this.batchSize = 100
    this.historyOpsKey = `Project:HistoryOps:{${this.project_id}}`
    this.firstOpTimestampKey = `ProjectHistory:FirstOpTimestamp:{${this.project_id}}`

    this.updates = [
      { v: 42, op: ['a', 'b', 'c', 'd'] },
      { v: 45, op: ['e', 'f', 'g', 'h'] },
    ]
    this.extraUpdates = [{ v: 100, op: ['i', 'j', 'k'] }]
    this.rawUpdates = this.updates.map(update => JSON.stringify(update))
    this.extraRawUpdates = this.extraUpdates.map(update =>
      JSON.stringify(update)
    )
  })

  describe('getOldestDocUpdates', function () {
    beforeEach(async function () {
      this.rclient.lrange.resolves(this.rawUpdates)
      this.batchSize = 3
      this.result = await this.RedisManager.promises.getOldestDocUpdates(
        this.project_id,
        this.batchSize
      )
    })

    it('should read the updates from redis', function () {
      this.rclient.lrange
        .calledWith(this.historyOpsKey, 0, this.batchSize - 1)
        .should.equal(true)
    })

    it('should call the callback with the unparsed ops', function () {
      this.result.should.equal(this.rawUpdates)
    })
  })

  describe('parseDocUpdates', function () {
    it('should return the parsed ops', function () {
      this.RedisManager.parseDocUpdates(this.rawUpdates).should.deep.equal(
        this.updates
      )
    })
  })

  describe('getUpdatesInBatches', function () {
    beforeEach(function () {
      this.runner = sinon.stub().resolves()
    })

    describe('single batch smaller than batch size', function () {
      beforeEach(async function () {
        this.rclient.lrange.resolves(this.rawUpdates)
        this.batchSize = 3
        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          this.batchSize,
          this.runner
        )
      })

      it('requests a single batch of updates', function () {
        this.rclient.lrange.should.have.been.calledOnce
        this.rclient.lrange.should.have.been.calledWith(
          this.historyOpsKey,
          0,
          this.batchSize - 1
        )
      })

      it('calls the runner once', function () {
        this.runner.callCount.should.equal(1)
      })

      it('calls the runner with the updates', function () {
        this.runner.calledWith(this.updates).should.equal(true)
      })

      it('deletes the applied updates', function () {
        for (const update of this.rawUpdates) {
          expect(this.rclient.lrem).to.have.been.calledWith(
            this.historyOpsKey,
            1,
            update
          )
        }
      })

      it('deletes the first op timestamp', function () {
        expect(this.rclient.del).to.have.been.calledWith(
          this.firstOpTimestampKey
        )
      })
    })

    describe('single batch at batch size', function () {
      beforeEach(async function () {
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves([])
        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner once', function () {
        this.runner.callCount.should.equal(1)
      })

      it('calls the runner with the updates', function () {
        this.runner.calledWith(this.updates).should.equal(true)
      })

      it('deletes the applied updates', function () {
        for (const update of this.rawUpdates) {
          expect(this.rclient.lrem).to.have.been.calledWith(
            this.historyOpsKey,
            1,
            update
          )
        }
      })

      it('deletes the first op timestamp', function () {
        expect(this.rclient.del).to.have.been.calledWith(
          this.firstOpTimestampKey
        )
      })
    })

    describe('single batch exceeding size limit on updates', function () {
      beforeEach(async function () {
        // set the threshold below the size of the first update
        this.RedisManager.setRawUpdateSizeThreshold(
          this.rawUpdates[0].length - 1
        )
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))

        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('deletes the first update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[0]
        )
      })

      it('calls the runner with the second update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the second set of applied updates', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[1]
        )
      })
    })

    describe('two batches with first update below and second update above the size limit on updates', function () {
      beforeEach(async function () {
        // set the threshold above the size of the first update, but below the total size
        this.RedisManager.setRawUpdateSizeThreshold(
          this.rawUpdates[0].length + 1
        )
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))
        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner.calledWith(this.updates.slice(0, 1)).should.equal(true)
      })

      it('deletes the first set applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[0]
        )
      })

      it('calls the runner with the second update', function () {
        this.runner.calledWith(this.updates.slice(1)).should.equal(true)
      })

      it('deletes the second applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[1]
        )
      })
    })

    describe('single batch exceeding op count limit on updates', function () {
      beforeEach(async function () {
        // set the threshold below the size of the first update
        this.RedisManager.setMaxUpdateOpLength(this.updates[0].op.length - 1)
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))

        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first updates', function () {
        this.runner.calledWith(this.updates.slice(0, 1)).should.equal(true)
      })

      it('deletes the first applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[0]
        )
      })

      it('calls the runner with the second updates', function () {
        this.runner.calledWith(this.updates.slice(1)).should.equal(true)
      })

      it('deletes the second applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[1]
        )
      })
    })

    describe('single batch exceeding doc content count', function () {
      beforeEach(async function () {
        this.updates = [{ resyncDocContent: 123 }, { resyncDocContent: 456 }]
        this.rawUpdates = this.updates.map(update => JSON.stringify(update))
        // set the threshold below the size of the first update
        this.RedisManager.setMaxNewDocContentCount(this.updates.length - 1)
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))

        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('deletes the first applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[0]
        )
      })

      it('calls the runner with the second update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the second set of applied updates', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[1]
        )
      })
    })

    describe('two batches with first update below and second update above the ops length limit on updates', function () {
      beforeEach(async function () {
        // set the threshold below the size of the first update
        this.RedisManager.setMaxUpdateOpLength(this.updates[0].op.length + 1)
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))

        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the first update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
      })

      it('deletes the first applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[0]
        )
      })

      it('calls the runner with the second update', function () {
        this.runner.should.have.been.calledWith(this.updates.slice(1))
      })

      it('deletes the second applied update', function () {
        expect(this.rclient.lrem).to.have.been.calledWith(
          this.historyOpsKey,
          1,
          this.rawUpdates[1]
        )
      })
    })

    describe('two batches', function () {
      beforeEach(async function () {
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).resolves(this.extraRawUpdates)
        await this.RedisManager.promises.getUpdatesInBatches(
          this.project_id,
          2,
          this.runner
        )
      })

      it('requests a second batch of updates', function () {
        this.rclient.lrange.should.have.been.calledTwice
      })

      it('calls the runner twice', function () {
        this.runner.callCount.should.equal(2)
      })

      it('calls the runner with the updates', function () {
        this.runner.should.have.been.calledWith(this.updates)
        this.runner.should.have.been.calledWith(this.extraUpdates)
      })

      it('deletes the first set of applied updates', function () {
        for (const update of this.rawUpdates) {
          expect(this.rclient.lrem).to.have.been.calledWith(
            this.historyOpsKey,
            1,
            update
          )
        }
      })

      it('deletes the second set of applied updates', function () {
        for (const update of this.extraRawUpdates) {
          expect(this.rclient.lrem).to.have.been.calledWith(
            this.historyOpsKey,
            1,
            update
          )
        }
      })
    })

    describe('error when first reading updates', function () {
      beforeEach(async function () {
        this.error = new Error('error')
        this.rclient.lrange.rejects(this.error)
        await expect(
          this.RedisManager.promises.getUpdatesInBatches(
            this.project_id,
            2,
            this.runner
          )
        ).to.be.rejected
      })

      it('does not delete any updates', function () {
        expect(this.rclient.lrem).not.to.have.been.called
      })
    })

    describe('error when reading updates for a second batch', function () {
      beforeEach(async function () {
        this.error = new Error('error')
        this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
        this.rclient.lrange.onCall(1).rejects(this.error)

        await expect(
          this.RedisManager.promises.getUpdatesInBatches(
            this.project_id,
            2,
            this.runner
          )
        ).to.be.rejected
      })

      it('deletes the first set of applied updates', function () {
        for (const update of this.rawUpdates) {
          expect(this.rclient.lrem).to.have.been.calledWith(
            this.historyOpsKey,
            1,
            update
          )
        }
      })

      it('deletes applied updates only once', function () {
        expect(this.rclient.lrem.callCount).to.equal(this.rawUpdates.length)
      })
    })
  })
})