mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
3ead64344f
Read updates from Redis in smaller batches GitOrigin-RevId: 06901e4a9e43976e446c014d5d46c2488691c205
556 lines
17 KiB
JavaScript
556 lines
17 KiB
JavaScript
import { expect } from 'chai'
|
|
import sinon from 'sinon'
|
|
import { strict as esmock } from 'esmock'
|
|
|
|
const MODULE_PATH = '../../../../app/js/RedisManager.js'
|
|
|
|
describe('RedisManager', function () {
|
|
beforeEach(async function () {
|
|
this.rclient = new FakeRedis()
|
|
this.RedisWrapper = {
|
|
createClient: sinon.stub().returns(this.rclient),
|
|
}
|
|
this.Settings = {
|
|
redis: {
|
|
project_history: {
|
|
key_schema: {
|
|
projectHistoryOps({ project_id: projectId }) {
|
|
return `Project:HistoryOps:{${projectId}}`
|
|
},
|
|
projectHistoryFirstOpTimestamp({ project_id: projectId }) {
|
|
return `ProjectHistory:FirstOpTimestamp:{${projectId}}`
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
this.Metrics = {
|
|
timing: sinon.stub(),
|
|
summary: sinon.stub(),
|
|
globalGauge: sinon.stub(),
|
|
}
|
|
this.RedisManager = await esmock(MODULE_PATH, {
|
|
'@overleaf/redis-wrapper': this.RedisWrapper,
|
|
'@overleaf/settings': this.Settings,
|
|
'@overleaf/metrics': this.Metrics,
|
|
})
|
|
|
|
this.projectId = 'project-id-123'
|
|
this.batchSize = 100
|
|
this.historyOpsKey = `Project:HistoryOps:{${this.projectId}}`
|
|
this.firstOpTimestampKey = `ProjectHistory:FirstOpTimestamp:{${this.projectId}}`
|
|
|
|
this.updates = [
|
|
{ v: 42, op: ['a', 'b', 'c', 'd'] },
|
|
{ v: 45, op: ['e', 'f', 'g', 'h'] },
|
|
]
|
|
this.extraUpdates = [{ v: 100, op: ['i', 'j', 'k'] }]
|
|
this.rawUpdates = this.updates.map(update => JSON.stringify(update))
|
|
this.extraRawUpdates = this.extraUpdates.map(update =>
|
|
JSON.stringify(update)
|
|
)
|
|
})
|
|
|
|
describe('getRawUpdatesBatch', function () {
|
|
it('gets a small number of updates in one batch', async function () {
|
|
const updates = makeUpdates(2)
|
|
const rawUpdates = makeRawUpdates(updates)
|
|
this.rclient.setList(this.historyOpsKey, rawUpdates)
|
|
const result = await this.RedisManager.promises.getRawUpdatesBatch(
|
|
this.projectId,
|
|
100
|
|
)
|
|
expect(result).to.deep.equal({ rawUpdates, hasMore: false })
|
|
})
|
|
|
|
it('gets a larger number of updates in several batches', async function () {
|
|
const updates = makeUpdates(
|
|
this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2 + 12
|
|
)
|
|
const rawUpdates = makeRawUpdates(updates)
|
|
this.rclient.setList(this.historyOpsKey, rawUpdates)
|
|
const result = await this.RedisManager.promises.getRawUpdatesBatch(
|
|
this.projectId,
|
|
5000
|
|
)
|
|
expect(result).to.deep.equal({ rawUpdates, hasMore: false })
|
|
})
|
|
|
|
it("doesn't return more than the number of updates requested", async function () {
|
|
const updates = makeUpdates(100)
|
|
const rawUpdates = makeRawUpdates(updates)
|
|
this.rclient.setList(this.historyOpsKey, rawUpdates)
|
|
const result = await this.RedisManager.promises.getRawUpdatesBatch(
|
|
this.projectId,
|
|
75
|
|
)
|
|
expect(result).to.deep.equal({
|
|
rawUpdates: rawUpdates.slice(0, 75),
|
|
hasMore: true,
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('parseDocUpdates', function () {
|
|
it('should return the parsed ops', function () {
|
|
const updates = makeUpdates(12)
|
|
const rawUpdates = makeRawUpdates(updates)
|
|
this.RedisManager.parseDocUpdates(rawUpdates).should.deep.equal(updates)
|
|
})
|
|
})
|
|
|
|
describe('getUpdatesInBatches', function () {
|
|
beforeEach(function () {
|
|
this.runner = sinon.stub().resolves()
|
|
})
|
|
|
|
describe('single batch smaller than batch size', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(2)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
3,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner once', function () {
|
|
this.runner.callCount.should.equal(1)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner.should.have.been.calledWith(this.updates)
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
|
|
it('deletes the first op timestamp', function () {
|
|
expect(this.rclient.del).to.have.been.calledWith(
|
|
this.firstOpTimestampKey
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('single batch at batch size', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(123)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
123,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner once', function () {
|
|
this.runner.callCount.should.equal(1)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner.should.have.been.calledWith(this.updates)
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
|
|
it('deletes the first op timestamp', function () {
|
|
expect(this.rclient.del).to.have.been.calledWith(
|
|
this.firstOpTimestampKey
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('single batch exceeding size limit on updates', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(2, [
|
|
'x'.repeat(this.RedisManager.RAW_UPDATE_SIZE_THRESHOLD),
|
|
])
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
123,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner
|
|
.getCall(1)
|
|
.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('two batches with first update below and second update above the size limit on updates', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(2, [
|
|
'x'.repeat(this.RedisManager.RAW_UPDATE_SIZE_THRESHOLD / 2),
|
|
])
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
123,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner
|
|
.getCall(1)
|
|
.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('single batch exceeding op count limit on updates', function () {
|
|
beforeEach(async function () {
|
|
const ops = Array(this.RedisManager.MAX_UPDATE_OP_LENGTH + 1).fill('op')
|
|
this.updates = makeUpdates(2, { op: ops })
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
123,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner
|
|
.getCall(1)
|
|
.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('single batch exceeding doc content count', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(
|
|
this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT + 3,
|
|
{ resyncDocContent: 123 }
|
|
)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
123,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first batch of updates', function () {
|
|
this.runner.should.have.been.calledWith(
|
|
this.updates.slice(0, this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT)
|
|
)
|
|
})
|
|
|
|
it('calls the runner with the second batch of updates', function () {
|
|
this.runner.should.have.been.calledWith(
|
|
this.updates.slice(this.RedisManager.MAX_NEW_DOC_CONTENT_COUNT)
|
|
)
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('two batches with first update below and second update above the ops length limit on updates', function () {
|
|
beforeEach(async function () {
|
|
// set the threshold below the size of the first update
|
|
this.updates = makeUpdates(2, { op: ['op1', 'op2'] })
|
|
this.updates[1].op = Array(
|
|
this.RedisManager.MAX_UPDATE_OP_LENGTH + 2
|
|
).fill('op')
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
123,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('two batches, one partial', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(15)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
10,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, 10))
|
|
this.runner
|
|
.getCall(1)
|
|
.should.have.been.calledWith(this.updates.slice(10))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('two full batches', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(20)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
10,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, 10))
|
|
this.runner
|
|
.getCall(1)
|
|
.should.have.been.calledWith(this.updates.slice(10))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('three full bathches, bigger than the Redis read batch size', function () {
|
|
beforeEach(async function () {
|
|
this.batchSize = this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2
|
|
this.updates = makeUpdates(this.batchSize * 3)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
this.batchSize,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(3)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, this.batchSize))
|
|
this.runner
|
|
.getCall(1)
|
|
.should.have.been.calledWith(
|
|
this.updates.slice(this.batchSize, this.batchSize * 2)
|
|
)
|
|
this.runner
|
|
.getCall(2)
|
|
.should.have.been.calledWith(this.updates.slice(this.batchSize * 2))
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal([])
|
|
})
|
|
})
|
|
|
|
describe('error when first reading updates', function () {
|
|
beforeEach(async function () {
|
|
this.updates = makeUpdates(10)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
this.rclient.throwErrorOnLrangeCall(0)
|
|
await expect(
|
|
this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
2,
|
|
this.runner
|
|
)
|
|
).to.be.rejected
|
|
})
|
|
|
|
it('does not delete any updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal(
|
|
this.rawUpdates
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('error when reading updates for a second batch', function () {
|
|
beforeEach(async function () {
|
|
this.batchSize = this.RedisManager.RAW_UPDATES_BATCH_SIZE - 1
|
|
this.updates = makeUpdates(this.RedisManager.RAW_UPDATES_BATCH_SIZE * 2)
|
|
this.rawUpdates = makeRawUpdates(this.updates)
|
|
this.rclient.setList(this.historyOpsKey, this.rawUpdates)
|
|
this.rclient.throwErrorOnLrangeCall(1)
|
|
await expect(
|
|
this.RedisManager.promises.getUpdatesInBatches(
|
|
this.projectId,
|
|
this.batchSize,
|
|
this.runner
|
|
)
|
|
).to.be.rejected
|
|
})
|
|
|
|
it('calls the runner with the first batch of updates', function () {
|
|
this.runner.should.have.been.calledOnce
|
|
this.runner
|
|
.getCall(0)
|
|
.should.have.been.calledWith(this.updates.slice(0, this.batchSize))
|
|
})
|
|
|
|
it('deletes only the first batch of applied updates', function () {
|
|
expect(this.rclient.getList(this.historyOpsKey)).to.deep.equal(
|
|
this.rawUpdates.slice(this.batchSize)
|
|
)
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
class FakeRedis {
|
|
constructor() {
|
|
this.data = new Map()
|
|
this.del = sinon.stub()
|
|
this.lrangeCallCount = -1
|
|
}
|
|
|
|
setList(key, list) {
|
|
this.data.set(key, list)
|
|
}
|
|
|
|
getList(key) {
|
|
return this.data.get(key)
|
|
}
|
|
|
|
throwErrorOnLrangeCall(callNum) {
|
|
this.lrangeCallThrowingError = callNum
|
|
}
|
|
|
|
async lrange(key, start, stop) {
|
|
this.lrangeCallCount += 1
|
|
if (
|
|
this.lrangeCallThrowingError != null &&
|
|
this.lrangeCallThrowingError === this.lrangeCallCount
|
|
) {
|
|
throw new Error('LRANGE failed!')
|
|
}
|
|
const list = this.data.get(key) ?? []
|
|
return list.slice(start, stop + 1)
|
|
}
|
|
|
|
async lrem(key, count, elementToRemove) {
|
|
expect(count).to.be.greaterThan(0)
|
|
const original = this.data.get(key) ?? []
|
|
const filtered = original.filter(element => {
|
|
if (count > 0 && element === elementToRemove) {
|
|
count--
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
this.data.set(key, filtered)
|
|
}
|
|
|
|
async exec() {
|
|
// Nothing to do
|
|
}
|
|
|
|
multi() {
|
|
return this
|
|
}
|
|
}
|
|
|
|
function makeUpdates(updateCount, extraFields = {}) {
|
|
const updates = []
|
|
for (let i = 0; i < updateCount; i++) {
|
|
updates.push({ v: i, ...extraFields })
|
|
}
|
|
return updates
|
|
}
|
|
|
|
function makeRawUpdates(updates) {
|
|
return updates.map(JSON.stringify)
|
|
}
|