mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-23 02:55:13 -04:00
a7c9e3b20f
Clean up and promisify RedisManager in project-history GitOrigin-RevId: 8bd8bb7d51a0a68f7b1a97ffa310a674086714ba
515 lines
16 KiB
JavaScript
515 lines
16 KiB
JavaScript
import { expect } from 'chai'
|
|
import sinon from 'sinon'
|
|
import { strict as esmock } from 'esmock'
|
|
|
|
const MODULE_PATH = '../../../../app/js/RedisManager.js'
|
|
|
|
describe('RedisManager', function () {
|
|
beforeEach(async function () {
|
|
this.rclient = {
|
|
auth: sinon.stub(),
|
|
exec: sinon.stub().resolves(),
|
|
lrange: sinon.stub(),
|
|
lrem: sinon.stub(),
|
|
srem: sinon.stub(),
|
|
del: sinon.stub(),
|
|
}
|
|
this.rclient.multi = sinon.stub().returns(this.rclient)
|
|
this.RedisWrapper = {
|
|
createClient: sinon.stub().returns(this.rclient),
|
|
}
|
|
this.Settings = {
|
|
redis: {
|
|
project_history: {
|
|
key_schema: {
|
|
projectHistoryOps({ project_id: projectId }) {
|
|
return `Project:HistoryOps:{${projectId}}`
|
|
},
|
|
projectHistoryFirstOpTimestamp({ project_id: projectId }) {
|
|
return `ProjectHistory:FirstOpTimestamp:{${projectId}}`
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
this.Metrics = {
|
|
timing: sinon.stub(),
|
|
summary: sinon.stub(),
|
|
globalGauge: sinon.stub(),
|
|
}
|
|
this.RedisManager = await esmock(MODULE_PATH, {
|
|
'@overleaf/redis-wrapper': this.RedisWrapper,
|
|
'@overleaf/settings': this.Settings,
|
|
'@overleaf/metrics': this.Metrics,
|
|
})
|
|
|
|
this.project_id = 'project-id-123'
|
|
this.batchSize = 100
|
|
this.historyOpsKey = `Project:HistoryOps:{${this.project_id}}`
|
|
this.firstOpTimestampKey = `ProjectHistory:FirstOpTimestamp:{${this.project_id}}`
|
|
|
|
this.updates = [
|
|
{ v: 42, op: ['a', 'b', 'c', 'd'] },
|
|
{ v: 45, op: ['e', 'f', 'g', 'h'] },
|
|
]
|
|
this.extraUpdates = [{ v: 100, op: ['i', 'j', 'k'] }]
|
|
this.rawUpdates = this.updates.map(update => JSON.stringify(update))
|
|
this.extraRawUpdates = this.extraUpdates.map(update =>
|
|
JSON.stringify(update)
|
|
)
|
|
})
|
|
|
|
describe('getOldestDocUpdates', function () {
|
|
beforeEach(async function () {
|
|
this.rclient.lrange.resolves(this.rawUpdates)
|
|
this.batchSize = 3
|
|
this.result = await this.RedisManager.promises.getOldestDocUpdates(
|
|
this.project_id,
|
|
this.batchSize
|
|
)
|
|
})
|
|
|
|
it('should read the updates from redis', function () {
|
|
this.rclient.lrange
|
|
.calledWith(this.historyOpsKey, 0, this.batchSize - 1)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should call the callback with the unparsed ops', function () {
|
|
this.result.should.equal(this.rawUpdates)
|
|
})
|
|
})
|
|
|
|
describe('parseDocUpdates', function () {
|
|
it('should return the parsed ops', function () {
|
|
this.RedisManager.parseDocUpdates(this.rawUpdates).should.deep.equal(
|
|
this.updates
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('getUpdatesInBatches', function () {
|
|
beforeEach(function () {
|
|
this.runner = sinon.stub().resolves()
|
|
})
|
|
|
|
describe('single batch smaller than batch size', function () {
|
|
beforeEach(async function () {
|
|
this.rclient.lrange.resolves(this.rawUpdates)
|
|
this.batchSize = 3
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
this.batchSize,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a single batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledOnce
|
|
this.rclient.lrange.should.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
0,
|
|
this.batchSize - 1
|
|
)
|
|
})
|
|
|
|
it('calls the runner once', function () {
|
|
this.runner.callCount.should.equal(1)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner.calledWith(this.updates).should.equal(true)
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
for (const update of this.rawUpdates) {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
update
|
|
)
|
|
}
|
|
})
|
|
|
|
it('deletes the first op timestamp', function () {
|
|
expect(this.rclient.del).to.have.been.calledWith(
|
|
this.firstOpTimestampKey
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('single batch at batch size', function () {
|
|
beforeEach(async function () {
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves([])
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner once', function () {
|
|
this.runner.callCount.should.equal(1)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner.calledWith(this.updates).should.equal(true)
|
|
})
|
|
|
|
it('deletes the applied updates', function () {
|
|
for (const update of this.rawUpdates) {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
update
|
|
)
|
|
}
|
|
})
|
|
|
|
it('deletes the first op timestamp', function () {
|
|
expect(this.rclient.del).to.have.been.calledWith(
|
|
this.firstOpTimestampKey
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('single batch exceeding size limit on updates', function () {
|
|
beforeEach(async function () {
|
|
// set the threshold below the size of the first update
|
|
this.RedisManager.setRawUpdateSizeThreshold(
|
|
this.rawUpdates[0].length - 1
|
|
)
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))
|
|
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('deletes the first update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[0]
|
|
)
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the second set of applied updates', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[1]
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('two batches with first update below and second update above the size limit on updates', function () {
|
|
beforeEach(async function () {
|
|
// set the threshold above the size of the first update, but below the total size
|
|
this.RedisManager.setRawUpdateSizeThreshold(
|
|
this.rawUpdates[0].length + 1
|
|
)
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner.calledWith(this.updates.slice(0, 1)).should.equal(true)
|
|
})
|
|
|
|
it('deletes the first set applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[0]
|
|
)
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner.calledWith(this.updates.slice(1)).should.equal(true)
|
|
})
|
|
|
|
it('deletes the second applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[1]
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('single batch exceeding op count limit on updates', function () {
|
|
beforeEach(async function () {
|
|
// set the threshold below the size of the first update
|
|
this.RedisManager.setMaxUpdateOpLength(this.updates[0].op.length - 1)
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))
|
|
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first updates', function () {
|
|
this.runner.calledWith(this.updates.slice(0, 1)).should.equal(true)
|
|
})
|
|
|
|
it('deletes the first applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[0]
|
|
)
|
|
})
|
|
|
|
it('calls the runner with the second updates', function () {
|
|
this.runner.calledWith(this.updates.slice(1)).should.equal(true)
|
|
})
|
|
|
|
it('deletes the second applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[1]
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('single batch exceeding doc content count', function () {
|
|
beforeEach(async function () {
|
|
this.updates = [{ resyncDocContent: 123 }, { resyncDocContent: 456 }]
|
|
this.rawUpdates = this.updates.map(update => JSON.stringify(update))
|
|
// set the threshold below the size of the first update
|
|
this.RedisManager.setMaxNewDocContentCount(this.updates.length - 1)
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))
|
|
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('deletes the first applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[0]
|
|
)
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the second set of applied updates', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[1]
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('two batches with first update below and second update above the ops length limit on updates', function () {
|
|
beforeEach(async function () {
|
|
// set the threshold below the size of the first update
|
|
this.RedisManager.setMaxUpdateOpLength(this.updates[0].op.length + 1)
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves(this.rawUpdates.slice(1))
|
|
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the first update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(0, 1))
|
|
})
|
|
|
|
it('deletes the first applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[0]
|
|
)
|
|
})
|
|
|
|
it('calls the runner with the second update', function () {
|
|
this.runner.should.have.been.calledWith(this.updates.slice(1))
|
|
})
|
|
|
|
it('deletes the second applied update', function () {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
this.rawUpdates[1]
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('two batches', function () {
|
|
beforeEach(async function () {
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).resolves(this.extraRawUpdates)
|
|
await this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
})
|
|
|
|
it('requests a second batch of updates', function () {
|
|
this.rclient.lrange.should.have.been.calledTwice
|
|
})
|
|
|
|
it('calls the runner twice', function () {
|
|
this.runner.callCount.should.equal(2)
|
|
})
|
|
|
|
it('calls the runner with the updates', function () {
|
|
this.runner.should.have.been.calledWith(this.updates)
|
|
this.runner.should.have.been.calledWith(this.extraUpdates)
|
|
})
|
|
|
|
it('deletes the first set of applied updates', function () {
|
|
for (const update of this.rawUpdates) {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
update
|
|
)
|
|
}
|
|
})
|
|
|
|
it('deletes the second set of applied updates', function () {
|
|
for (const update of this.extraRawUpdates) {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
update
|
|
)
|
|
}
|
|
})
|
|
})
|
|
|
|
describe('error when first reading updates', function () {
|
|
beforeEach(async function () {
|
|
this.error = new Error('error')
|
|
this.rclient.lrange.rejects(this.error)
|
|
await expect(
|
|
this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
).to.be.rejected
|
|
})
|
|
|
|
it('does not delete any updates', function () {
|
|
expect(this.rclient.lrem).not.to.have.been.called
|
|
})
|
|
})
|
|
|
|
describe('error when reading updates for a second batch', function () {
|
|
beforeEach(async function () {
|
|
this.error = new Error('error')
|
|
this.rclient.lrange.onCall(0).resolves(this.rawUpdates)
|
|
this.rclient.lrange.onCall(1).rejects(this.error)
|
|
|
|
await expect(
|
|
this.RedisManager.promises.getUpdatesInBatches(
|
|
this.project_id,
|
|
2,
|
|
this.runner
|
|
)
|
|
).to.be.rejected
|
|
})
|
|
|
|
it('deletes the first set of applied updates', function () {
|
|
for (const update of this.rawUpdates) {
|
|
expect(this.rclient.lrem).to.have.been.calledWith(
|
|
this.historyOpsKey,
|
|
1,
|
|
update
|
|
)
|
|
}
|
|
})
|
|
|
|
it('deletes applied updates only once', function () {
|
|
expect(this.rclient.lrem.callCount).to.equal(this.rawUpdates.length)
|
|
})
|
|
})
|
|
})
|
|
})
|