overleaf/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
Eric Mc Sween 2f513700ec Merge pull request #17010 from overleaf/em-doc-length
Add history_doc_length property to history updates

GitOrigin-RevId: ccad09f23ae9c038480fb7228a987d8fc6fb6274
2024-02-14 09:06:18 +00:00

735 lines
22 KiB
JavaScript

// @ts-check
const sinon = require('sinon')
const { expect } = require('chai')
const SandboxedModule = require('sandboxed-module')
const MODULE_PATH = '../../../../app/js/UpdateManager.js'
describe('UpdateManager', function () {
beforeEach(function () {
this.project_id = 'project-id-123'
this.projectHistoryId = 'history-id-123'
this.doc_id = 'document-id-123'
this.lockValue = 'mock-lock-value'
this.pathname = '/a/b/c.tex'
this.Metrics = {
inc: sinon.stub(),
Timer: class Timer {},
}
this.Metrics.Timer.prototype.done = sinon.stub()
this.Profiler = class Profiler {}
this.Profiler.prototype.log = sinon.stub().returns({ end: sinon.stub() })
this.Profiler.prototype.end = sinon.stub()
this.LockManager = {
promises: {
tryLock: sinon.stub().resolves(this.lockValue),
getLock: sinon.stub().resolves(this.lockValue),
releaseLock: sinon.stub().resolves(),
},
}
this.RedisManager = {
promises: {
setDocument: sinon.stub().resolves(),
updateDocument: sinon.stub(),
},
}
this.RealTimeRedisManager = {
sendData: sinon.stub(),
promises: {
getUpdatesLength: sinon.stub(),
getPendingUpdatesForDoc: sinon.stub(),
},
}
this.ShareJsUpdateManager = {
promises: {
applyUpdate: sinon.stub(),
},
}
this.HistoryManager = {
recordAndFlushHistoryOps: sinon.stub(),
}
this.Settings = {}
this.DocumentManager = {
promises: {
getDoc: sinon.stub(),
},
}
this.RangesManager = {
applyUpdate: sinon.stub(),
}
this.SnapshotManager = {
promises: {
recordSnapshot: sinon.stub().resolves(),
},
}
this.ProjectHistoryRedisManager = {
promises: {
queueOps: sinon
.stub()
.callsFake(async (projectId, ...ops) => ops.length),
},
}
this.UpdateManager = SandboxedModule.require(MODULE_PATH, {
requires: {
'./LockManager': this.LockManager,
'./RedisManager': this.RedisManager,
'./RealTimeRedisManager': this.RealTimeRedisManager,
'./ShareJsUpdateManager': this.ShareJsUpdateManager,
'./HistoryManager': this.HistoryManager,
'./Metrics': this.Metrics,
'@overleaf/settings': this.Settings,
'./DocumentManager': this.DocumentManager,
'./RangesManager': this.RangesManager,
'./SnapshotManager': this.SnapshotManager,
'./Profiler': this.Profiler,
'./ProjectHistoryRedisManager': this.ProjectHistoryRedisManager,
},
})
})
describe('processOutstandingUpdates', function () {
beforeEach(async function () {
this.UpdateManager.promises.fetchAndApplyUpdates = sinon.stub().resolves()
await this.UpdateManager.promises.processOutstandingUpdates(
this.project_id,
this.doc_id
)
})
it('should apply the updates', function () {
this.UpdateManager.promises.fetchAndApplyUpdates
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should time the execution', function () {
this.Metrics.Timer.prototype.done.called.should.equal(true)
})
})
describe('processOutstandingUpdatesWithLock', function () {
describe('when the lock is free', function () {
beforeEach(function () {
this.UpdateManager.promises.continueProcessingUpdatesWithLock = sinon
.stub()
.resolves()
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.resolves()
})
describe('successfully', function () {
beforeEach(async function () {
await this.UpdateManager.promises.processOutstandingUpdatesWithLock(
this.project_id,
this.doc_id
)
})
it('should acquire the lock', function () {
this.LockManager.promises.tryLock
.calledWith(this.doc_id)
.should.equal(true)
})
it('should free the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
it('should process the outstanding updates', function () {
this.UpdateManager.promises.processOutstandingUpdates
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should do everything with the lock acquired', function () {
this.UpdateManager.promises.processOutstandingUpdates
.calledAfter(this.LockManager.promises.tryLock)
.should.equal(true)
this.UpdateManager.promises.processOutstandingUpdates
.calledBefore(this.LockManager.promises.releaseLock)
.should.equal(true)
})
it('should continue processing new updates that may have come in', function () {
this.UpdateManager.promises.continueProcessingUpdatesWithLock
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
})
describe('when processOutstandingUpdates returns an error', function () {
beforeEach(async function () {
this.error = new Error('Something went wrong')
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.rejects(this.error)
await expect(
this.UpdateManager.promises.processOutstandingUpdatesWithLock(
this.project_id,
this.doc_id
)
).to.be.rejectedWith(this.error)
})
it('should free the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
})
})
describe('when the lock is taken', function () {
beforeEach(async function () {
this.LockManager.promises.tryLock.resolves(null)
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.resolves()
await this.UpdateManager.promises.processOutstandingUpdatesWithLock(
this.project_id,
this.doc_id
)
})
it('should not process the updates', function () {
this.UpdateManager.promises.processOutstandingUpdates.called.should.equal(
false
)
})
})
})
describe('continueProcessingUpdatesWithLock', function () {
describe('when there are outstanding updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(3)
this.UpdateManager.promises.processOutstandingUpdatesWithLock = sinon
.stub()
.resolves()
await this.UpdateManager.promises.continueProcessingUpdatesWithLock(
this.project_id,
this.doc_id
)
})
it('should process the outstanding updates', function () {
this.UpdateManager.promises.processOutstandingUpdatesWithLock
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
})
describe('when there are no outstanding updates', function () {
beforeEach(async function () {
this.RealTimeRedisManager.promises.getUpdatesLength.resolves(0)
this.UpdateManager.promises.processOutstandingUpdatesWithLock = sinon
.stub()
.resolves()
await this.UpdateManager.promises.continueProcessingUpdatesWithLock(
this.project_id,
this.doc_id
)
})
it('should not try to process the outstanding updates', function () {
this.UpdateManager.promises.processOutstandingUpdatesWithLock.called.should.equal(
false
)
})
})
})
describe('fetchAndApplyUpdates', function () {
describe('with updates', function () {
beforeEach(async function () {
this.updates = [{ p: 1, t: 'foo' }]
this.updatedDocLines = ['updated', 'lines']
this.version = 34
this.RealTimeRedisManager.promises.getPendingUpdatesForDoc.resolves(
this.updates
)
this.UpdateManager.promises.applyUpdate = sinon.stub().resolves()
await this.UpdateManager.promises.fetchAndApplyUpdates(
this.project_id,
this.doc_id
)
})
it('should get the pending updates', function () {
this.RealTimeRedisManager.promises.getPendingUpdatesForDoc
.calledWith(this.doc_id)
.should.equal(true)
})
it('should apply the updates', function () {
this.updates.map(update =>
this.UpdateManager.promises.applyUpdate
.calledWith(this.project_id, this.doc_id, update)
.should.equal(true)
)
})
})
describe('when there are no updates', function () {
beforeEach(async function () {
this.updates = []
this.RealTimeRedisManager.promises.getPendingUpdatesForDoc.resolves(
this.updates
)
this.UpdateManager.promises.applyUpdate = sinon.stub().resolves()
await this.UpdateManager.promises.fetchAndApplyUpdates(
this.project_id,
this.doc_id
)
})
it('should not call applyUpdate', function () {
this.UpdateManager.promises.applyUpdate.called.should.equal(false)
})
})
})
describe('applyUpdate', function () {
beforeEach(function () {
this.updateMeta = { user_id: 'last-author-fake-id' }
this.update = { op: [{ p: 42, i: 'foo' }], meta: this.updateMeta }
this.updatedDocLines = ['updated', 'lines']
this.version = 34
this.lines = ['original', 'lines']
this.ranges = { entries: 'mock', comments: 'mock' }
this.updated_ranges = { entries: 'updated', comments: 'updated' }
this.appliedOps = [
{ v: 42, op: 'mock-op-42' },
{ v: 45, op: 'mock-op-45' },
]
this.historyUpdates = [
'history-update-1',
'history-update-2',
'history-update-3',
]
this.project_ops_length = 123
this.DocumentManager.promises.getDoc.resolves({
lines: this.lines,
version: this.version,
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
historyRangesSupport: false,
})
this.RangesManager.applyUpdate.returns({
newRanges: this.updated_ranges,
rangesWereCollapsed: false,
historyUpdates: this.historyUpdates,
})
this.ShareJsUpdateManager.promises.applyUpdate = sinon.stub().resolves({
updatedDocLines: this.updatedDocLines,
version: this.version,
appliedOps: this.appliedOps,
})
this.RedisManager.promises.updateDocument.resolves()
this.UpdateManager.promises._addMetadataToHistoryUpdates = sinon.stub()
})
describe('normally', function () {
beforeEach(async function () {
await this.UpdateManager.promises.applyUpdate(
this.project_id,
this.doc_id,
this.update
)
})
it('should apply the updates via ShareJS', function () {
this.ShareJsUpdateManager.promises.applyUpdate
.calledWith(
this.project_id,
this.doc_id,
this.update,
this.lines,
this.version
)
.should.equal(true)
})
it('should update the ranges', function () {
this.RangesManager.applyUpdate
.calledWith(
this.project_id,
this.doc_id,
this.ranges,
this.appliedOps,
this.updatedDocLines
)
.should.equal(true)
})
it('should save the document', function () {
this.RedisManager.promises.updateDocument
.calledWith(
this.project_id,
this.doc_id,
this.updatedDocLines,
this.version,
this.appliedOps,
this.updated_ranges,
this.updateMeta
)
.should.equal(true)
})
it('should add metadata to the ops', function () {
this.UpdateManager.promises._addMetadataToHistoryUpdates.should.have.been.calledWith(
this.appliedOps,
this.pathname,
this.projectHistoryId,
this.lines
)
})
it('should push the applied ops into the history queue', function () {
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWith(
this.project_id,
...this.appliedOps.map(op => JSON.stringify(op))
)
this.HistoryManager.recordAndFlushHistoryOps.should.have.been.calledWith(
this.project_id,
this.appliedOps,
this.appliedOps.length
)
})
})
describe('with UTF-16 surrogate pairs in the update', function () {
beforeEach(async function () {
this.update = { op: [{ p: 42, i: '\uD835\uDC00' }] }
await this.UpdateManager.promises.applyUpdate(
this.project_id,
this.doc_id,
this.update
)
})
it('should apply the update but with surrogate pairs removed', function () {
this.ShareJsUpdateManager.promises.applyUpdate
.calledWith(this.project_id, this.doc_id, this.update)
.should.equal(true)
// \uFFFD is 'replacement character'
this.update.op[0].i.should.equal('\uFFFD\uFFFD')
})
})
describe('with an error', function () {
beforeEach(async function () {
this.error = new Error('something went wrong')
this.ShareJsUpdateManager.promises.applyUpdate.rejects(this.error)
await expect(
this.UpdateManager.promises.applyUpdate(
this.project_id,
this.doc_id,
this.update
)
).to.be.rejectedWith(this.error)
})
it('should call RealTimeRedisManager.sendData with the error', function () {
this.RealTimeRedisManager.sendData
.calledWith({
project_id: this.project_id,
doc_id: this.doc_id,
error: this.error.message,
})
.should.equal(true)
})
})
describe('when ranges get collapsed', function () {
beforeEach(async function () {
this.RangesManager.applyUpdate.returns({
newRanges: this.updated_ranges,
rangesWereCollapsed: true,
historyUpdates: this.historyUpdates,
})
await this.UpdateManager.promises.applyUpdate(
this.project_id,
this.doc_id,
this.update
)
})
it('should increment the doc-snapshot metric', function () {
this.Metrics.inc.calledWith('doc-snapshot').should.equal(true)
})
it('should call SnapshotManager.recordSnapshot', function () {
this.SnapshotManager.promises.recordSnapshot
.calledWith(
this.project_id,
this.doc_id,
this.version,
this.pathname,
this.lines,
this.ranges
)
.should.equal(true)
})
})
describe('when history ranges are supported', function () {
beforeEach(async function () {
this.DocumentManager.promises.getDoc.resolves({
lines: this.lines,
version: this.version,
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
historyRangesSupport: true,
})
await this.UpdateManager.promises.applyUpdate(
this.project_id,
this.doc_id,
this.update
)
})
it('should push the history updates into the history queue', function () {
this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWith(
this.project_id,
...this.historyUpdates.map(op => JSON.stringify(op))
)
this.HistoryManager.recordAndFlushHistoryOps.should.have.been.calledWith(
this.project_id,
this.historyUpdates,
this.historyUpdates.length
)
})
})
})
describe('_addMetadataToHistoryUpdates', function () {
it('should add projectHistoryId, pathname and doc_length metadata to the ops', function () {
const lines = ['some', 'test', 'data']
const historyUpdates = [
{
v: 42,
op: [
{ i: 'bing', p: 12, trackedDeleteRejection: true },
{ i: 'foo', p: 4 },
{ i: 'bar', p: 6 },
],
},
{
v: 45,
op: [
{ d: 'qux', p: 4 },
{ i: 'bazbaz', p: 14 },
{ d: 'bong', p: 28, u: true },
],
meta: {
tc: 'tracking-info',
},
},
{
v: 47,
op: [{ d: 'so', p: 0 }],
},
{ v: 49, op: [{ i: 'penguin', p: 18 }] },
]
const ranges = {
changes: [
{ op: { d: 'bingbong', p: 12 } },
{ op: { i: 'test', p: 5 } },
],
}
this.UpdateManager._addMetadataToHistoryUpdates(
historyUpdates,
this.pathname,
this.projectHistoryId,
lines,
ranges,
true
)
historyUpdates.should.deep.equal([
{
projectHistoryId: this.projectHistoryId,
v: 42,
op: [
{ i: 'bing', p: 12, trackedDeleteRejection: true },
{ i: 'foo', p: 4 },
{ i: 'bar', p: 6 },
],
meta: {
pathname: this.pathname,
doc_length: 14,
history_doc_length: 22,
},
},
{
projectHistoryId: this.projectHistoryId,
v: 45,
op: [
{ d: 'qux', p: 4 },
{ i: 'bazbaz', p: 14 },
{ d: 'bong', p: 28, u: true },
],
meta: {
pathname: this.pathname,
doc_length: 24, // 14 + 'bing' + 'foo' + 'bar'
history_doc_length: 28, // 22 + 'foo' + 'bar'
tc: 'tracking-info',
},
},
{
projectHistoryId: this.projectHistoryId,
v: 47,
op: [{ d: 'so', p: 0 }],
meta: {
pathname: this.pathname,
doc_length: 23, // 24 - 'qux' + 'bazbaz' - 'bong'
history_doc_length: 30, // 28 - 'bong' + 'bazbaz'
},
},
{
projectHistoryId: this.projectHistoryId,
v: 49,
op: [{ i: 'penguin', p: 18 }],
meta: {
pathname: this.pathname,
doc_length: 21, // 23 - 'so'
history_doc_length: 28, // 30 - 'so'
},
},
])
})
})
describe('lockUpdatesAndDo', function () {
beforeEach(function () {
this.method = sinon
.stub()
.yields(null, this.response_arg1, this.response_arg2)
this.arg1 = 'argument 1'
this.response_arg1 = 'response argument 1'
this.response_arg2 = 'response argument 2'
})
describe('successfully', function () {
beforeEach(async function () {
this.UpdateManager.promises.continueProcessingUpdatesWithLock = sinon
.stub()
.resolves()
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.resolves()
this.response = await this.UpdateManager.promises.lockUpdatesAndDo(
this.method,
this.project_id,
this.doc_id,
this.arg1
)
})
it('should lock the doc', function () {
this.LockManager.promises.getLock
.calledWith(this.doc_id)
.should.equal(true)
})
it('should process any outstanding updates', function () {
this.UpdateManager.promises.processOutstandingUpdates.should.have.been.calledWith(
this.project_id,
this.doc_id
)
})
it('should call the method', function () {
this.method
.calledWith(this.project_id, this.doc_id, this.arg1)
.should.equal(true)
})
it('should return the method response arguments', function () {
expect(this.response).to.deep.equal([
this.response_arg1,
this.response_arg2,
])
})
it('should release the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
it('should continue processing updates', function () {
this.UpdateManager.promises.continueProcessingUpdatesWithLock
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
})
describe('when processOutstandingUpdates returns an error', function () {
beforeEach(async function () {
this.error = new Error('Something went wrong')
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.rejects(this.error)
await expect(
this.UpdateManager.promises.lockUpdatesAndDo(
this.method,
this.project_id,
this.doc_id,
this.arg1
)
).to.be.rejectedWith(this.error)
})
it('should free the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
})
describe('when the method returns an error', function () {
beforeEach(async function () {
this.error = new Error('something went wrong')
this.UpdateManager.promises.processOutstandingUpdates = sinon
.stub()
.resolves()
this.method = sinon.stub().yields(this.error)
await expect(
this.UpdateManager.promises.lockUpdatesAndDo(
this.method,
this.project_id,
this.doc_id,
this.arg1
)
).to.be.rejectedWith(this.error)
})
it('should free the lock', function () {
this.LockManager.promises.releaseLock
.calledWith(this.doc_id, this.lockValue)
.should.equal(true)
})
})
})
})