overleaf/services/track-changes/test/unit/js/PackManager/PackManagerTests.js
2021-07-13 12:04:43 +01:00

704 lines
22 KiB
JavaScript

/* eslint-disable
mocha/no-identical-title,
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
const { assert, expect } = require('chai')
const modulePath = '../../../../app/js/PackManager.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb')
const _ = require('underscore')
const tk = require('timekeeper')
describe('PackManager', function () {
beforeEach(function () {
tk.freeze(new Date())
this.PackManager = SandboxedModule.require(modulePath, {
requires: {
bson: require('bson'),
'./mongodb': { db: (this.db = {}), ObjectId },
'./LockManager': {},
'./MongoAWS': {},
'@overleaf/metrics': { inc() {} },
'./ProjectIterator': require('../../../../app/js/ProjectIterator.js'), // Cache for speed
'@overleaf/settings': {
redis: { lock: { key_schema: {} } },
},
},
})
this.callback = sinon.stub()
this.doc_id = ObjectId().toString()
this.project_id = ObjectId().toString()
return (this.PackManager.MAX_COUNT = 512)
})
afterEach(function () {
return tk.reset()
})
describe('insertCompressedUpdates', function () {
beforeEach(function () {
this.lastUpdate = {
_id: '12345',
pack: [
{ op: 'op-1', meta: 'meta-1', v: 1 },
{ op: 'op-2', meta: 'meta-2', v: 2 },
],
n: 2,
sz: 100,
}
this.newUpdates = [
{ op: 'op-3', meta: 'meta-3', v: 3 },
{ op: 'op-4', meta: 'meta-4', v: 4 },
]
return (this.db.docHistory = {
insertOne: sinon.stub().yields(),
insert: sinon.stub().callsArg(1),
updateOne: sinon.stub().yields(),
findAndModify: sinon.stub().callsArg(1),
})
})
describe('with no last update', function () {
beforeEach(function () {
this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
return this.PackManager.insertCompressedUpdates(
this.project_id,
this.doc_id,
null,
this.newUpdates,
true,
this.callback
)
})
describe('for a small update', function () {
it('should insert the update into a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(this.project_id, this.doc_id, this.newUpdates, true)
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
return describe('for many small updates', function () {
beforeEach(function () {
this.newUpdates = __range__(0, 2048, true).map(i => ({
op: `op-${i}`,
meta: `meta-${i}`,
v: i,
}))
return this.PackManager.insertCompressedUpdates(
this.project_id,
this.doc_id,
null,
this.newUpdates,
false,
this.callback
)
})
it('should append the initial updates to the existing pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(0, 512),
false
)
.should.equal(true)
})
it('should insert the first set remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(512, 1024),
false
)
.should.equal(true)
})
it('should insert the second set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(1024, 1536),
false
)
.should.equal(true)
})
it('should insert the third set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(1536, 2048),
false
)
.should.equal(true)
})
it('should insert the final set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(2048, 2049),
false
)
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
})
describe('with an existing pack as the last update', function () {
beforeEach(function () {
this.PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5)
this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
return this.PackManager.insertCompressedUpdates(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
false,
this.callback
)
})
describe('for a small update', function () {
it('should append the update to the existing pack', function () {
return this.PackManager.appendUpdatesToExistingPack
.calledWith(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
false
)
.should.equal(true)
})
it('should not insert any new packs', function () {
return this.PackManager.insertUpdatesIntoNewPack.called.should.equal(
false
)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
describe('for many small updates', function () {
beforeEach(function () {
this.newUpdates = __range__(0, 2048, true).map(i => ({
op: `op-${i}`,
meta: `meta-${i}`,
v: i,
}))
return this.PackManager.insertCompressedUpdates(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
false,
this.callback
)
})
it('should append the initial updates to the existing pack', function () {
return this.PackManager.appendUpdatesToExistingPack
.calledWith(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates.slice(0, 510),
false
)
.should.equal(true)
})
it('should insert the first set remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(510, 1022),
false
)
.should.equal(true)
})
it('should insert the second set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(1022, 1534),
false
)
.should.equal(true)
})
it('should insert the third set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(1534, 2046),
false
)
.should.equal(true)
})
it('should insert the final set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(2046, 2049),
false
)
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
return describe('for many big updates', function () {
beforeEach(function () {
const longString = __range__(
0,
0.75 * this.PackManager.MAX_SIZE,
true
)
.map(j => 'a')
.join('')
this.newUpdates = [0, 1, 2, 3, 4].map(i => ({
op: `op-${i}-${longString}`,
meta: `meta-${i}`,
v: i,
}))
return this.PackManager.insertCompressedUpdates(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
false,
this.callback
)
})
it('should append the initial updates to the existing pack', function () {
return this.PackManager.appendUpdatesToExistingPack
.calledWith(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates.slice(0, 1),
false
)
.should.equal(true)
})
it('should insert the first set remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(1, 2),
false
)
.should.equal(true)
})
it('should insert the second set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(2, 3),
false
)
.should.equal(true)
})
it('should insert the third set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(3, 4),
false
)
.should.equal(true)
})
it('should insert the final set of remaining updates as a new pack', function () {
return this.PackManager.insertUpdatesIntoNewPack
.calledWith(
this.project_id,
this.doc_id,
this.newUpdates.slice(4, 5),
false
)
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
})
describe('flushCompressedUpdates', function () {
return describe('when there is no previous update', function () {
beforeEach(function () {
return this.PackManager.flushCompressedUpdates(
this.project_id,
this.doc_id,
null,
this.newUpdates,
true,
this.callback
)
})
return describe('for a small update that will expire', function () {
it('should insert the update into mongo', function () {
return this.db.docHistory.insertOne
.calledWithMatch({
pack: this.newUpdates,
project_id: ObjectId(this.project_id),
doc_id: ObjectId(this.doc_id),
n: this.newUpdates.length,
v: this.newUpdates[0].v,
v_end: this.newUpdates[this.newUpdates.length - 1].v,
})
.should.equal(true)
})
it('should set an expiry time in the future', function () {
return this.db.docHistory.insertOne
.calledWithMatch({
expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000),
})
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
})
})
describe('when there is a recent previous update in mongo that expires', function () {
beforeEach(function () {
this.lastUpdate = {
_id: '12345',
pack: [
{ op: 'op-1', meta: 'meta-1', v: 1 },
{ op: 'op-2', meta: 'meta-2', v: 2 },
],
n: 2,
sz: 100,
meta: { start_ts: Date.now() - 6 * 3600 * 1000 },
expiresAt: new Date(Date.now()),
}
return this.PackManager.flushCompressedUpdates(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
true,
this.callback
)
})
return describe('for a small update that will expire', function () {
it('should append the update in mongo', function () {
return this.db.docHistory.updateOne
.calledWithMatch(
{ _id: this.lastUpdate._id },
{
$push: { pack: { $each: this.newUpdates } },
$set: { v_end: this.newUpdates[this.newUpdates.length - 1].v },
}
)
.should.equal(true)
})
it('should set an expiry time in the future', function () {
return this.db.docHistory.updateOne
.calledWithMatch(sinon.match.any, {
$set: { expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) },
})
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
})
describe('when there is a recent previous update in mongo that expires', function () {
beforeEach(function () {
this.PackManager.updateIndex = sinon.stub().callsArg(2)
this.lastUpdate = {
_id: '12345',
pack: [
{ op: 'op-1', meta: 'meta-1', v: 1 },
{ op: 'op-2', meta: 'meta-2', v: 2 },
],
n: 2,
sz: 100,
meta: { start_ts: Date.now() - 6 * 3600 * 1000 },
expiresAt: new Date(Date.now()),
}
return this.PackManager.flushCompressedUpdates(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
false,
this.callback
)
})
return describe('for a small update that will not expire', function () {
it('should insert the update into mongo', function () {
return this.db.docHistory.insertOne
.calledWithMatch({
pack: this.newUpdates,
project_id: ObjectId(this.project_id),
doc_id: ObjectId(this.doc_id),
n: this.newUpdates.length,
v: this.newUpdates[0].v,
v_end: this.newUpdates[this.newUpdates.length - 1].v,
})
.should.equal(true)
})
it('should not set any expiry time', function () {
return this.db.docHistory.insertOne
.neverCalledWithMatch(sinon.match.has('expiresAt'))
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
})
return describe('when there is an old previous update in mongo', function () {
beforeEach(function () {
this.lastUpdate = {
_id: '12345',
pack: [
{ op: 'op-1', meta: 'meta-1', v: 1 },
{ op: 'op-2', meta: 'meta-2', v: 2 },
],
n: 2,
sz: 100,
meta: { start_ts: Date.now() - 30 * 24 * 3600 * 1000 },
expiresAt: new Date(Date.now() - 30 * 24 * 3600 * 1000),
}
return this.PackManager.flushCompressedUpdates(
this.project_id,
this.doc_id,
this.lastUpdate,
this.newUpdates,
true,
this.callback
)
})
return describe('for a small update that will expire', function () {
it('should insert the update into mongo', function () {
return this.db.docHistory.insertOne
.calledWithMatch({
pack: this.newUpdates,
project_id: ObjectId(this.project_id),
doc_id: ObjectId(this.doc_id),
n: this.newUpdates.length,
v: this.newUpdates[0].v,
v_end: this.newUpdates[this.newUpdates.length - 1].v,
})
.should.equal(true)
})
it('should set an expiry time in the future', function () {
return this.db.docHistory.insertOne
.calledWithMatch({
expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000),
})
.should.equal(true)
})
return it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
})
})
})
describe('getOpsByVersionRange', function () {})
describe('loadPacksByVersionRange', function () {})
describe('fetchPacksIfNeeded', function () {})
describe('makeProjectIterator', function () {})
describe('getPackById', function () {})
describe('increaseTTL', function () {})
describe('getIndex', function () {})
describe('getPackFromIndex', function () {})
// getLastPackFromIndex:
// getIndexWithKeys
// initialiseIndex
// updateIndex
// findCompletedPacks
// findUnindexedPacks
// insertPacksIntoIndexWithLock
// _insertPacksIntoIndex
// archivePack
// checkArchivedPack
// processOldPack
// updateIndexIfNeeded
// findUnarchivedPacks
return describe('checkArchiveNotInProgress', function () {
describe('when an archive is in progress', function () {
beforeEach(function () {
this.db.docHistoryIndex = {
findOne: sinon.stub().callsArgWith(2, null, { inS3: false }),
}
return this.PackManager.checkArchiveNotInProgress(
this.project_id,
this.doc_id,
this.pack_id,
this.callback
)
})
it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
return it('should return an error', function () {
return this.callback
.calledWith(sinon.match.has('message'))
.should.equal(true)
})
})
describe('when an archive is completed', function () {
beforeEach(function () {
this.db.docHistoryIndex = {
findOne: sinon.stub().callsArgWith(2, null, { inS3: true }),
}
return this.PackManager.checkArchiveNotInProgress(
this.project_id,
this.doc_id,
this.pack_id,
this.callback
)
})
it('should call the callback', function () {
return this.callback.called.should.equal(true)
})
return it('should return an error', function () {
return this.callback
.calledWith(sinon.match.has('message'))
.should.equal(true)
})
})
return describe('when the archive has not started or completed', function () {
beforeEach(function () {
this.db.docHistoryIndex = {
findOne: sinon.stub().callsArgWith(2, null, {}),
}
return this.PackManager.checkArchiveNotInProgress(
this.project_id,
this.doc_id,
this.pack_id,
this.callback
)
})
it('should call the callback with no error', function () {
return this.callback.called.should.equal(true)
})
return it('should return with no error', function () {
return (typeof this.callback.lastCall.args[0]).should.equal('undefined')
})
})
})
})
// describe "setTTLOnArchivedPack", ->
// beforeEach ->
// @pack_id = "somepackid"
// @onedayinms = 86400000
// @db.docHistory =
// findAndModify : sinon.stub().callsArgWith(1)
// it "should set expires to 1 day", (done)->
// #@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms)
// @PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, =>
// args = @db.docHistory.findAndModify.args[0][0]
// args.query._id.should.equal @pack_id
// args.update['$set'].expiresAt.should.equal @onedayinms
// done()
// describe "_getOneDayInFutureWithRandomDelay", ->
// beforeEach ->
// @onedayinms = 86400000
// @thirtyMins = 1000 * 60 * 30
// it "should give 1 day + 30 mins random time", (done)->
// loops = 10000
// while --loops > 0
// randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms)
// randomDelay.should.be.above(0)
// randomDelay.should.be.below(@thirtyMins + 1)
// done()
function __range__(left, right, inclusive) {
const range = []
const ascending = left < right
const end = !inclusive ? right : ascending ? right + 1 : right - 1
for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
range.push(i)
}
return range
}