mirror of
https://github.com/overleaf/overleaf.git
synced 2024-10-31 21:21:03 -04:00
704 lines
22 KiB
JavaScript
704 lines
22 KiB
JavaScript
/* eslint-disable
|
|
mocha/no-identical-title,
|
|
no-return-assign,
|
|
no-unused-vars,
|
|
*/
|
|
// TODO: This file was created by bulk-decaffeinate.
|
|
// Fix any style issues and re-enable lint.
|
|
/*
|
|
* decaffeinate suggestions:
|
|
* DS102: Remove unnecessary code created because of implicit returns
|
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
|
*/
|
|
const sinon = require('sinon')
|
|
const { assert, expect } = require('chai')
|
|
const modulePath = '../../../../app/js/PackManager.js'
|
|
const SandboxedModule = require('sandboxed-module')
|
|
const { ObjectId } = require('mongodb')
|
|
const _ = require('underscore')
|
|
|
|
const tk = require('timekeeper')
|
|
|
|
describe('PackManager', function () {
|
|
beforeEach(function () {
|
|
tk.freeze(new Date())
|
|
this.PackManager = SandboxedModule.require(modulePath, {
|
|
requires: {
|
|
bson: require('bson'),
|
|
'./mongodb': { db: (this.db = {}), ObjectId },
|
|
'./LockManager': {},
|
|
'./MongoAWS': {},
|
|
'@overleaf/metrics': { inc() {} },
|
|
'./ProjectIterator': require('../../../../app/js/ProjectIterator.js'), // Cache for speed
|
|
'@overleaf/settings': {
|
|
redis: { lock: { key_schema: {} } },
|
|
},
|
|
},
|
|
})
|
|
this.callback = sinon.stub()
|
|
this.doc_id = ObjectId().toString()
|
|
this.project_id = ObjectId().toString()
|
|
return (this.PackManager.MAX_COUNT = 512)
|
|
})
|
|
|
|
afterEach(function () {
|
|
return tk.reset()
|
|
})
|
|
|
|
describe('insertCompressedUpdates', function () {
|
|
beforeEach(function () {
|
|
this.lastUpdate = {
|
|
_id: '12345',
|
|
pack: [
|
|
{ op: 'op-1', meta: 'meta-1', v: 1 },
|
|
{ op: 'op-2', meta: 'meta-2', v: 2 },
|
|
],
|
|
n: 2,
|
|
sz: 100,
|
|
}
|
|
this.newUpdates = [
|
|
{ op: 'op-3', meta: 'meta-3', v: 3 },
|
|
{ op: 'op-4', meta: 'meta-4', v: 4 },
|
|
]
|
|
return (this.db.docHistory = {
|
|
insertOne: sinon.stub().yields(),
|
|
insert: sinon.stub().callsArg(1),
|
|
updateOne: sinon.stub().yields(),
|
|
findAndModify: sinon.stub().callsArg(1),
|
|
})
|
|
})
|
|
|
|
describe('with no last update', function () {
|
|
beforeEach(function () {
|
|
this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
|
|
return this.PackManager.insertCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
null,
|
|
this.newUpdates,
|
|
true,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
describe('for a small update', function () {
|
|
it('should insert the update into a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(this.project_id, this.doc_id, this.newUpdates, true)
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
|
|
return describe('for many small updates', function () {
|
|
beforeEach(function () {
|
|
this.newUpdates = __range__(0, 2048, true).map(i => ({
|
|
op: `op-${i}`,
|
|
meta: `meta-${i}`,
|
|
v: i,
|
|
}))
|
|
return this.PackManager.insertCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
null,
|
|
this.newUpdates,
|
|
false,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
it('should append the initial updates to the existing pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(0, 512),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the first set remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(512, 1024),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the second set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(1024, 1536),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the third set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(1536, 2048),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the final set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(2048, 2049),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('with an existing pack as the last update', function () {
|
|
beforeEach(function () {
|
|
this.PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5)
|
|
this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4)
|
|
return this.PackManager.insertCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
false,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
describe('for a small update', function () {
|
|
it('should append the update to the existing pack', function () {
|
|
return this.PackManager.appendUpdatesToExistingPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
it('should not insert any new packs', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack.called.should.equal(
|
|
false
|
|
)
|
|
})
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
|
|
describe('for many small updates', function () {
|
|
beforeEach(function () {
|
|
this.newUpdates = __range__(0, 2048, true).map(i => ({
|
|
op: `op-${i}`,
|
|
meta: `meta-${i}`,
|
|
v: i,
|
|
}))
|
|
return this.PackManager.insertCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
false,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
it('should append the initial updates to the existing pack', function () {
|
|
return this.PackManager.appendUpdatesToExistingPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates.slice(0, 510),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the first set remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(510, 1022),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the second set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(1022, 1534),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the third set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(1534, 2046),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the final set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(2046, 2049),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
|
|
return describe('for many big updates', function () {
|
|
beforeEach(function () {
|
|
const longString = __range__(
|
|
0,
|
|
0.75 * this.PackManager.MAX_SIZE,
|
|
true
|
|
)
|
|
.map(j => 'a')
|
|
.join('')
|
|
this.newUpdates = [0, 1, 2, 3, 4].map(i => ({
|
|
op: `op-${i}-${longString}`,
|
|
meta: `meta-${i}`,
|
|
v: i,
|
|
}))
|
|
return this.PackManager.insertCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
false,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
it('should append the initial updates to the existing pack', function () {
|
|
return this.PackManager.appendUpdatesToExistingPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates.slice(0, 1),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the first set remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(1, 2),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the second set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(2, 3),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the third set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(3, 4),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should insert the final set of remaining updates as a new pack', function () {
|
|
return this.PackManager.insertUpdatesIntoNewPack
|
|
.calledWith(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.newUpdates.slice(4, 5),
|
|
false
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('flushCompressedUpdates', function () {
|
|
return describe('when there is no previous update', function () {
|
|
beforeEach(function () {
|
|
return this.PackManager.flushCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
null,
|
|
this.newUpdates,
|
|
true,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
return describe('for a small update that will expire', function () {
|
|
it('should insert the update into mongo', function () {
|
|
return this.db.docHistory.insertOne
|
|
.calledWithMatch({
|
|
pack: this.newUpdates,
|
|
project_id: ObjectId(this.project_id),
|
|
doc_id: ObjectId(this.doc_id),
|
|
n: this.newUpdates.length,
|
|
v: this.newUpdates[0].v,
|
|
v_end: this.newUpdates[this.newUpdates.length - 1].v,
|
|
})
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should set an expiry time in the future', function () {
|
|
return this.db.docHistory.insertOne
|
|
.calledWithMatch({
|
|
expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000),
|
|
})
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('when there is a recent previous update in mongo that expires', function () {
|
|
beforeEach(function () {
|
|
this.lastUpdate = {
|
|
_id: '12345',
|
|
pack: [
|
|
{ op: 'op-1', meta: 'meta-1', v: 1 },
|
|
{ op: 'op-2', meta: 'meta-2', v: 2 },
|
|
],
|
|
n: 2,
|
|
sz: 100,
|
|
meta: { start_ts: Date.now() - 6 * 3600 * 1000 },
|
|
expiresAt: new Date(Date.now()),
|
|
}
|
|
|
|
return this.PackManager.flushCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
true,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
return describe('for a small update that will expire', function () {
|
|
it('should append the update in mongo', function () {
|
|
return this.db.docHistory.updateOne
|
|
.calledWithMatch(
|
|
{ _id: this.lastUpdate._id },
|
|
{
|
|
$push: { pack: { $each: this.newUpdates } },
|
|
$set: { v_end: this.newUpdates[this.newUpdates.length - 1].v },
|
|
}
|
|
)
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should set an expiry time in the future', function () {
|
|
return this.db.docHistory.updateOne
|
|
.calledWithMatch(sinon.match.any, {
|
|
$set: { expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) },
|
|
})
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('when there is a recent previous update in mongo that expires', function () {
|
|
beforeEach(function () {
|
|
this.PackManager.updateIndex = sinon.stub().callsArg(2)
|
|
|
|
this.lastUpdate = {
|
|
_id: '12345',
|
|
pack: [
|
|
{ op: 'op-1', meta: 'meta-1', v: 1 },
|
|
{ op: 'op-2', meta: 'meta-2', v: 2 },
|
|
],
|
|
n: 2,
|
|
sz: 100,
|
|
meta: { start_ts: Date.now() - 6 * 3600 * 1000 },
|
|
expiresAt: new Date(Date.now()),
|
|
}
|
|
|
|
return this.PackManager.flushCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
false,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
return describe('for a small update that will not expire', function () {
|
|
it('should insert the update into mongo', function () {
|
|
return this.db.docHistory.insertOne
|
|
.calledWithMatch({
|
|
pack: this.newUpdates,
|
|
project_id: ObjectId(this.project_id),
|
|
doc_id: ObjectId(this.doc_id),
|
|
n: this.newUpdates.length,
|
|
v: this.newUpdates[0].v,
|
|
v_end: this.newUpdates[this.newUpdates.length - 1].v,
|
|
})
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should not set any expiry time', function () {
|
|
return this.db.docHistory.insertOne
|
|
.neverCalledWithMatch(sinon.match.has('expiresAt'))
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
})
|
|
|
|
return describe('when there is an old previous update in mongo', function () {
|
|
beforeEach(function () {
|
|
this.lastUpdate = {
|
|
_id: '12345',
|
|
pack: [
|
|
{ op: 'op-1', meta: 'meta-1', v: 1 },
|
|
{ op: 'op-2', meta: 'meta-2', v: 2 },
|
|
],
|
|
n: 2,
|
|
sz: 100,
|
|
meta: { start_ts: Date.now() - 30 * 24 * 3600 * 1000 },
|
|
expiresAt: new Date(Date.now() - 30 * 24 * 3600 * 1000),
|
|
}
|
|
|
|
return this.PackManager.flushCompressedUpdates(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.lastUpdate,
|
|
this.newUpdates,
|
|
true,
|
|
this.callback
|
|
)
|
|
})
|
|
|
|
return describe('for a small update that will expire', function () {
|
|
it('should insert the update into mongo', function () {
|
|
return this.db.docHistory.insertOne
|
|
.calledWithMatch({
|
|
pack: this.newUpdates,
|
|
project_id: ObjectId(this.project_id),
|
|
doc_id: ObjectId(this.doc_id),
|
|
n: this.newUpdates.length,
|
|
v: this.newUpdates[0].v,
|
|
v_end: this.newUpdates[this.newUpdates.length - 1].v,
|
|
})
|
|
.should.equal(true)
|
|
})
|
|
|
|
it('should set an expiry time in the future', function () {
|
|
return this.db.docHistory.insertOne
|
|
.calledWithMatch({
|
|
expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000),
|
|
})
|
|
.should.equal(true)
|
|
})
|
|
|
|
return it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('getOpsByVersionRange', function () {})
|
|
|
|
describe('loadPacksByVersionRange', function () {})
|
|
|
|
describe('fetchPacksIfNeeded', function () {})
|
|
|
|
describe('makeProjectIterator', function () {})
|
|
|
|
describe('getPackById', function () {})
|
|
|
|
describe('increaseTTL', function () {})
|
|
|
|
describe('getIndex', function () {})
|
|
|
|
describe('getPackFromIndex', function () {})
|
|
// getLastPackFromIndex:
|
|
// getIndexWithKeys
|
|
// initialiseIndex
|
|
// updateIndex
|
|
// findCompletedPacks
|
|
// findUnindexedPacks
|
|
// insertPacksIntoIndexWithLock
|
|
// _insertPacksIntoIndex
|
|
// archivePack
|
|
// checkArchivedPack
|
|
// processOldPack
|
|
// updateIndexIfNeeded
|
|
// findUnarchivedPacks
|
|
|
|
return describe('checkArchiveNotInProgress', function () {
|
|
describe('when an archive is in progress', function () {
|
|
beforeEach(function () {
|
|
this.db.docHistoryIndex = {
|
|
findOne: sinon.stub().callsArgWith(2, null, { inS3: false }),
|
|
}
|
|
return this.PackManager.checkArchiveNotInProgress(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.pack_id,
|
|
this.callback
|
|
)
|
|
})
|
|
it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
return it('should return an error', function () {
|
|
return this.callback
|
|
.calledWith(sinon.match.has('message'))
|
|
.should.equal(true)
|
|
})
|
|
})
|
|
|
|
describe('when an archive is completed', function () {
|
|
beforeEach(function () {
|
|
this.db.docHistoryIndex = {
|
|
findOne: sinon.stub().callsArgWith(2, null, { inS3: true }),
|
|
}
|
|
return this.PackManager.checkArchiveNotInProgress(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.pack_id,
|
|
this.callback
|
|
)
|
|
})
|
|
it('should call the callback', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
return it('should return an error', function () {
|
|
return this.callback
|
|
.calledWith(sinon.match.has('message'))
|
|
.should.equal(true)
|
|
})
|
|
})
|
|
|
|
return describe('when the archive has not started or completed', function () {
|
|
beforeEach(function () {
|
|
this.db.docHistoryIndex = {
|
|
findOne: sinon.stub().callsArgWith(2, null, {}),
|
|
}
|
|
return this.PackManager.checkArchiveNotInProgress(
|
|
this.project_id,
|
|
this.doc_id,
|
|
this.pack_id,
|
|
this.callback
|
|
)
|
|
})
|
|
it('should call the callback with no error', function () {
|
|
return this.callback.called.should.equal(true)
|
|
})
|
|
return it('should return with no error', function () {
|
|
return (typeof this.callback.lastCall.args[0]).should.equal('undefined')
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
// describe "setTTLOnArchivedPack", ->
|
|
// beforeEach ->
|
|
// @pack_id = "somepackid"
|
|
// @onedayinms = 86400000
|
|
// @db.docHistory =
|
|
// findAndModify : sinon.stub().callsArgWith(1)
|
|
|
|
// it "should set expires to 1 day", (done)->
|
|
// #@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms)
|
|
// @PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, =>
|
|
// args = @db.docHistory.findAndModify.args[0][0]
|
|
// args.query._id.should.equal @pack_id
|
|
// args.update['$set'].expiresAt.should.equal @onedayinms
|
|
// done()
|
|
|
|
// describe "_getOneDayInFutureWithRandomDelay", ->
|
|
// beforeEach ->
|
|
// @onedayinms = 86400000
|
|
// @thirtyMins = 1000 * 60 * 30
|
|
|
|
// it "should give 1 day + 30 mins random time", (done)->
|
|
// loops = 10000
|
|
// while --loops > 0
|
|
// randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms)
|
|
// randomDelay.should.be.above(0)
|
|
// randomDelay.should.be.below(@thirtyMins + 1)
|
|
// done()
|
|
|
|
function __range__(left, right, inclusive) {
|
|
const range = []
|
|
const ascending = left < right
|
|
const end = !inclusive ? right : ascending ? right + 1 : right - 1
|
|
for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) {
|
|
range.push(i)
|
|
}
|
|
return range
|
|
}
|