/* eslint-disable mocha/no-identical-title, no-return-assign, no-unused-vars, */ // TODO: This file was created by bulk-decaffeinate. // Fix any style issues and re-enable lint. /* * decaffeinate suggestions: * DS102: Remove unnecessary code created because of implicit returns * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md */ const sinon = require('sinon') const { assert, expect } = require('chai') const modulePath = '../../../../app/js/PackManager.js' const SandboxedModule = require('sandboxed-module') const { ObjectId } = require('mongodb') const _ = require('underscore') const tk = require('timekeeper') describe('PackManager', function () { beforeEach(function () { tk.freeze(new Date()) this.PackManager = SandboxedModule.require(modulePath, { requires: { bson: require('bson'), './mongodb': { db: (this.db = {}), ObjectId }, './LockManager': {}, './MongoAWS': {}, '@overleaf/metrics': { inc() {} }, './ProjectIterator': require('../../../../app/js/ProjectIterator.js'), // Cache for speed '@overleaf/settings': { redis: { lock: { key_schema: {} } } } } }) this.callback = sinon.stub() this.doc_id = ObjectId().toString() this.project_id = ObjectId().toString() return (this.PackManager.MAX_COUNT = 512) }) afterEach(function () { return tk.reset() }) describe('insertCompressedUpdates', function () { beforeEach(function () { this.lastUpdate = { _id: '12345', pack: [ { op: 'op-1', meta: 'meta-1', v: 1 }, { op: 'op-2', meta: 'meta-2', v: 2 } ], n: 2, sz: 100 } this.newUpdates = [ { op: 'op-3', meta: 'meta-3', v: 3 }, { op: 'op-4', meta: 'meta-4', v: 4 } ] return (this.db.docHistory = { insertOne: sinon.stub().yields(), insert: sinon.stub().callsArg(1), updateOne: sinon.stub().yields(), findAndModify: sinon.stub().callsArg(1) }) }) describe('with no last update', function () { beforeEach(function () { this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4) return this.PackManager.insertCompressedUpdates( this.project_id, this.doc_id, null, this.newUpdates, true, this.callback ) }) describe('for a small update', function () { it('should insert the update into a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith(this.project_id, this.doc_id, this.newUpdates, true) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) return describe('for many small updates', function () { beforeEach(function () { this.newUpdates = __range__(0, 2048, true).map((i) => ({ op: `op-${i}`, meta: `meta-${i}`, v: i })) return this.PackManager.insertCompressedUpdates( this.project_id, this.doc_id, null, this.newUpdates, false, this.callback ) }) it('should append the initial updates to the existing pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(0, 512), false ) .should.equal(true) }) it('should insert the first set remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(512, 1024), false ) .should.equal(true) }) it('should insert the second set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(1024, 1536), false ) .should.equal(true) }) it('should insert the third set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(1536, 2048), false ) .should.equal(true) }) it('should insert the final set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(2048, 2049), false ) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) }) describe('with an existing pack as the last update', function () { beforeEach(function () { this.PackManager.appendUpdatesToExistingPack = sinon.stub().callsArg(5) this.PackManager.insertUpdatesIntoNewPack = sinon.stub().callsArg(4) return this.PackManager.insertCompressedUpdates( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback ) }) describe('for a small update', function () { it('should append the update to the existing pack', function () { return this.PackManager.appendUpdatesToExistingPack .calledWith( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false ) .should.equal(true) }) it('should not insert any new packs', function () { return this.PackManager.insertUpdatesIntoNewPack.called.should.equal( false ) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) describe('for many small updates', function () { beforeEach(function () { this.newUpdates = __range__(0, 2048, true).map((i) => ({ op: `op-${i}`, meta: `meta-${i}`, v: i })) return this.PackManager.insertCompressedUpdates( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback ) }) it('should append the initial updates to the existing pack', function () { return this.PackManager.appendUpdatesToExistingPack .calledWith( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates.slice(0, 510), false ) .should.equal(true) }) it('should insert the first set remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(510, 1022), false ) .should.equal(true) }) it('should insert the second set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(1022, 1534), false ) .should.equal(true) }) it('should insert the third set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(1534, 2046), false ) .should.equal(true) }) it('should insert the final set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(2046, 2049), false ) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) return describe('for many big updates', function () { beforeEach(function () { const longString = __range__( 0, 0.75 * this.PackManager.MAX_SIZE, true ) .map((j) => 'a') .join('') this.newUpdates = [0, 1, 2, 3, 4].map((i) => ({ op: `op-${i}-${longString}`, meta: `meta-${i}`, v: i })) return this.PackManager.insertCompressedUpdates( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback ) }) it('should append the initial updates to the existing pack', function () { return this.PackManager.appendUpdatesToExistingPack .calledWith( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates.slice(0, 1), false ) .should.equal(true) }) it('should insert the first set remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(1, 2), false ) .should.equal(true) }) it('should insert the second set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(2, 3), false ) .should.equal(true) }) it('should insert the third set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(3, 4), false ) .should.equal(true) }) it('should insert the final set of remaining updates as a new pack', function () { return this.PackManager.insertUpdatesIntoNewPack .calledWith( this.project_id, this.doc_id, this.newUpdates.slice(4, 5), false ) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) }) describe('flushCompressedUpdates', function () { return describe('when there is no previous update', function () { beforeEach(function () { return this.PackManager.flushCompressedUpdates( this.project_id, this.doc_id, null, this.newUpdates, true, this.callback ) }) return describe('for a small update that will expire', function () { it('should insert the update into mongo', function () { return this.db.docHistory.insertOne .calledWithMatch({ pack: this.newUpdates, project_id: ObjectId(this.project_id), doc_id: ObjectId(this.doc_id), n: this.newUpdates.length, v: this.newUpdates[0].v, v_end: this.newUpdates[this.newUpdates.length - 1].v }) .should.equal(true) }) it('should set an expiry time in the future', function () { return this.db.docHistory.insertOne .calledWithMatch({ expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) }) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) }) }) describe('when there is a recent previous update in mongo that expires', function () { beforeEach(function () { this.lastUpdate = { _id: '12345', pack: [ { op: 'op-1', meta: 'meta-1', v: 1 }, { op: 'op-2', meta: 'meta-2', v: 2 } ], n: 2, sz: 100, meta: { start_ts: Date.now() - 6 * 3600 * 1000 }, expiresAt: new Date(Date.now()) } return this.PackManager.flushCompressedUpdates( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, true, this.callback ) }) return describe('for a small update that will expire', function () { it('should append the update in mongo', function () { return this.db.docHistory.updateOne .calledWithMatch( { _id: this.lastUpdate._id }, { $push: { pack: { $each: this.newUpdates } }, $set: { v_end: this.newUpdates[this.newUpdates.length - 1].v } } ) .should.equal(true) }) it('should set an expiry time in the future', function () { return this.db.docHistory.updateOne .calledWithMatch(sinon.match.any, { $set: { expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) } }) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) }) describe('when there is a recent previous update in mongo that expires', function () { beforeEach(function () { this.PackManager.updateIndex = sinon.stub().callsArg(2) this.lastUpdate = { _id: '12345', pack: [ { op: 'op-1', meta: 'meta-1', v: 1 }, { op: 'op-2', meta: 'meta-2', v: 2 } ], n: 2, sz: 100, meta: { start_ts: Date.now() - 6 * 3600 * 1000 }, expiresAt: new Date(Date.now()) } return this.PackManager.flushCompressedUpdates( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, false, this.callback ) }) return describe('for a small update that will not expire', function () { it('should insert the update into mongo', function () { return this.db.docHistory.insertOne .calledWithMatch({ pack: this.newUpdates, project_id: ObjectId(this.project_id), doc_id: ObjectId(this.doc_id), n: this.newUpdates.length, v: this.newUpdates[0].v, v_end: this.newUpdates[this.newUpdates.length - 1].v }) .should.equal(true) }) it('should not set any expiry time', function () { return this.db.docHistory.insertOne .neverCalledWithMatch(sinon.match.has('expiresAt')) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) }) return describe('when there is an old previous update in mongo', function () { beforeEach(function () { this.lastUpdate = { _id: '12345', pack: [ { op: 'op-1', meta: 'meta-1', v: 1 }, { op: 'op-2', meta: 'meta-2', v: 2 } ], n: 2, sz: 100, meta: { start_ts: Date.now() - 30 * 24 * 3600 * 1000 }, expiresAt: new Date(Date.now() - 30 * 24 * 3600 * 1000) } return this.PackManager.flushCompressedUpdates( this.project_id, this.doc_id, this.lastUpdate, this.newUpdates, true, this.callback ) }) return describe('for a small update that will expire', function () { it('should insert the update into mongo', function () { return this.db.docHistory.insertOne .calledWithMatch({ pack: this.newUpdates, project_id: ObjectId(this.project_id), doc_id: ObjectId(this.doc_id), n: this.newUpdates.length, v: this.newUpdates[0].v, v_end: this.newUpdates[this.newUpdates.length - 1].v }) .should.equal(true) }) it('should set an expiry time in the future', function () { return this.db.docHistory.insertOne .calledWithMatch({ expiresAt: new Date(Date.now() + 7 * 24 * 3600 * 1000) }) .should.equal(true) }) return it('should call the callback', function () { return this.callback.called.should.equal(true) }) }) }) }) describe('getOpsByVersionRange', function () {}) describe('loadPacksByVersionRange', function () {}) describe('fetchPacksIfNeeded', function () {}) describe('makeProjectIterator', function () {}) describe('getPackById', function () {}) describe('increaseTTL', function () {}) describe('getIndex', function () {}) describe('getPackFromIndex', function () {}) // getLastPackFromIndex: // getIndexWithKeys // initialiseIndex // updateIndex // findCompletedPacks // findUnindexedPacks // insertPacksIntoIndexWithLock // _insertPacksIntoIndex // archivePack // checkArchivedPack // processOldPack // updateIndexIfNeeded // findUnarchivedPacks return describe('checkArchiveNotInProgress', function () { describe('when an archive is in progress', function () { beforeEach(function () { this.db.docHistoryIndex = { findOne: sinon.stub().callsArgWith(2, null, { inS3: false }) } return this.PackManager.checkArchiveNotInProgress( this.project_id, this.doc_id, this.pack_id, this.callback ) }) it('should call the callback', function () { return this.callback.called.should.equal(true) }) return it('should return an error', function () { return this.callback .calledWith(sinon.match.has('message')) .should.equal(true) }) }) describe('when an archive is completed', function () { beforeEach(function () { this.db.docHistoryIndex = { findOne: sinon.stub().callsArgWith(2, null, { inS3: true }) } return this.PackManager.checkArchiveNotInProgress( this.project_id, this.doc_id, this.pack_id, this.callback ) }) it('should call the callback', function () { return this.callback.called.should.equal(true) }) return it('should return an error', function () { return this.callback .calledWith(sinon.match.has('message')) .should.equal(true) }) }) return describe('when the archive has not started or completed', function () { beforeEach(function () { this.db.docHistoryIndex = { findOne: sinon.stub().callsArgWith(2, null, {}) } return this.PackManager.checkArchiveNotInProgress( this.project_id, this.doc_id, this.pack_id, this.callback ) }) it('should call the callback with no error', function () { return this.callback.called.should.equal(true) }) return it('should return with no error', function () { return (typeof this.callback.lastCall.args[0]).should.equal('undefined') }) }) }) }) // describe "setTTLOnArchivedPack", -> // beforeEach -> // @pack_id = "somepackid" // @onedayinms = 86400000 // @db.docHistory = // findAndModify : sinon.stub().callsArgWith(1) // it "should set expires to 1 day", (done)-> // #@PackManager._getOneDayInFutureWithRandomDelay = sinon.stub().returns(@onedayinms) // @PackManager.setTTLOnArchivedPack @project_id, @doc_id, @pack_id, => // args = @db.docHistory.findAndModify.args[0][0] // args.query._id.should.equal @pack_id // args.update['$set'].expiresAt.should.equal @onedayinms // done() // describe "_getOneDayInFutureWithRandomDelay", -> // beforeEach -> // @onedayinms = 86400000 // @thirtyMins = 1000 * 60 * 30 // it "should give 1 day + 30 mins random time", (done)-> // loops = 10000 // while --loops > 0 // randomDelay = @PackManager._getOneDayInFutureWithRandomDelay() - new Date(Date.now() + @onedayinms) // randomDelay.should.be.above(0) // randomDelay.should.be.below(@thirtyMins + 1) // done() function __range__(left, right, inclusive) { const range = [] const ascending = left < right const end = !inclusive ? right : ascending ? right + 1 : right - 1 for (let i = left; ascending ? i < end : i > end; ascending ? i++ : i--) { range.push(i) } return range }