overleaf/services/docstore/test/unit/js/DocArchiveManagerTests.js

578 lines
16 KiB
JavaScript
Raw Normal View History

/* eslint-disable
camelcase,
handle-callback-err,
no-return-assign,
no-unused-vars,
*/
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const { assert } = require('chai')
const sinon = require('sinon')
const chai = require('chai')
const should = chai.should()
const { expect } = chai
const modulePath = '../../../app/js/DocArchiveManager.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongojs')
const Errors = require('../../../app/js/Errors')
const crypto = require('crypto')
describe('DocArchiveManager', function() {
beforeEach(function() {
this.settings = {
docstore: {
s3: {
secret: 'secret',
key: 'this_key',
bucket: 'doc-archive-unit-test'
}
}
}
this.request = {
put: {},
get: {},
del: {}
}
this.archivedDocs = [
{
_id: ObjectId(),
inS3: true,
rev: 2
},
{
_id: ObjectId(),
inS3: true,
rev: 4
},
{
_id: ObjectId(),
inS3: true,
rev: 6
}
]
this.mongoDocs = [
{
_id: ObjectId(),
lines: ['one', 'two', 'three'],
rev: 2
},
{
_id: ObjectId(),
lines: ['aaa', 'bbb', 'ccc'],
rev: 4
},
{
_id: ObjectId(),
inS3: true,
rev: 6
},
{
_id: ObjectId(),
inS3: true,
rev: 6
},
{
_id: ObjectId(),
lines: ['111', '222', '333'],
rev: 6
}
]
this.unarchivedDocs = [
{
_id: ObjectId(),
lines: ['wombat', 'potato', 'banana'],
rev: 2
},
{
_id: ObjectId(),
lines: ['llama', 'turnip', 'apple'],
rev: 4
},
{
_id: ObjectId(),
lines: ['elephant', 'swede', 'nectarine'],
rev: 6
}
]
this.mixedDocs = this.archivedDocs.concat(this.unarchivedDocs)
this.MongoManager = {
markDocAsArchived: sinon.stub().callsArgWith(2, null),
upsertIntoDocCollection: sinon.stub().callsArgWith(3, null),
getProjectsDocs: sinon.stub().callsArgWith(3, null, this.mongoDocs),
getArchivedProjectDocs: sinon.stub().callsArgWith(2, null, this.mongoDocs)
}
this.requires = {
'settings-sharelatex': this.settings,
'./MongoManager': this.MongoManager,
request: this.request,
'./RangeManager': (this.RangeManager = {}),
'logger-sharelatex': {
log() {},
err() {}
}
}
this.globals = { JSON }
this.error = 'my errror'
this.project_id = ObjectId().toString()
this.stubbedError = new Errors.NotFoundError('Error in S3 request')
return (this.DocArchiveManager = SandboxedModule.require(modulePath, {
requires: this.requires,
globals: this.globals
}))
})
describe('archiveDoc', function() {
it('should use correct options', function(done) {
this.request.put = sinon
.stub()
.callsArgWith(1, null, { statusCode: 200, headers: { etag: '' } })
return this.DocArchiveManager.archiveDoc(
this.project_id,
this.mongoDocs[0],
err => {
const opts = this.request.put.args[0][0]
assert.deepEqual(opts.aws, {
key: this.settings.docstore.s3.key,
secret: this.settings.docstore.s3.secret,
bucket: this.settings.docstore.s3.bucket
})
opts.body.should.equal(
JSON.stringify({
lines: this.mongoDocs[0].lines,
ranges: this.mongoDocs[0].ranges,
schema_v: 1
})
)
opts.timeout.should.equal(30 * 1000)
opts.uri.should.equal(
`https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
)
return done()
}
)
})
it('should return no md5 error', function(done) {
const data = JSON.stringify({
lines: this.mongoDocs[0].lines,
ranges: this.mongoDocs[0].ranges,
schema_v: 1
})
this.md5 = crypto
.createHash('md5')
.update(data)
.digest('hex')
this.request.put = sinon
.stub()
.callsArgWith(1, null, { statusCode: 200, headers: { etag: this.md5 } })
return this.DocArchiveManager.archiveDoc(
this.project_id,
this.mongoDocs[0],
err => {
should.not.exist(err)
return done()
}
)
})
return it('should return the error', function(done) {
this.request.put = sinon.stub().callsArgWith(1, this.stubbedError, {
statusCode: 400,
headers: { etag: '' }
})
return this.DocArchiveManager.archiveDoc(
this.project_id,
this.mongoDocs[0],
err => {
should.exist(err)
return done()
}
)
})
})
describe('unarchiveDoc', function() {
it('should use correct options', function(done) {
this.request.get = sinon
.stub()
.callsArgWith(1, null, { statusCode: 200 }, this.mongoDocs[0].lines)
this.request.del = sinon
.stub()
.callsArgWith(1, null, { statusCode: 204 }, {})
return this.DocArchiveManager.unarchiveDoc(
this.project_id,
this.mongoDocs[0]._id,
err => {
const opts = this.request.get.args[0][0]
assert.deepEqual(opts.aws, {
key: this.settings.docstore.s3.key,
secret: this.settings.docstore.s3.secret,
bucket: this.settings.docstore.s3.bucket
})
opts.json.should.equal(true)
opts.timeout.should.equal(30 * 1000)
opts.uri.should.equal(
`https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
)
return done()
}
)
})
it('should return the error', function(done) {
this.request.get = sinon.stub().callsArgWith(1, this.stubbedError, {}, {})
return this.DocArchiveManager.unarchiveDoc(
this.project_id,
this.mongoDocs[0],
err => {
should.exist(err)
return done()
}
)
})
return it('should error if the doc lines are a string not an array', function(done) {
this.request.get = sinon
.stub()
.callsArgWith(1, null, { statusCode: 200 }, 'this is a string')
this.request.del = sinon.stub()
return this.DocArchiveManager.unarchiveDoc(
this.project_id,
this.mongoDocs[0],
err => {
should.exist(err)
this.request.del.called.should.equal(false)
return done()
}
)
})
})
describe('archiveAllDocs', function() {
it('should archive all project docs which are not in s3', function(done) {
this.MongoManager.getProjectsDocs = sinon
.stub()
.callsArgWith(3, null, this.mongoDocs)
this.DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null)
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
this.DocArchiveManager.archiveDoc
.calledWith(this.project_id, this.mongoDocs[0])
.should.equal(true)
this.DocArchiveManager.archiveDoc
.calledWith(this.project_id, this.mongoDocs[1])
.should.equal(true)
this.DocArchiveManager.archiveDoc
.calledWith(this.project_id, this.mongoDocs[4])
.should.equal(true)
this.DocArchiveManager.archiveDoc
.calledWith(this.project_id, this.mongoDocs[2])
.should.equal(false)
this.DocArchiveManager.archiveDoc
.calledWith(this.project_id, this.mongoDocs[3])
.should.equal(false)
should.not.exist(err)
return done()
})
})
it('should return error if have no docs', function(done) {
this.MongoManager.getProjectsDocs = sinon
.stub()
.callsArgWith(3, null, null)
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
should.exist(err)
return done()
})
})
it('should return the error', function(done) {
this.MongoManager.getProjectsDocs = sinon
.stub()
.callsArgWith(3, this.error, null)
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
err.should.equal(this.error)
return done()
})
})
return describe('when most have been already put in s3', function() {
beforeEach(function() {
let numberOfDocs = 10 * 1000
this.mongoDocs = []
while (--numberOfDocs !== 0) {
this.mongoDocs.push({ inS3: true, _id: ObjectId() })
}
this.MongoManager.getProjectsDocs = sinon
.stub()
.callsArgWith(3, null, this.mongoDocs)
return (this.DocArchiveManager.archiveDoc = sinon
.stub()
.callsArgWith(2, null))
})
return it('should not throw and error', function(done) {
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
should.not.exist(err)
return done()
})
})
})
})
describe('unArchiveAllDocs', function() {
it('should unarchive all inS3 docs', function(done) {
this.MongoManager.getArchivedProjectDocs = sinon
.stub()
.callsArgWith(1, null, this.archivedDocs)
this.DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null)
return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
for (const doc of Array.from(this.archivedDocs)) {
this.DocArchiveManager.unarchiveDoc
.calledWith(this.project_id, doc._id)
.should.equal(true)
}
should.not.exist(err)
return done()
})
})
it('should return error if have no docs', function(done) {
this.MongoManager.getArchivedProjectDocs = sinon
.stub()
.callsArgWith(1, null, null)
return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
should.exist(err)
return done()
})
})
return it('should return the error', function(done) {
this.MongoManager.getArchivedProjectDocs = sinon
.stub()
.callsArgWith(1, this.error, null)
return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
err.should.equal(this.error)
return done()
})
})
})
describe('destroyAllDocs', function() {
beforeEach(function() {
this.request.del = sinon
.stub()
.callsArgWith(1, null, { statusCode: 204 }, {})
this.MongoManager.getProjectsDocs = sinon
.stub()
.callsArgWith(3, null, this.mixedDocs)
this.MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null)
this.MongoManager.destroyDoc = sinon.stub().yields()
return Array.from(this.mixedDocs).map(doc =>
this.MongoManager.findDoc
.withArgs(this.project_id, doc._id)
.callsArgWith(3, null, doc)
)
})
it('should destroy all the docs', function(done) {
this.DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null)
return this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
for (const doc of Array.from(this.mixedDocs)) {
this.DocArchiveManager.destroyDoc
.calledWith(this.project_id, doc._id)
.should.equal(true)
}
should.not.exist(err)
return done()
})
})
it('should only the s3 docs from s3', function(done) {
const docOpts = doc => {
return JSON.parse(
JSON.stringify({
aws: {
key: this.settings.docstore.s3.key,
secret: this.settings.docstore.s3.secret,
bucket: this.settings.docstore.s3.bucket
},
json: true,
timeout: 30 * 1000,
uri: `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${doc._id}`
})
)
}
return this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
let doc
expect(err).not.to.exist
for (doc of Array.from(this.archivedDocs)) {
sinon.assert.calledWith(this.request.del, docOpts(doc))
}
for (doc of Array.from(this.unarchivedDocs)) {
expect(this.request.del.calledWith(docOpts(doc))).to.equal(false)
} // no notCalledWith
return done()
})
})
return it('should remove the docs from mongo', function(done) {
this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
return expect(err).not.to.exist
})
for (const doc of Array.from(this.mixedDocs)) {
sinon.assert.calledWith(this.MongoManager.destroyDoc, doc._id)
}
return done()
})
})
describe('_s3DocToMongoDoc', function() {
describe('with the old schema', function() {
return it('should return the docs lines', function(done) {
return this.DocArchiveManager._s3DocToMongoDoc(
['doc', 'lines'],
(error, doc) => {
expect(doc).to.deep.equal({
lines: ['doc', 'lines']
})
return done()
}
)
})
})
describe('with the new schema', function() {
it('should return the doc lines and ranges', function(done) {
this.RangeManager.jsonRangesToMongo = sinon
.stub()
.returns({ mongo: 'ranges' })
return this.DocArchiveManager._s3DocToMongoDoc(
{
lines: ['doc', 'lines'],
ranges: { json: 'ranges' },
schema_v: 1
},
(error, doc) => {
expect(doc).to.deep.equal({
lines: ['doc', 'lines'],
ranges: { mongo: 'ranges' }
})
return done()
}
)
})
return it('should return just the doc lines when there are no ranges', function(done) {
return this.DocArchiveManager._s3DocToMongoDoc(
{
lines: ['doc', 'lines'],
schema_v: 1
},
(error, doc) => {
expect(doc).to.deep.equal({
lines: ['doc', 'lines']
})
return done()
}
)
})
})
return describe('with an unrecognised schema', function() {
return it('should return an error', function(done) {
return this.DocArchiveManager._s3DocToMongoDoc(
{
schema_v: 2
},
(error, doc) => {
expect(error).to.exist
return done()
}
)
})
})
})
return describe('_mongoDocToS3Doc', function() {
describe('with a valid doc', function() {
return it('should return the json version', function(done) {
let doc
return this.DocArchiveManager._mongoDocToS3Doc(
(doc = {
lines: ['doc', 'lines'],
ranges: { mock: 'ranges' }
}),
(err, s3_doc) => {
expect(s3_doc).to.equal(
JSON.stringify({
lines: ['doc', 'lines'],
ranges: { mock: 'ranges' },
schema_v: 1
})
)
return done()
}
)
})
})
describe('with null bytes in the result', function() {
beforeEach(function() {
this._stringify = JSON.stringify
return (JSON.stringify = sinon.stub().returns('{"bad": "\u0000"}'))
})
afterEach(function() {
return (JSON.stringify = this._stringify)
})
return it('should return an error', function(done) {
return this.DocArchiveManager._mongoDocToS3Doc(
{
lines: ['doc', 'lines'],
ranges: { mock: 'ranges' }
},
(err, s3_doc) => {
expect(err).to.exist
return done()
}
)
})
})
return describe('without doc lines', function() {
return it('should return an error', function(done) {
return this.DocArchiveManager._mongoDocToS3Doc({}, (err, s3_doc) => {
expect(err).to.exist
return done()
})
})
})
})
})