mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-23 02:55:13 -04:00
577 lines
16 KiB
JavaScript
577 lines
16 KiB
JavaScript
/* eslint-disable
|
|
camelcase,
|
|
handle-callback-err,
|
|
no-return-assign,
|
|
no-unused-vars,
|
|
*/
|
|
// TODO: This file was created by bulk-decaffeinate.
|
|
// Fix any style issues and re-enable lint.
|
|
/*
|
|
* decaffeinate suggestions:
|
|
* DS101: Remove unnecessary use of Array.from
|
|
* DS102: Remove unnecessary code created because of implicit returns
|
|
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
|
|
*/
|
|
const { assert } = require('chai')
|
|
const sinon = require('sinon')
|
|
const chai = require('chai')
|
|
const should = chai.should()
|
|
const { expect } = chai
|
|
const modulePath = '../../../app/js/DocArchiveManager.js'
|
|
const SandboxedModule = require('sandboxed-module')
|
|
const { ObjectId } = require('mongojs')
|
|
const Errors = require('../../../app/js/Errors')
|
|
const crypto = require('crypto')
|
|
|
|
describe('DocArchiveManager', function() {
|
|
beforeEach(function() {
|
|
this.settings = {
|
|
docstore: {
|
|
s3: {
|
|
secret: 'secret',
|
|
key: 'this_key',
|
|
bucket: 'doc-archive-unit-test'
|
|
}
|
|
}
|
|
}
|
|
|
|
this.request = {
|
|
put: {},
|
|
get: {},
|
|
del: {}
|
|
}
|
|
|
|
this.archivedDocs = [
|
|
{
|
|
_id: ObjectId(),
|
|
inS3: true,
|
|
rev: 2
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
inS3: true,
|
|
rev: 4
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
inS3: true,
|
|
rev: 6
|
|
}
|
|
]
|
|
|
|
this.mongoDocs = [
|
|
{
|
|
_id: ObjectId(),
|
|
lines: ['one', 'two', 'three'],
|
|
rev: 2
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
lines: ['aaa', 'bbb', 'ccc'],
|
|
rev: 4
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
inS3: true,
|
|
rev: 6
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
inS3: true,
|
|
rev: 6
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
lines: ['111', '222', '333'],
|
|
rev: 6
|
|
}
|
|
]
|
|
|
|
this.unarchivedDocs = [
|
|
{
|
|
_id: ObjectId(),
|
|
lines: ['wombat', 'potato', 'banana'],
|
|
rev: 2
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
lines: ['llama', 'turnip', 'apple'],
|
|
rev: 4
|
|
},
|
|
{
|
|
_id: ObjectId(),
|
|
lines: ['elephant', 'swede', 'nectarine'],
|
|
rev: 6
|
|
}
|
|
]
|
|
|
|
this.mixedDocs = this.archivedDocs.concat(this.unarchivedDocs)
|
|
|
|
this.MongoManager = {
|
|
markDocAsArchived: sinon.stub().callsArgWith(2, null),
|
|
upsertIntoDocCollection: sinon.stub().callsArgWith(3, null),
|
|
getProjectsDocs: sinon.stub().callsArgWith(3, null, this.mongoDocs),
|
|
getArchivedProjectDocs: sinon.stub().callsArgWith(2, null, this.mongoDocs)
|
|
}
|
|
|
|
this.requires = {
|
|
'settings-sharelatex': this.settings,
|
|
'./MongoManager': this.MongoManager,
|
|
request: this.request,
|
|
'./RangeManager': (this.RangeManager = {}),
|
|
'logger-sharelatex': {
|
|
log() {},
|
|
err() {}
|
|
}
|
|
}
|
|
this.globals = { JSON }
|
|
|
|
this.error = 'my errror'
|
|
this.project_id = ObjectId().toString()
|
|
this.stubbedError = new Errors.NotFoundError('Error in S3 request')
|
|
return (this.DocArchiveManager = SandboxedModule.require(modulePath, {
|
|
requires: this.requires,
|
|
globals: this.globals
|
|
}))
|
|
})
|
|
|
|
describe('archiveDoc', function() {
|
|
it('should use correct options', function(done) {
|
|
this.request.put = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, { statusCode: 200, headers: { etag: '' } })
|
|
return this.DocArchiveManager.archiveDoc(
|
|
this.project_id,
|
|
this.mongoDocs[0],
|
|
err => {
|
|
const opts = this.request.put.args[0][0]
|
|
assert.deepEqual(opts.aws, {
|
|
key: this.settings.docstore.s3.key,
|
|
secret: this.settings.docstore.s3.secret,
|
|
bucket: this.settings.docstore.s3.bucket
|
|
})
|
|
opts.body.should.equal(
|
|
JSON.stringify({
|
|
lines: this.mongoDocs[0].lines,
|
|
ranges: this.mongoDocs[0].ranges,
|
|
schema_v: 1
|
|
})
|
|
)
|
|
opts.timeout.should.equal(30 * 1000)
|
|
opts.uri.should.equal(
|
|
`https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
|
|
)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
|
|
it('should return no md5 error', function(done) {
|
|
const data = JSON.stringify({
|
|
lines: this.mongoDocs[0].lines,
|
|
ranges: this.mongoDocs[0].ranges,
|
|
schema_v: 1
|
|
})
|
|
this.md5 = crypto
|
|
.createHash('md5')
|
|
.update(data)
|
|
.digest('hex')
|
|
this.request.put = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, { statusCode: 200, headers: { etag: this.md5 } })
|
|
return this.DocArchiveManager.archiveDoc(
|
|
this.project_id,
|
|
this.mongoDocs[0],
|
|
err => {
|
|
should.not.exist(err)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
|
|
return it('should return the error', function(done) {
|
|
this.request.put = sinon.stub().callsArgWith(1, this.stubbedError, {
|
|
statusCode: 400,
|
|
headers: { etag: '' }
|
|
})
|
|
return this.DocArchiveManager.archiveDoc(
|
|
this.project_id,
|
|
this.mongoDocs[0],
|
|
err => {
|
|
should.exist(err)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('unarchiveDoc', function() {
|
|
it('should use correct options', function(done) {
|
|
this.request.get = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, { statusCode: 200 }, this.mongoDocs[0].lines)
|
|
this.request.del = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, { statusCode: 204 }, {})
|
|
return this.DocArchiveManager.unarchiveDoc(
|
|
this.project_id,
|
|
this.mongoDocs[0]._id,
|
|
err => {
|
|
const opts = this.request.get.args[0][0]
|
|
assert.deepEqual(opts.aws, {
|
|
key: this.settings.docstore.s3.key,
|
|
secret: this.settings.docstore.s3.secret,
|
|
bucket: this.settings.docstore.s3.bucket
|
|
})
|
|
opts.json.should.equal(true)
|
|
opts.timeout.should.equal(30 * 1000)
|
|
opts.uri.should.equal(
|
|
`https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${this.mongoDocs[0]._id}`
|
|
)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
|
|
it('should return the error', function(done) {
|
|
this.request.get = sinon.stub().callsArgWith(1, this.stubbedError, {}, {})
|
|
return this.DocArchiveManager.unarchiveDoc(
|
|
this.project_id,
|
|
this.mongoDocs[0],
|
|
err => {
|
|
should.exist(err)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
|
|
return it('should error if the doc lines are a string not an array', function(done) {
|
|
this.request.get = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, { statusCode: 200 }, 'this is a string')
|
|
this.request.del = sinon.stub()
|
|
return this.DocArchiveManager.unarchiveDoc(
|
|
this.project_id,
|
|
this.mongoDocs[0],
|
|
err => {
|
|
should.exist(err)
|
|
this.request.del.called.should.equal(false)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('archiveAllDocs', function() {
|
|
it('should archive all project docs which are not in s3', function(done) {
|
|
this.MongoManager.getProjectsDocs = sinon
|
|
.stub()
|
|
.callsArgWith(3, null, this.mongoDocs)
|
|
this.DocArchiveManager.archiveDoc = sinon.stub().callsArgWith(2, null)
|
|
|
|
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
|
|
this.DocArchiveManager.archiveDoc
|
|
.calledWith(this.project_id, this.mongoDocs[0])
|
|
.should.equal(true)
|
|
this.DocArchiveManager.archiveDoc
|
|
.calledWith(this.project_id, this.mongoDocs[1])
|
|
.should.equal(true)
|
|
this.DocArchiveManager.archiveDoc
|
|
.calledWith(this.project_id, this.mongoDocs[4])
|
|
.should.equal(true)
|
|
|
|
this.DocArchiveManager.archiveDoc
|
|
.calledWith(this.project_id, this.mongoDocs[2])
|
|
.should.equal(false)
|
|
this.DocArchiveManager.archiveDoc
|
|
.calledWith(this.project_id, this.mongoDocs[3])
|
|
.should.equal(false)
|
|
|
|
should.not.exist(err)
|
|
return done()
|
|
})
|
|
})
|
|
|
|
it('should return error if have no docs', function(done) {
|
|
this.MongoManager.getProjectsDocs = sinon
|
|
.stub()
|
|
.callsArgWith(3, null, null)
|
|
|
|
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
|
|
should.exist(err)
|
|
return done()
|
|
})
|
|
})
|
|
|
|
it('should return the error', function(done) {
|
|
this.MongoManager.getProjectsDocs = sinon
|
|
.stub()
|
|
.callsArgWith(3, this.error, null)
|
|
|
|
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
|
|
err.should.equal(this.error)
|
|
return done()
|
|
})
|
|
})
|
|
|
|
return describe('when most have been already put in s3', function() {
|
|
beforeEach(function() {
|
|
let numberOfDocs = 10 * 1000
|
|
this.mongoDocs = []
|
|
while (--numberOfDocs !== 0) {
|
|
this.mongoDocs.push({ inS3: true, _id: ObjectId() })
|
|
}
|
|
|
|
this.MongoManager.getProjectsDocs = sinon
|
|
.stub()
|
|
.callsArgWith(3, null, this.mongoDocs)
|
|
return (this.DocArchiveManager.archiveDoc = sinon
|
|
.stub()
|
|
.callsArgWith(2, null))
|
|
})
|
|
|
|
return it('should not throw and error', function(done) {
|
|
return this.DocArchiveManager.archiveAllDocs(this.project_id, err => {
|
|
should.not.exist(err)
|
|
return done()
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('unArchiveAllDocs', function() {
|
|
it('should unarchive all inS3 docs', function(done) {
|
|
this.MongoManager.getArchivedProjectDocs = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, this.archivedDocs)
|
|
this.DocArchiveManager.unarchiveDoc = sinon.stub().callsArgWith(2, null)
|
|
return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
|
|
for (const doc of Array.from(this.archivedDocs)) {
|
|
this.DocArchiveManager.unarchiveDoc
|
|
.calledWith(this.project_id, doc._id)
|
|
.should.equal(true)
|
|
}
|
|
should.not.exist(err)
|
|
return done()
|
|
})
|
|
})
|
|
|
|
it('should return error if have no docs', function(done) {
|
|
this.MongoManager.getArchivedProjectDocs = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, null)
|
|
return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
|
|
should.exist(err)
|
|
return done()
|
|
})
|
|
})
|
|
|
|
return it('should return the error', function(done) {
|
|
this.MongoManager.getArchivedProjectDocs = sinon
|
|
.stub()
|
|
.callsArgWith(1, this.error, null)
|
|
return this.DocArchiveManager.unArchiveAllDocs(this.project_id, err => {
|
|
err.should.equal(this.error)
|
|
return done()
|
|
})
|
|
})
|
|
})
|
|
|
|
describe('destroyAllDocs', function() {
|
|
beforeEach(function() {
|
|
this.request.del = sinon
|
|
.stub()
|
|
.callsArgWith(1, null, { statusCode: 204 }, {})
|
|
this.MongoManager.getProjectsDocs = sinon
|
|
.stub()
|
|
.callsArgWith(3, null, this.mixedDocs)
|
|
this.MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null)
|
|
this.MongoManager.destroyDoc = sinon.stub().yields()
|
|
return Array.from(this.mixedDocs).map(doc =>
|
|
this.MongoManager.findDoc
|
|
.withArgs(this.project_id, doc._id)
|
|
.callsArgWith(3, null, doc)
|
|
)
|
|
})
|
|
|
|
it('should destroy all the docs', function(done) {
|
|
this.DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null)
|
|
return this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
|
|
for (const doc of Array.from(this.mixedDocs)) {
|
|
this.DocArchiveManager.destroyDoc
|
|
.calledWith(this.project_id, doc._id)
|
|
.should.equal(true)
|
|
}
|
|
should.not.exist(err)
|
|
return done()
|
|
})
|
|
})
|
|
|
|
it('should only the s3 docs from s3', function(done) {
|
|
const docOpts = doc => {
|
|
return JSON.parse(
|
|
JSON.stringify({
|
|
aws: {
|
|
key: this.settings.docstore.s3.key,
|
|
secret: this.settings.docstore.s3.secret,
|
|
bucket: this.settings.docstore.s3.bucket
|
|
},
|
|
json: true,
|
|
timeout: 30 * 1000,
|
|
uri: `https://${this.settings.docstore.s3.bucket}.s3.amazonaws.com/${this.project_id}/${doc._id}`
|
|
})
|
|
)
|
|
}
|
|
|
|
return this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
|
|
let doc
|
|
expect(err).not.to.exist
|
|
|
|
for (doc of Array.from(this.archivedDocs)) {
|
|
sinon.assert.calledWith(this.request.del, docOpts(doc))
|
|
}
|
|
for (doc of Array.from(this.unarchivedDocs)) {
|
|
expect(this.request.del.calledWith(docOpts(doc))).to.equal(false)
|
|
} // no notCalledWith
|
|
|
|
return done()
|
|
})
|
|
})
|
|
|
|
return it('should remove the docs from mongo', function(done) {
|
|
this.DocArchiveManager.destroyAllDocs(this.project_id, err => {
|
|
return expect(err).not.to.exist
|
|
})
|
|
|
|
for (const doc of Array.from(this.mixedDocs)) {
|
|
sinon.assert.calledWith(this.MongoManager.destroyDoc, doc._id)
|
|
}
|
|
|
|
return done()
|
|
})
|
|
})
|
|
|
|
describe('_s3DocToMongoDoc', function() {
|
|
describe('with the old schema', function() {
|
|
return it('should return the docs lines', function(done) {
|
|
return this.DocArchiveManager._s3DocToMongoDoc(
|
|
['doc', 'lines'],
|
|
(error, doc) => {
|
|
expect(doc).to.deep.equal({
|
|
lines: ['doc', 'lines']
|
|
})
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('with the new schema', function() {
|
|
it('should return the doc lines and ranges', function(done) {
|
|
this.RangeManager.jsonRangesToMongo = sinon
|
|
.stub()
|
|
.returns({ mongo: 'ranges' })
|
|
return this.DocArchiveManager._s3DocToMongoDoc(
|
|
{
|
|
lines: ['doc', 'lines'],
|
|
ranges: { json: 'ranges' },
|
|
schema_v: 1
|
|
},
|
|
(error, doc) => {
|
|
expect(doc).to.deep.equal({
|
|
lines: ['doc', 'lines'],
|
|
ranges: { mongo: 'ranges' }
|
|
})
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
|
|
return it('should return just the doc lines when there are no ranges', function(done) {
|
|
return this.DocArchiveManager._s3DocToMongoDoc(
|
|
{
|
|
lines: ['doc', 'lines'],
|
|
schema_v: 1
|
|
},
|
|
(error, doc) => {
|
|
expect(doc).to.deep.equal({
|
|
lines: ['doc', 'lines']
|
|
})
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
|
|
return describe('with an unrecognised schema', function() {
|
|
return it('should return an error', function(done) {
|
|
return this.DocArchiveManager._s3DocToMongoDoc(
|
|
{
|
|
schema_v: 2
|
|
},
|
|
(error, doc) => {
|
|
expect(error).to.exist
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
})
|
|
|
|
return describe('_mongoDocToS3Doc', function() {
|
|
describe('with a valid doc', function() {
|
|
return it('should return the json version', function(done) {
|
|
let doc
|
|
return this.DocArchiveManager._mongoDocToS3Doc(
|
|
(doc = {
|
|
lines: ['doc', 'lines'],
|
|
ranges: { mock: 'ranges' }
|
|
}),
|
|
(err, s3_doc) => {
|
|
expect(s3_doc).to.equal(
|
|
JSON.stringify({
|
|
lines: ['doc', 'lines'],
|
|
ranges: { mock: 'ranges' },
|
|
schema_v: 1
|
|
})
|
|
)
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
|
|
describe('with null bytes in the result', function() {
|
|
beforeEach(function() {
|
|
this._stringify = JSON.stringify
|
|
return (JSON.stringify = sinon.stub().returns('{"bad": "\u0000"}'))
|
|
})
|
|
|
|
afterEach(function() {
|
|
return (JSON.stringify = this._stringify)
|
|
})
|
|
|
|
return it('should return an error', function(done) {
|
|
return this.DocArchiveManager._mongoDocToS3Doc(
|
|
{
|
|
lines: ['doc', 'lines'],
|
|
ranges: { mock: 'ranges' }
|
|
},
|
|
(err, s3_doc) => {
|
|
expect(err).to.exist
|
|
return done()
|
|
}
|
|
)
|
|
})
|
|
})
|
|
|
|
return describe('without doc lines', function() {
|
|
return it('should return an error', function(done) {
|
|
return this.DocArchiveManager._mongoDocToS3Doc({}, (err, s3_doc) => {
|
|
expect(err).to.exist
|
|
return done()
|
|
})
|
|
})
|
|
})
|
|
})
|
|
})
|