diff --git a/services/docstore/app.js b/services/docstore/app.js index 7cb774b9f8..931914e0f5 100644 --- a/services/docstore/app.js +++ b/services/docstore/app.js @@ -54,6 +54,7 @@ app.get('/project/:project_id/ranges', HttpController.getAllRanges) app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc) app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted) app.get('/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc) +app.get('/project/:project_id/doc/:doc_id/peek', HttpController.peekDoc) // Add 64kb overhead for the JSON encoding, and double the size to allow for ranges in the json payload app.post( '/project/:project_id/doc/:doc_id', diff --git a/services/docstore/app/js/DocManager.js b/services/docstore/app/js/DocManager.js index 51a974d5a7..17aff5d989 100644 --- a/services/docstore/app/js/DocManager.js +++ b/services/docstore/app/js/DocManager.js @@ -124,6 +124,70 @@ module.exports = DocManager = { ) }, + // returns the doc without any version information + _peekRawDoc(project_id, doc_id, callback) { + MongoManager.findDoc( + project_id, + doc_id, + { + lines: true, + rev: true, + deleted: true, + version: true, + ranges: true, + inS3: true, + }, + (err, doc) => { + if (err) return callback(err) + if (doc == null) { + return callback( + new Errors.NotFoundError( + `No such doc: ${doc_id} in project ${project_id}` + ) + ) + } + if (doc && !doc.inS3) { + return callback(null, doc) + } + // skip the unarchiving to mongo when getting a doc + DocArchive.getDoc(project_id, doc_id, function (err, archivedDoc) { + if (err != null) { + logger.err( + { err, project_id, doc_id }, + 'error getting doc from archive' + ) + return callback(err) + } + doc = _.extend(doc, archivedDoc) + callback(null, doc) + }) + } + ) + }, + + // get the doc from mongo if possible, or from the persistent store otherwise, + // without unarchiving it (avoids unnecessary writes to mongo) + peekDoc(project_id, doc_id, callback) { + DocManager._peekRawDoc(project_id, doc_id, (err, doc) => { + if (err) { + return callback(err) + } + MongoManager.WithRevCheck( + doc, + MongoManager.getDocVersion, + function (error, version) { + // If the doc has been modified while we were retrieving it, we + // will get a DocModified error + if (error != null) { + return callback(error) + } + doc.version = version + return callback(err, doc) + } + ) + }) + }, + getDocLines(project_id, doc_id, callback) { if (callback == null) { callback = function (err, doc) {} diff --git a/services/docstore/app/js/Errors.js b/services/docstore/app/js/Errors.js index 4eaa5481d3..3cf5ad74a4 100644 --- a/services/docstore/app/js/Errors.js +++ b/services/docstore/app/js/Errors.js @@ -4,7 +4,10 @@ const { Errors } = require('@overleaf/object-persistor') class Md5MismatchError extends OError {} +class DocModifiedError extends OError {} + module.exports = { Md5MismatchError, + DocModifiedError, ...Errors, } diff --git a/services/docstore/app/js/HttpController.js b/services/docstore/app/js/HttpController.js index f53c9068bc..909836066b 100644 --- a/services/docstore/app/js/HttpController.js +++ b/services/docstore/app/js/HttpController.js @@ -44,6 +44,22 @@ module.exports = HttpController = { }) }, + peekDoc(req, res, next) { + const { project_id } = req.params + const { doc_id } = req.params + logger.log({ project_id, doc_id }, 'peeking doc') + DocManager.peekDoc(project_id, doc_id, function (error, doc) { + if (error) { + return next(error) + } + if (doc == null) { + return res.sendStatus(404) + } else { + return res.json(HttpController._buildDocView(doc)) + } + }) + }, + isDocDeleted(req, res, next) { const { doc_id: docId, project_id: projectId } = req.params DocManager.isDocDeleted(projectId, docId, function (error, deleted) { diff --git a/services/docstore/app/js/MongoManager.js b/services/docstore/app/js/MongoManager.js index 5434db581d..ef90d966ff 100644 --- a/services/docstore/app/js/MongoManager.js +++ b/services/docstore/app/js/MongoManager.js @@ -15,6 +15,7 @@ const { db, ObjectId } = require('./mongodb') const logger = require('logger-sharelatex') const metrics = require('@overleaf/metrics') const Settings = require('@overleaf/settings') +const { DocModifiedError } = require('./Errors') const { promisify } = require('util') module.exports = MongoManager = { @@ -178,6 +179,45 @@ module.exports = MongoManager = { ) }, + getDocRev(doc_id, callback) { + db.docs.findOne( + { + _id: ObjectId(doc_id.toString()), + }, + { + projection: { rev: 1 }, + }, + function (err, doc) { + if (err != null) { + return callback(err) + } + callback(null, doc && doc.rev) + } + ) + }, + + // Helper method to support optimistic locking. Call the provided method for + // an existing doc and return the result if the rev in mongo is unchanged when + // checked afterwards. If the rev has changed, return a DocModifiedError. + WithRevCheck(doc, method, callback) { + method(doc._id, function (err, result) { + if (err) return callback(err) + MongoManager.getDocRev(doc._id, function (err, currentRev) { + if (err) return callback(err) + if (doc.rev !== currentRev) { + return callback( + new DocModifiedError('doc rev has changed', { + doc_id: doc._id, + rev: doc.rev, + currentRev, + }) + ) + } + return callback(null, result) + }) + }) + }, + destroyDoc(doc_id, callback) { db.docs.deleteOne( { diff --git a/services/docstore/test/acceptance/js/GettingDocsFromArchiveTest.js b/services/docstore/test/acceptance/js/GettingDocsFromArchiveTest.js new file mode 100644 index 0000000000..714721e079 --- /dev/null +++ b/services/docstore/test/acceptance/js/GettingDocsFromArchiveTest.js @@ -0,0 +1,99 @@ +/* eslint-disable + camelcase, + handle-callback-err, + no-unused-vars, +*/ +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +process.env.BACKEND = 'gcs' +const Settings = require('@overleaf/settings') +const { expect } = require('chai') +const { db, ObjectId } = require('../../../app/js/mongodb') +const async = require('async') +const DocstoreApp = require('./helpers/DocstoreApp') +const DocstoreClient = require('./helpers/DocstoreClient') +const { Storage } = require('@google-cloud/storage') +const Persistor = require('../../../app/js/PersistorManager') +const Streamifier = require('streamifier') + +function uploadContent(path, json, callback) { + const stream = Streamifier.createReadStream(JSON.stringify(json)) + Persistor.sendStream(Settings.docstore.bucket, path, stream) + .then(() => callback()) + .catch(callback) +} + +describe('Getting A Doc from Archive', function () { + before(function (done) { + return DocstoreApp.ensureRunning(done) + }) + + before(async function () { + const storage = new Storage(Settings.docstore.gcs.endpoint) + await storage.createBucket(Settings.docstore.bucket) + await storage.createBucket(`${Settings.docstore.bucket}-deleted`) + }) + + describe('archiving a single doc', function () { + before(function (done) { + this.project_id = ObjectId() + this.timeout(1000 * 30) + this.doc = { + _id: ObjectId(), + lines: ['foo', 'bar'], + ranges: {}, + version: 2, + } + DocstoreClient.createDoc( + this.project_id, + this.doc._id, + this.doc.lines, + this.doc.version, + this.doc.ranges, + error => { + if (error) { + return done(error) + } + DocstoreClient.archiveDocById( + this.project_id, + this.doc._id, + (error, res) => { + this.res = res + if (error) { + return done(error) + } + done() + } + ) + } + ) + }) + + it('should successully archive the doc', function (done) { + this.res.statusCode.should.equal(204) + done() + }) + + it('should get the doc lines and version', function (done) { + return DocstoreClient.peekDoc( + this.project_id, + this.doc._id, + {}, + (error, res, doc) => { + res.statusCode.should.equal(200) + doc.lines.should.deep.equal(this.doc.lines) + doc.version.should.equal(this.doc.version) + doc.ranges.should.deep.equal(this.doc.ranges) + return done() + } + ) + }) + }) +}) diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js index 1164540bd3..4e80cc6c72 100644 --- a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js +++ b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js @@ -60,6 +60,20 @@ module.exports = DocstoreClient = { ) }, + peekDoc(project_id, doc_id, qs, callback) { + if (callback == null) { + callback = function (error, res, body) {} + } + return request.get( + { + url: `http://localhost:${settings.internal.docstore.port}/project/${project_id}/doc/${doc_id}/peek`, + json: true, + qs, + }, + callback + ) + }, + isDocDeleted(project_id, doc_id, callback) { request.get( { diff --git a/services/docstore/test/unit/js/MongoManagerTests.js b/services/docstore/test/unit/js/MongoManagerTests.js index 50cc8268c3..eb209a24c2 100644 --- a/services/docstore/test/unit/js/MongoManagerTests.js +++ b/services/docstore/test/unit/js/MongoManagerTests.js @@ -17,6 +17,7 @@ const modulePath = require('path').join( ) const { ObjectId } = require('mongodb') const { assert } = require('chai') +const Errors = require('../../../app/js/Errors') describe('MongoManager', function () { beforeEach(function () { @@ -28,6 +29,7 @@ describe('MongoManager', function () { }, '@overleaf/metrics': { timeAsyncMethod: sinon.stub() }, '@overleaf/settings': { max_deleted_docs: 42 }, + './Errors': { Errors }, }, }) this.project_id = ObjectId().toString()