Merge pull request #40 from overleaf/spd-destroy-docs

Add endpoint to completely purge a project's documents

services/docstore/app.coffee

@@ -42,6 +42,7 @@ app.del  '/project/:project_id/doc/:doc_id', HttpController.deleteDoc
 
 app.post  '/project/:project_id/archive', HttpController.archiveAllDocs
 app.post  '/project/:project_id/unarchive', HttpController.unArchiveAllDocs
+app.post  '/project/:project_id/destroy', HttpController.destroyAllDocs
 
 app.get "/health_check",  HttpController.healthCheck
 

services/docstore/app/coffee/DocArchiveManager.coffee

@@ -79,11 +79,43 @@ module.exports = DocArchive =
 				MongoManager.upsertIntoDocCollection project_id, doc_id.toString(), mongo_doc, (err) ->
 					return callback(err) if err?
 					logger.log project_id: project_id, doc_id: doc_id, "deleting doc from s3"
-					request.del options, (err, res, body)->
-						if err? || res.statusCode != 204
-							logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws"
-							return callback new Errors.NotFoundError("Error in S3 request")
-						callback()
+					DocArchive._deleteDocFromS3 project_id, doc_id, callback
+
+	destroyAllDocs: (project_id, callback = (err) ->) ->
+		MongoManager.getProjectsDocs project_id, {include_deleted: true}, {_id: 1}, (err, docs) ->
+			if err?
+				logger.err err:err, project_id:project_id, "error getting project's docs"
+				return callback(err)
+			else if !docs?
+				return callback()
+			jobs = _.map docs, (doc) ->
+				(cb)->
+					DocArchive.destroyDoc(project_id, doc._id, cb)
+			async.parallelLimit jobs, 5, callback
+
+	destroyDoc: (project_id, doc_id, callback)->
+		logger.log project_id: project_id, doc_id: doc_id, "removing doc from mongo and s3"
+		MongoManager.findDoc project_id, doc_id, {inS3: 1}, (error, doc) ->
+			return callback error if error?
+			return callback new Errors.NotFoundError("Doc not found in Mongo") unless doc?
+			if doc.inS3 == true
+				DocArchive._deleteDocFromS3 project_id, doc_id, (err) ->
+					return err if err?
+					MongoManager.destroyDoc doc_id, callback
+			else
+				MongoManager.destroyDoc doc_id, callback
+
+	_deleteDocFromS3: (project_id, doc_id, callback) ->
+		try
+			options = DocArchive.buildS3Options(project_id+"/"+doc_id)
+		catch e
+			return callback e
+		options.json = true
+		request.del options, (err, res, body)->
+			if err? || res.statusCode != 204
+				logger.err err:err, res:res, project_id:project_id, doc_id:doc_id, "something went wrong deleting doc from aws"
+				return callback new Error("Error in S3 request")
+			callback()
 
 	_s3DocToMongoDoc: (doc, callback = (error, mongo_doc) ->) ->
 		mongo_doc = {}

services/docstore/app/coffee/HttpController.coffee

@@ -117,6 +117,13 @@ module.exports = HttpController =
 			return next(error) if error?
 			res.send 200
 
+	destroyAllDocs: (req, res, next = (error) ->) ->
+		project_id = req.params.project_id
+		logger.log project_id: project_id, "destroying all docs"
+		DocArchive.destroyAllDocs project_id, (error) ->
+			return next(error) if error?
+			res.send 204
+
 	healthCheck: (req, res)->
 		HealthChecker.check (err)->
 			if err?

services/docstore/app/coffee/MongoManager.coffee

@@ -72,6 +72,14 @@ module.exports = MongoManager =
 			upsert: true
 		}, callback
 
+	destroyDoc: (doc_id, callback) ->
+		db.docs.remove {
+			_id: ObjectId(doc_id)
+		}, (err) ->
+			return callback(err) if err?
+			db.docOps.remove {
+				doc_id: ObjectId(doc_id)
+			}, callback
 
 [
 	'findDoc',

services/docstore/test/acceptance/coffee/DeletingDocsTests.coffee

@@ -2,6 +2,7 @@ sinon = require "sinon"
 chai = require("chai")
 chai.should()
 {db, ObjectId} = require "../../../app/js/mongojs"
+expect = chai.expect
 DocstoreApp = require "./helpers/DocstoreApp"
 
 DocstoreClient = require "./helpers/DocstoreClient"
@@ -40,3 +41,45 @@ describe "Deleting a doc", ->
 				res.statusCode.should.equal 404
 				done()
 
+describe "Destroying a project's documents", ->
+	describe "when the doc exists", ->
+		beforeEach (done) ->
+			db.docOps.insert {doc_id: ObjectId(@doc_id), version: 1}, (err) ->
+				return done(err) if err?
+				DocstoreClient.destroyAllDoc @project_id, done
+
+		it "should remove the doc from the docs collection", (done) ->
+			db.docs.find _id: @doc_id, (err, docs) ->
+				expect(err).not.to.exist
+				expect(docs).to.deep.equal []
+				done()
+
+		it "should remove the docOps from the docOps collection", (done) ->
+			db.docOps.find doc_id: @doc_id, (err, docOps) ->
+				expect(err).not.to.exist
+				expect(docOps).to.deep.equal []
+				done()
+
+	describe "when the doc is archived", ->
+		beforeEach (done) ->
+			DocstoreClient.archiveAllDoc @project_id, (err) ->
+				return done(err) if err?
+				DocstoreClient.destroyAllDoc @project_id, done
+
+		it "should remove the doc from the docs collection", (done) ->
+			db.docs.find _id: @doc_id, (err, docs) ->
+				expect(err).not.to.exist
+				expect(docs).to.deep.equal []
+				done()
+
+		it "should remove the docOps from the docOps collection", (done) ->
+			db.docOps.find doc_id: @doc_id, (err, docOps) ->
+				expect(err).not.to.exist
+				expect(docOps).to.deep.equal []
+				done()
+
+		it "should remove the doc contents from s3", (done) ->
+			DocstoreClient.getS3Doc @project_id, @doc_id, (error, res, s3_doc) =>
+				throw error if error?
+				expect(res.statusCode).to.equal 404
+				done()

services/docstore/test/acceptance/coffee/helpers/DocstoreClient.coffee

@@ -44,8 +44,12 @@ module.exports = DocstoreClient =
 	archiveAllDoc: (project_id, callback = (error, res, body) ->) ->
 		request.post {
 			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/archive"
-		}, callback	
+		}, callback
 
+	destroyAllDoc: (project_id, callback = (error, res, body) ->) ->
+		request.post {
+			url: "http://localhost:#{settings.internal.docstore.port}/project/#{project_id}/destroy"
+		}, callback
 
 	getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) ->
 		options = DocArchiveManager.buildS3Options(project_id+"/"+doc_id)

services/docstore/test/unit/coffee/DocArchiveManagerTests.coffee

@@ -61,6 +61,22 @@ describe "DocArchiveManager", ->
 			rev: 6
 		}]
 
+		@unarchivedDocs = [{
+			_id: ObjectId()
+			lines: ["wombat", "potato", "banana"]
+			rev: 2
+		}, {
+			_id: ObjectId()
+			lines: ["llama", "turnip", "apple"]
+			rev: 4
+		}, {
+			_id: ObjectId()
+			lines: ["elephant", "swede", "nectarine"]
+			rev: 6
+		}]
+
+		@mixedDocs = @archivedDocs.concat(@unarchivedDocs)
+
 		@MongoManager =
 			markDocAsArchived: sinon.stub().callsArgWith(2, null)
 			upsertIntoDocCollection: sinon.stub().callsArgWith(3, null)
@@ -214,6 +230,51 @@ describe "DocArchiveManager", ->
 			@DocArchiveManager.unArchiveAllDocs @project_id, (err)=>
 				err.should.equal @error
 				done()
+
+	describe "destroyAllDocs", ->
+		beforeEach ->
+			@request.del = sinon.stub().callsArgWith(1, null, statusCode:204, {})
+			@MongoManager.getProjectsDocs = sinon.stub().callsArgWith(3, null, @mixedDocs)
+			@MongoManager.findDoc = sinon.stub().callsArgWith(3, null, null)
+			@MongoManager.destroyDoc = sinon.stub().yields()
+			for doc in @mixedDocs
+				@MongoManager.findDoc.withArgs(@project_id, doc._id).callsArgWith(3, null, doc)
+
+		it "should destroy all the docs", (done)->
+			@DocArchiveManager.destroyDoc = sinon.stub().callsArgWith(2, null)
+			@DocArchiveManager.destroyAllDocs @project_id, (err)=>
+				for doc in @mixedDocs
+					@DocArchiveManager.destroyDoc.calledWith(@project_id, doc._id).should.equal true
+				should.not.exist err
+				done()
+
+		it "should only the s3 docs from s3", (done)->
+			docOpts = (doc) =>
+				JSON.parse(JSON.stringify({
+					aws: {key:@settings.docstore.s3.key, secret:@settings.docstore.s3.secret, bucket:@settings.docstore.s3.bucket},
+					json: true,
+					timeout: 30 * 1000
+					uri:"https://#{@settings.docstore.s3.bucket}.s3.amazonaws.com/#{@project_id}/#{doc._id}"
+				}))
+
+			@DocArchiveManager.destroyAllDocs @project_id, (err)=>
+				expect(err).not.to.exist
+
+				for doc in @archivedDocs
+					sinon.assert.calledWith(@request.del, docOpts(doc))
+				for doc in @unarchivedDocs
+					expect(@request.del.calledWith(docOpts(doc))).to.equal false  # no notCalledWith
+
+				done()
+
+		it "should remove the docs from mongo", (done)->
+			@DocArchiveManager.destroyAllDocs @project_id, (err)=>
+				expect(err).not.to.exist
+
+			for doc in @mixedDocs
+				sinon.assert.calledWith(@MongoManager.destroyDoc, doc._id)
+
+			done()
 	
 	describe "_s3DocToMongoDoc", ->
 		describe "with the old schema", ->

services/docstore/test/unit/coffee/HttpControllerTests.coffee

@@ -326,3 +326,17 @@ describe "HttpController", ->
 			@res.send
 				.calledWith(204)
 				.should.equal true
+
+	describe "destroyAllDocs", ->
+		beforeEach ->
+			@req.params =
+				project_id: @project_id
+			@DocArchiveManager.destroyAllDocs = sinon.stub().callsArg(1)
+			@HttpController.destroyAllDocs @req, @res, @next
+
+		it "should destroy the docs", ->
+			sinon.assert.calledWith(@DocArchiveManager.destroyAllDocs, @project_id)
+
+		it "should return 204", ->
+			sinon.assert.calledWith(@res.send, 204)
+

services/docstore/test/unit/coffee/MongoManagerTests.coffee

@@ -110,6 +110,18 @@ describe "MongoManager", ->
 				err.should.equal @stubbedErr
 				done()
 
+	describe "destroyDoc", ->
+		beforeEach (done) ->
+			@db.docs.remove = sinon.stub().yields()
+			@db.docOps.remove = sinon.stub().yields()
+			@MongoManager.destroyDoc '123456789012', done
+
+		it "should destroy the doc", ->
+			sinon.assert.calledWith(@db.docs.remove, {_id: ObjectId('123456789012')})
+
+		it "should destroy the docOps", ->
+			sinon.assert.calledWith(@db.docOps.remove, {doc_id: ObjectId('123456789012')})
+
 	describe "getDocVersion", ->
 		describe "when the doc exists", ->
 			beforeEach ->