diff --git a/services/track-changes/app/coffee/DocArchiveManager.coffee b/services/track-changes/app/coffee/DocArchiveManager.coffee index c639a84c99..d557278b8e 100644 --- a/services/track-changes/app/coffee/DocArchiveManager.coffee +++ b/services/track-changes/app/coffee/DocArchiveManager.coffee @@ -25,39 +25,38 @@ module.exports = DocArchiveManager = LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback) archiveDocChanges: (project_id, doc_id, callback)-> + MongoManager.getArchivedDocStatus doc_id, (error, result) -> + return callback(error) if error? + if result?.inS3 is true + logger.log {project_id, doc_id}, "document history is already archived" + return callback() + if result?.inS3? + logger.log {project_id, doc_id}, "document history archive is already in progress" + return callback() MongoManager.getDocChangesCount doc_id, (error, count) -> return callback(error) if error? if count == 0 logger.log {project_id, doc_id}, "document history is empty, not archiving" return callback() - else if count == 1 - logger.log {project_id, doc_id}, "document history only has one entry, not archiving" - return callback() - else - MongoManager.getArchivedDocChanges doc_id, (error, count) -> + MongoManager.peekLastCompressedUpdate doc_id, (error, update, lastVersion) -> + return callback(error) if error? + logger.log {doc_id, project_id}, "archiving got last compressed update" + MongoManager.markDocHistoryAsArchiveInProgress doc_id, lastVersion, (error) -> return callback(error) if error? - if count != 0 - logger.log {project_id, doc_id}, "document history contains archived entries, not archiving" - return callback() - MongoManager.getLastCompressedUpdate doc_id, (error, update) -> - return callback(error) if error? - logger.log {doc_id, project_id}, "archiving got last compressed update" - MongoManager.markDocHistoryAsArchiveInProgress doc_id, update, (error) -> - return callback(error) if error? - logger.log {doc_id, project_id}, "marked doc history as archive in progress" - MongoAWS.archiveDocHistory project_id, doc_id, update, (error) -> - if error? - logger.log {doc_id, project_id, error}, "error exporting document to S3" - MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) -> - return callback(err) if err? - logger.log {doc_id, project_id}, "cleared archive in progress flag" - callback(error) - else - logger.log doc_id:doc_id, project_id:project_id, "exported document to S3" - MongoManager.markDocHistoryAsArchived doc_id, update, (error) -> - return callback(error) if error? - logger.log {doc_id, project_id}, "marked doc history as archived" - callback() + logger.log {doc_id, project_id}, "marked doc history as archive in progress" + MongoAWS.archiveDocHistory project_id, doc_id, update, (error) -> + if error? + logger.log {doc_id, project_id, error}, "error exporting document to S3" + MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) -> + return callback(err) if err? + logger.log {doc_id, project_id}, "cleared archive in progress flag" + callback(error) + else + logger.log doc_id:doc_id, project_id:project_id, "exported document to S3" + MongoManager.markDocHistoryAsArchived doc_id, lastVersion, (error) -> + return callback(error) if error? + logger.log {doc_id, project_id}, "marked doc history as archived" + callback() unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) -> DocstoreHandler.getAllDocs project_id, (error, docs) -> @@ -75,9 +74,9 @@ module.exports = DocArchiveManager = LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback) unArchiveDocChanges: (project_id, doc_id, callback)-> - MongoManager.getArchivedDocChanges doc_id, (error, count) -> + MongoManager.getArchivedDocStatus doc_id, (error, result) -> return callback(error) if error? - if count == 0 + if result?.inS3 isnt true logger.log {project_id, doc_id}, "no changes marked as in s3, not unarchiving" return callback() else diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 31ec778d71..692e82bca7 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -19,7 +19,7 @@ module.exports = MongoAWS = query = { doc_id: ObjectId(doc_id) - v: {$lt: update.v} + v: {$lte: update.v} expiresAt: {$exists : false} } diff --git a/services/track-changes/app/coffee/MongoManager.coffee b/services/track-changes/app/coffee/MongoManager.coffee index 83b89bce34..98a9025d8d 100644 --- a/services/track-changes/app/coffee/MongoManager.coffee +++ b/services/track-changes/app/coffee/MongoManager.coffee @@ -157,25 +157,23 @@ module.exports = MongoManager = db.docHistoryStats.findOne {doc_id: ObjectId(doc_id.toString()), inS3: {$exists:true}}, {inS3: true, lastVersion: true}, callback getDocChangesCount: (doc_id, callback)-> - db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, callback + db.docHistory.count { doc_id : ObjectId(doc_id.toString())}, callback - getArchivedDocChanges: (doc_id, callback)-> - db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: { $exists: true }}, callback - - markDocHistoryAsArchiveInProgress: (doc_id, update, callback) -> - db.docHistory.update { _id: update._id }, { $set : { inS3 : false } }, callback + markDocHistoryAsArchiveInProgress: (doc_id, lastVersion, callback) -> + db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: false, lastVersion: lastVersion}}, {upsert:true}, callback clearDocHistoryAsArchiveInProgress: (doc_id, update, callback) -> - db.docHistory.update { _id: update._id }, { $unset : { inS3 : true } }, callback + db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$unset : {inS3: true, lastVersion: true}}, callback - markDocHistoryAsArchived: (doc_id, update, callback)-> - db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)-> + markDocHistoryAsArchived: (doc_id, lastVersion, callback)-> + db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: true}}, {upsert:true}, (error)-> return callback(error) if error? - db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)-> + # clear the archived entries from the docHistory now we have finally succeeded + db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), v: {$lte : lastVersion}, expiresAt: {$exists : false} }, (error)-> return callback(error) if error? callback(error) markDocHistoryAsUnarchived: (doc_id, callback)-> # note this removes any inS3 field, regardless of its value (true/false/null) - db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)-> + db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, { $unset : { inS3: true, lastVersion: true} }, (error)-> callback(error) diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee index ea8caa3226..eb04a976ab 100644 --- a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -71,27 +71,27 @@ describe "Archiving updates", -> throw error if error? done() - it "should remain one doc change", (done) -> + it "should remain zero doc change", (done) -> db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> throw error if error? - count.should.equal 1 + count.should.equal 0 done() - it "should remained doc marked as inS3", (done) -> - db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> + it "should have docHistoryStats marked as inS3", (done) -> + db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> throw error if error? doc.inS3.should.equal true done() - it "should remained doc have last version", (done) -> - db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> + it "should have docHistoryStats with the last version", (done) -> + db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> throw error if error? - doc.v.should.equal 20 + doc.lastVersion.should.equal 20 done() - it "should store nineteen doc changes in S3", (done) -> + it "should store twenty doc changes in S3", (done) -> TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) => - doc.length.should.equal 19 + doc.length.should.equal 20 done() describe "unarchiving a doc's updates", -> @@ -107,7 +107,8 @@ describe "Archiving updates", -> done() it "should remove doc marked as inS3", (done) -> - db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) -> + db.docHistoryStats.findOne {doc_id: ObjectId(@doc_id)}, (error, doc) -> throw error if error? - count.should.equal 0 + doc.should.not.contain.key('inS3') + doc.should.not.contain.key('lastVersion') done() diff --git a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee index fe648b6e67..cc1b4ec3e7 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee @@ -61,8 +61,8 @@ describe "DocArchiveManager", -> beforeEach -> @update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"} @MongoManager.getDocChangesCount = sinon.stub().callsArg(1) - @MongoManager.getArchivedDocChanges = sinon.stub().callsArgWith(1, null, 0) - @MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update) + @MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, 0) + @MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update, @update.v) @MongoAWS.archiveDocHistory = sinon.stub().callsArg(3) @MongoManager.markDocHistoryAsArchiveInProgress = sinon.stub().callsArg(2) @MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2) @@ -71,7 +71,7 @@ describe "DocArchiveManager", -> it "should run markDocHistoryAsArchived with doc_id and update", -> @MongoManager.markDocHistoryAsArchived .calledWith( - @doc_id, @update + @doc_id, @update.v ) .should.equal true it "should call the callback", -> @@ -107,7 +107,7 @@ describe "DocArchiveManager", -> describe "unArchiveDocChanges", -> beforeEach -> - @MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1) + @MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, {inS3: true}) @MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2) @MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1) @DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback diff --git a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee index 300357e406..d1787c1649 100644 --- a/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee +++ b/services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee @@ -406,21 +406,20 @@ describe "MongoManager", -> @db.docHistory.count .calledWith({ doc_id: ObjectId(@doc_id) - inS3 : { $exists : false } }) .should.equal true it "should call the callback", -> @callback.called.should.equal true - describe "getArchivedDocChanges", -> + describe "getArchivedDocStatus", -> beforeEach -> - @db.docHistory = - count: sinon.stub().callsArg(1) - @MongoManager.getArchivedDocChanges @doc_id, @callback + @db.docHistoryStats = + findOne: sinon.stub().callsArg(2) + @MongoManager.getArchivedDocStatus @doc_id, @callback it "should return if there is any archived doc changes", -> - @db.docHistory.count + @db.docHistoryStats.findOne .calledWith({ doc_id: ObjectId(@doc_id) inS3: {$exists: true} @@ -433,15 +432,16 @@ describe "MongoManager", -> describe "markDocHistoryAsArchived", -> beforeEach -> @update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"} + @db.docHistoryStats = + update: sinon.stub().callsArg(3) @db.docHistory = - update: sinon.stub().callsArg(2) remove: sinon.stub().callsArg(1) - @MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback + @MongoManager.markDocHistoryAsArchived @doc_id, @update.v, @callback - it "should update last doc change with inS3 flag", -> - @db.docHistory.update + it "should update doc status with inS3 flag", -> + @db.docHistoryStats.update .calledWith({ - _id: ObjectId(@update._id) + doc_id: ObjectId(@doc_id) },{ $set : { inS3 : true } }) @@ -451,8 +451,7 @@ describe "MongoManager", -> @db.docHistory.remove .calledWith({ doc_id: ObjectId(@doc_id) - inS3 : { $exists : false } - v: { $lt : @update.v } + v: { $lte : @update.v } expiresAt: {$exists : false} }) .should.equal true @@ -462,18 +461,16 @@ describe "MongoManager", -> describe "markDocHistoryAsUnarchived", -> beforeEach -> - @db.docHistory = - update: sinon.stub().callsArg(3) + @db.docHistoryStats = + update: sinon.stub().callsArg(2) @MongoManager.markDocHistoryAsUnarchived @doc_id, @callback it "should remove any doc changes inS3 flag", -> - @db.docHistory.update + @db.docHistoryStats.update .calledWith({ doc_id: ObjectId(@doc_id) },{ - $unset : { inS3 : true } - },{ - multi: true + $unset : { inS3 : true, lastVersion: true } }) .should.equal true