diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index f12691027b..1e5c4e1635 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -48,6 +48,9 @@ app.post "/project/:project_id/flush", HttpController.flushProject app.post "/project/:project_id/doc/:doc_id/version/:version/restore", HttpController.restore +app.post '/project/:project_id/doc/:doc_id/push', HttpController.pushDocHistory +app.post '/project/:project_id/doc/:doc_id/pull', HttpController.pullDocHistory + packWorker = null # use a single packing worker app.post "/pack", (req, res, next) -> diff --git a/services/track-changes/app/coffee/HttpController.coffee b/services/track-changes/app/coffee/HttpController.coffee index 6c357c505f..eecc618330 100644 --- a/services/track-changes/app/coffee/HttpController.coffee +++ b/services/track-changes/app/coffee/HttpController.coffee @@ -77,6 +77,22 @@ module.exports = HttpController = return next(error) if error? res.send 204 + pushDocHistory: (req, res, next = (error) ->) -> + project_id = req.params.project_id + doc_id = req.params.doc_id + logger.log {project_id, doc_id}, "pushing all finalised changes to s3" + PackManager.pushOldPacks project_id, doc_id, (error) -> + return next(error) if error? + res.send 204 + + pullDocHistory: (req, res, next = (error) ->) -> + project_id = req.params.project_id + doc_id = req.params.doc_id + logger.log {project_id, doc_id}, "pulling all packs from s3" + PackManager.pullOldPacks project_id, doc_id, (error) -> + return next(error) if error? + res.send 204 + healthCheck: (req, res)-> HealthChecker.check (err)-> if err? diff --git a/services/track-changes/app/coffee/PackManager.coffee b/services/track-changes/app/coffee/PackManager.coffee index 449e07e110..4d48018190 100644 --- a/services/track-changes/app/coffee/PackManager.coffee +++ b/services/track-changes/app/coffee/PackManager.coffee @@ -349,6 +349,17 @@ module.exports = PackManager = else logger.err {pack, result, jsondiff: JSON.stringify(pack) is JSON.stringify(result)}, "difference when comparing packs" callback new Error("pack retrieved from s3 does not match pack in mongo") + # Extra methods to test archive/unarchive for a doc_id + + pushOldPacks: (project_id, doc_id, callback) -> + PackManager.findCompletedPacks project_id, doc_id, (err, packs) -> + return callback(err) if err? + return callback() if not packs?.length + PackManager.processOldPack project_id, doc_id, packs[0]._id, callback + + pullOldPacks: (project_id, doc_id, callback) -> + PackManager.loadPacksByVersionRange project_id, doc_id, null, null, callback + # Processing old packs via worker diff --git a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee index ab06b991b8..02422d3456 100644 --- a/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee +++ b/services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee @@ -42,15 +42,15 @@ describe "Archiving updates", -> sinon.spy MockDocStoreApi, "getAllDoc" @updates = [] - for i in [0..9] + for i in [0..1024+9] @updates.push { op: [{ i: "a", p: 0 }] - meta: { ts: @now - (9 - i) * @hours - 2 * @minutes, user_id: @user_id } + meta: { ts: @now - i * @hours, user_id: @user_id } v: 2 * i + 1 } @updates.push { op: [{ i: "b", p: 0 }] - meta: { ts: @now - (9 - i) * @hours, user_id: @user_id } + meta: { ts: @now - i * @hours + 10*@minutes, user_id: @user_id } v: 2 * i + 2 } @@ -67,49 +67,53 @@ describe "Archiving updates", -> describe "archiving a doc's updates", -> before (done) -> - TrackChangesClient.archiveProject @project_id, (error) -> + TrackChangesClient.pushDocHistory @project_id, @doc_id, (error) -> throw error if error? done() - it "should remain zero doc change", (done) -> - db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> - throw error if error? - count.should.equal 0 - done() - - it "should have docHistoryStats marked as inS3", (done) -> - db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> - throw error if error? - doc.inS3.should.equal true - done() - - it "should have docHistoryStats with the last version", (done) -> - db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) -> - throw error if error? - doc.lastVersion.should.equal 20 - done() - - it "should store twenty doc changes in S3 in one pack", (done) -> - TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) => - doc.length.should.equal 1 - doc[0].pack.length.should.equal 20 - done() - - describe "unarchiving a doc's updates", -> - before (done) -> - TrackChangesClient.unarchiveProject @project_id, (error) -> - throw error if error? - done() - - it "should restore doc changes", (done) -> - db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> + it "should have one cached pack", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id), expiresAt:{$exists:true}}, (error, count) -> throw error if error? count.should.equal 1 done() - it "should remove doc marked as inS3", (done) -> - db.docHistoryStats.findOne {doc_id: ObjectId(@doc_id)}, (error, doc) -> + it "should have one remaining pack after cache is expired", (done) -> + db.docHistory.remove { + doc_id: ObjectId(@doc_id), + expiresAt:{$exists:true} + }, (err, result) => throw error if error? - doc.should.not.contain.key('inS3') - doc.should.not.contain.key('lastVersion') + db.docHistory.count { doc_id: ObjectId(@doc_id)}, (error, count) -> + throw error if error? + count.should.equal 1 + done() + + it "should have a docHistoryIndex entry marked as inS3", (done) -> + db.docHistoryIndex.findOne { _id: ObjectId(@doc_id) }, (error, index) -> + throw error if error? + index.packs[0].inS3.should.equal true + done() + + it "should have a docHistoryIndex entry with the last version", (done) -> + db.docHistoryIndex.findOne { _id: ObjectId(@doc_id) }, (error, index) -> + throw error if error? + index.packs[0].v_end.should.equal 100 + done() + + # it "should store twenty doc changes in S3 in one pack", (done) -> + # TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) => + # doc.length.should.equal 1 + # doc[0].pack.length.should.equal 20 + # done() + + describe "unarchiving a doc's updates", -> + before (done) -> + TrackChangesClient.pullDocHistory @project_id, @doc_id, (error) -> + throw error if error? + done() + + it "should restore both packs", (done) -> + db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) -> + throw error if error? + count.should.equal 2 done() diff --git a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee index d7872e288a..0140b637ad 100644 --- a/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee +++ b/services/track-changes/test/acceptance/coffee/helpers/TrackChangesClient.coffee @@ -74,16 +74,16 @@ module.exports = TrackChangesClient = response.statusCode.should.equal 204 callback null - archiveProject: (project_id, callback = (error) ->) -> + pushDocHistory: (project_id, doc_id, callback = (error) ->) -> request.post { - url: "http://localhost:3015/project/#{project_id}/archive" + url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/push" }, (error, response, body) => response.statusCode.should.equal 204 callback(error) - unarchiveProject: (project_id, callback = (error) ->) -> + pullDocHistory: (project_id, doc_id, callback = (error) ->) -> request.post { - url: "http://localhost:3015/project/#{project_id}/unarchive" + url: "http://localhost:3015/project/#{project_id}/doc/#{doc_id}/pull" }, (error, response, body) => response.statusCode.should.equal 204 callback(error) @@ -91,12 +91,12 @@ module.exports = TrackChangesClient = buildS3Options: (content, key)-> return { aws: - key: Settings.filestore.s3.key - secret: Settings.filestore.s3.secret - bucket: Settings.filestore.stores.user_files + key: Settings.trackchanges.s3.key + secret: Settings.trackchanges.s3.secret + bucket: Settings.trackchanges.stores.doc_history timeout: 30 * 1000 json: content - uri:"https://#{Settings.filestore.stores.user_files}.s3.amazonaws.com/#{key}" + uri:"https://#{Settings.trackchanges.stores.doc_history}.s3.amazonaws.com/#{key}" } getS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> @@ -105,4 +105,4 @@ module.exports = TrackChangesClient = removeS3Doc: (project_id, doc_id, callback = (error, res, body) ->) -> options = TrackChangesClient.buildS3Options(true, project_id+"/changes-"+doc_id) - request.del options, callback \ No newline at end of file + request.del options, callback