github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-03 22:54:07 +00:00
Code Issues Projects Releases Packages Wiki Activity

archiving document history now sends all changes to s3

Browse source
This commit is contained in:
Brian Gough 2016-01-15 15:49:07 +00:00
parent 5153ed8217
commit dc564fd5d0
6 changed files with 70 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
services/track-changes/app/coffee/DocArchiveManager.coffee
View file

@ -25,39 +25,38 @@ module.exports = DocArchiveManager =
LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
archiveDocChanges: (project_id, doc_id, callback)->
MongoManager.getArchivedDocStatus doc_id, (error, result) ->
return callback(error) if error?
if result?.inS3 is true
logger.log {project_id, doc_id}, "document history is already archived"
return callback()
if result?.inS3?
logger.log {project_id, doc_id}, "document history archive is already in progress"
return callback()
MongoManager.getDocChangesCount doc_id, (error, count) ->
return callback(error) if error?
if count == 0
logger.log {project_id, doc_id}, "document history is empty, not archiving"
return callback()
else if count == 1
logger.log {project_id, doc_id}, "document history only has one entry, not archiving"
return callback()
else
MongoManager.getArchivedDocChanges doc_id, (error, count) ->
MongoManager.peekLastCompressedUpdate doc_id, (error, update, lastVersion) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "archiving got last compressed update"
MongoManager.markDocHistoryAsArchiveInProgress doc_id, lastVersion, (error) ->
return callback(error) if error?
if count != 0
logger.log {project_id, doc_id}, "document history contains archived entries, not archiving"
return callback()
MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "archiving got last compressed update"
MongoManager.markDocHistoryAsArchiveInProgress doc_id, update, (error) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "marked doc history as archive in progress"
MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
if error?
logger.log {doc_id, project_id, error}, "error exporting document to S3"
MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
return callback(err) if err?
logger.log {doc_id, project_id}, "cleared archive in progress flag"
callback(error)
else
logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
MongoManager.markDocHistoryAsArchived doc_id, update, (error) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "marked doc history as archived"
callback()
logger.log {doc_id, project_id}, "marked doc history as archive in progress"
MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
if error?
logger.log {doc_id, project_id, error}, "error exporting document to S3"
MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
return callback(err) if err?
logger.log {doc_id, project_id}, "cleared archive in progress flag"
callback(error)
else
logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
MongoManager.markDocHistoryAsArchived doc_id, lastVersion, (error) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "marked doc history as archived"
callback()
unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) ->
DocstoreHandler.getAllDocs project_id, (error, docs) ->
@ -75,9 +74,9 @@ module.exports = DocArchiveManager =
LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
unArchiveDocChanges: (project_id, doc_id, callback)->
MongoManager.getArchivedDocChanges doc_id, (error, count) ->
MongoManager.getArchivedDocStatus doc_id, (error, result) ->
return callback(error) if error?
if count == 0
if result?.inS3 isnt true
logger.log {project_id, doc_id}, "no changes marked as in s3, not unarchiving"
return callback()
else

2
services/track-changes/app/coffee/MongoAWS.coffee
View file

@ -19,7 +19,7 @@ module.exports = MongoAWS =
query = {
doc_id: ObjectId(doc_id)
v: {$lt: update.v}
v: {$lte: update.v}
expiresAt: {$exists : false}
}

20
services/track-changes/app/coffee/MongoManager.coffee
View file

@ -157,25 +157,23 @@ module.exports = MongoManager =
db.docHistoryStats.findOne {doc_id: ObjectId(doc_id.toString()), inS3: {$exists:true}}, {inS3: true, lastVersion: true}, callback
getDocChangesCount: (doc_id, callback)->
db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, callback
db.docHistory.count { doc_id : ObjectId(doc_id.toString())}, callback
getArchivedDocChanges: (doc_id, callback)->
db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: { $exists: true }}, callback
markDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
db.docHistory.update { _id: update._id }, { $set : { inS3 : false } }, callback
markDocHistoryAsArchiveInProgress: (doc_id, lastVersion, callback) ->
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: false, lastVersion: lastVersion}}, {upsert:true}, callback
clearDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
db.docHistory.update { _id: update._id }, { $unset : { inS3 : true } }, callback
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$unset : {inS3: true, lastVersion: true}}, callback
markDocHistoryAsArchived: (doc_id, update, callback)->
db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)->
markDocHistoryAsArchived: (doc_id, lastVersion, callback)->
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: true}}, {upsert:true}, (error)->
return callback(error) if error?
db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)->
# clear the archived entries from the docHistory now we have finally succeeded
db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), v: {$lte : lastVersion}, expiresAt: {$exists : false} }, (error)->
return callback(error) if error?
callback(error)
markDocHistoryAsUnarchived: (doc_id, callback)->
# note this removes any inS3 field, regardless of its value (true/false/null)
db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)->
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, { $unset : { inS3: true, lastVersion: true} }, (error)->
callback(error)

23
services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee
View file

@ -71,27 +71,27 @@ describe "Archiving updates", ->
throw error if error?
done()
it "should remain one doc change", (done) ->
it "should remain zero doc change", (done) ->
db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
throw error if error?
count.should.equal 1
count.should.equal 0
done()
it "should remained doc marked as inS3", (done) ->
db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
it "should have docHistoryStats marked as inS3", (done) ->
db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
throw error if error?
doc.inS3.should.equal true
done()
it "should remained doc have last version", (done) ->
db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
it "should have docHistoryStats with the last version", (done) ->
db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
throw error if error?
doc.v.should.equal 20
doc.lastVersion.should.equal 20
done()
it "should store nineteen doc changes in S3", (done) ->
it "should store twenty doc changes in S3", (done) ->
TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) =>
doc.length.should.equal 19
doc.length.should.equal 20
done()
describe "unarchiving a doc's updates", ->
@ -107,7 +107,8 @@ describe "Archiving updates", ->
done()
it "should remove doc marked as inS3", (done) ->
db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) ->
db.docHistoryStats.findOne {doc_id: ObjectId(@doc_id)}, (error, doc) ->
throw error if error?
count.should.equal 0
doc.should.not.contain.key('inS3')
doc.should.not.contain.key('lastVersion')
done()

8
services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee
View file

@ -61,8 +61,8 @@ describe "DocArchiveManager", ->
beforeEach ->
@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
@MongoManager.getDocChangesCount = sinon.stub().callsArg(1)
@MongoManager.getArchivedDocChanges = sinon.stub().callsArgWith(1, null, 0)
@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, 0)
@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update, @update.v)
@MongoAWS.archiveDocHistory = sinon.stub().callsArg(3)
@MongoManager.markDocHistoryAsArchiveInProgress = sinon.stub().callsArg(2)
@MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2)
@ -71,7 +71,7 @@ describe "DocArchiveManager", ->
it "should run markDocHistoryAsArchived with doc_id and update", ->
@MongoManager.markDocHistoryAsArchived
.calledWith(
@doc_id, @update
@doc_id, @update.v
)
.should.equal true
it "should call the callback", ->
@ -107,7 +107,7 @@ describe "DocArchiveManager", ->
describe "unArchiveDocChanges", ->
beforeEach ->
@MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1)
@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, {inS3: true})
@MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2)
@MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1)
@DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback

35
services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee
View file

@ -406,21 +406,20 @@ describe "MongoManager", ->
@db.docHistory.count
.calledWith({
doc_id: ObjectId(@doc_id)
inS3 : { $exists : false }
})
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "getArchivedDocChanges", ->
describe "getArchivedDocStatus", ->
beforeEach ->
@db.docHistory =
count: sinon.stub().callsArg(1)
@MongoManager.getArchivedDocChanges @doc_id, @callback
@db.docHistoryStats =
findOne: sinon.stub().callsArg(2)
@MongoManager.getArchivedDocStatus @doc_id, @callback
it "should return if there is any archived doc changes", ->
@db.docHistory.count
@db.docHistoryStats.findOne
.calledWith({
doc_id: ObjectId(@doc_id)
inS3: {$exists: true}
@ -433,15 +432,16 @@ describe "MongoManager", ->
describe "markDocHistoryAsArchived", ->
beforeEach ->
@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
@db.docHistoryStats =
update: sinon.stub().callsArg(3)
@db.docHistory =
update: sinon.stub().callsArg(2)
remove: sinon.stub().callsArg(1)
@MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback
@MongoManager.markDocHistoryAsArchived @doc_id, @update.v, @callback
it "should update last doc change with inS3 flag", ->
@db.docHistory.update
it "should update doc status with inS3 flag", ->
@db.docHistoryStats.update
.calledWith({
_id: ObjectId(@update._id)
doc_id: ObjectId(@doc_id)
},{
$set : { inS3 : true }
})
@ -451,8 +451,7 @@ describe "MongoManager", ->
@db.docHistory.remove
.calledWith({
doc_id: ObjectId(@doc_id)
inS3 : { $exists : false }
v: { $lt : @update.v }
v: { $lte : @update.v }
expiresAt: {$exists : false}
})
.should.equal true
@ -462,18 +461,16 @@ describe "MongoManager", ->
describe "markDocHistoryAsUnarchived", ->
beforeEach ->
@db.docHistory =
update: sinon.stub().callsArg(3)
@db.docHistoryStats =
update: sinon.stub().callsArg(2)
@MongoManager.markDocHistoryAsUnarchived @doc_id, @callback
it "should remove any doc changes inS3 flag", ->
@db.docHistory.update
@db.docHistoryStats.update
.calledWith({
doc_id: ObjectId(@doc_id)
},{
$unset : { inS3 : true }
},{
multi: true
$unset : { inS3 : true, lastVersion: true }
})
.should.equal true