github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-04-02 09:25:47 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #11 from sharelatex/archive-all-data-in-history

Browse source 
Archive all data in history
This commit is contained in:
Brian Gough 2016-01-15 16:43:54 +00:00
commit 184204130c
8 changed files with 95 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
services/track-changes/app/coffee/DocArchiveManager.coffee
View file

@ -25,39 +25,38 @@ module.exports = DocArchiveManager =
LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
archiveDocChanges: (project_id, doc_id, callback)->
MongoManager.getArchivedDocStatus doc_id, (error, result) ->
return callback(error) if error?
if result?.inS3 is true
logger.log {project_id, doc_id}, "document history is already archived"
return callback()
if result?.inS3?
logger.log {project_id, doc_id}, "document history archive is already in progress"
return callback()
MongoManager.getDocChangesCount doc_id, (error, count) ->
return callback(error) if error?
if count == 0
logger.log {project_id, doc_id}, "document history is empty, not archiving"
return callback()
else if count == 1
logger.log {project_id, doc_id}, "document history only has one entry, not archiving"
return callback()
else
MongoManager.getArchivedDocChanges doc_id, (error, count) ->
MongoManager.peekLastCompressedUpdate doc_id, (error, update, lastVersion) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "archiving got last compressed update"
MongoManager.markDocHistoryAsArchiveInProgress doc_id, lastVersion, (error) ->
return callback(error) if error?
if count != 0
logger.log {project_id, doc_id}, "document history contains archived entries, not archiving"
return callback()
MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "archiving got last compressed update"
MongoManager.markDocHistoryAsArchiveInProgress doc_id, update, (error) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "marked doc history as archive in progress"
MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
if error?
logger.log {doc_id, project_id, error}, "error exporting document to S3"
MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
return callback(err) if err?
logger.log {doc_id, project_id}, "cleared archive in progress flag"
callback(error)
else
logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
MongoManager.markDocHistoryAsArchived doc_id, update, (error) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "marked doc history as archived"
callback()
logger.log {doc_id, project_id}, "marked doc history as archive in progress"
MongoAWS.archiveDocHistory project_id, doc_id, update, (error) ->
if error?
logger.log {doc_id, project_id, error}, "error exporting document to S3"
MongoManager.clearDocHistoryAsArchiveInProgress doc_id, update, (err) ->
return callback(err) if err?
logger.log {doc_id, project_id}, "cleared archive in progress flag"
callback(error)
else
logger.log doc_id:doc_id, project_id:project_id, "exported document to S3"
MongoManager.markDocHistoryAsArchived doc_id, lastVersion, (error) ->
return callback(error) if error?
logger.log {doc_id, project_id}, "marked doc history as archived"
callback()
unArchiveAllDocsChanges: (project_id, callback = (error, docs) ->) ->
DocstoreHandler.getAllDocs project_id, (error, docs) ->
@ -75,9 +74,9 @@ module.exports = DocArchiveManager =
LockManager.runWithLock("HistoryLock:#{doc_id}", job, callback)
unArchiveDocChanges: (project_id, doc_id, callback)->
MongoManager.getArchivedDocChanges doc_id, (error, count) ->
MongoManager.getArchivedDocStatus doc_id, (error, result) ->
return callback(error) if error?
if count == 0
if result?.inS3 isnt true
logger.log {project_id, doc_id}, "no changes marked as in s3, not unarchiving"
return callback()
else

2
services/track-changes/app/coffee/MongoAWS.coffee
View file

@ -19,7 +19,7 @@ module.exports = MongoAWS =
query = {
doc_id: ObjectId(doc_id)
v: {$lt: update.v}
v: {$lte: update.v}
expiresAt: {$exists : false}
}

40
services/track-changes/app/coffee/MongoManager.coffee
View file

@ -18,7 +18,7 @@ module.exports = MongoManager =
peekLastCompressedUpdate: (doc_id, callback = (error, update, version) ->) ->
# under normal use we pass back the last update as
# callback(null,update).
# callback(null,update,version).
#
# when we have an existing last update but want to force a new one
# to start, we pass it back as callback(null,null,version), just
@ -26,17 +26,18 @@ module.exports = MongoManager =
MongoManager.getLastCompressedUpdate doc_id, (error, update) ->
return callback(error) if error?
if update?
if update.inS3?
# we want to force a new update, but ensure that it is
# consistent with the version of the existing one in S3
return callback null, null, update.v
else if update.broken
if update.broken
# the update is marked as broken so we will force a new op
return callback null, null
else if update.pack?
return callback null, update, update.pack[0]?.v
else
return callback null, update
return callback null, update, update.v
else
callback null, null
MongoManager.getArchivedDocStatus doc_id, (error, status) ->
return callback(error) if error?
return callback(null, null, status.lastVersion) if status?.inS3? and status?.lastVersion?
callback null, null
insertCompressedUpdates: (project_id, doc_id, updates, temporary, callback = (error) ->) ->
jobs = []
@ -152,26 +153,27 @@ module.exports = MongoManager =
db.docHistoryStats.ensureIndex { doc_id: 1 }, { background: true }
db.docHistoryStats.ensureIndex { updates: -1, doc_id: 1 }, { background: true }
getArchivedDocStatus: (doc_id, callback)->
db.docHistoryStats.findOne {doc_id: ObjectId(doc_id.toString()), inS3: {$exists:true}}, {inS3: true, lastVersion: true}, callback
getDocChangesCount: (doc_id, callback)->
db.docHistory.count { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }}, callback
db.docHistory.count { doc_id : ObjectId(doc_id.toString())}, callback
getArchivedDocChanges: (doc_id, callback)->
db.docHistory.count { doc_id: ObjectId(doc_id.toString()) , inS3: { $exists: true }}, callback
markDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
db.docHistory.update { _id: update._id }, { $set : { inS3 : false } }, callback
markDocHistoryAsArchiveInProgress: (doc_id, lastVersion, callback) ->
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: false, lastVersion: lastVersion}}, {upsert:true}, callback
clearDocHistoryAsArchiveInProgress: (doc_id, update, callback) ->
db.docHistory.update { _id: update._id }, { $unset : { inS3 : true } }, callback
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$unset : {inS3: true, lastVersion: true}}, callback
markDocHistoryAsArchived: (doc_id, update, callback)->
db.docHistory.update { _id: update._id }, { $set : { inS3 : true } }, (error)->
markDocHistoryAsArchived: (doc_id, lastVersion, callback)->
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, {$set : {inS3: true}}, {upsert:true}, (error)->
return callback(error) if error?
db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), inS3 : { $exists : false }, v: { $lt : update.v }, expiresAt: {$exists : false} }, (error)->
# clear the archived entries from the docHistory now we have finally succeeded
db.docHistory.remove { doc_id : ObjectId(doc_id.toString()), v: {$lte : lastVersion}, expiresAt: {$exists : false} }, (error)->
return callback(error) if error?
callback(error)
markDocHistoryAsUnarchived: (doc_id, callback)->
# note this removes any inS3 field, regardless of its value (true/false/null)
db.docHistory.update { doc_id: ObjectId(doc_id.toString()) }, { $unset : { inS3 : true } }, { multi: true }, (error)->
db.docHistoryStats.update {doc_id: ObjectId(doc_id.toString())}, { $unset : { inS3: true, lastVersion: true} }, (error)->
callback(error)

16
services/track-changes/app/coffee/UpdatesManager.coffee
View file

@ -17,17 +17,13 @@ module.exports = UpdatesManager =
return callback()
MongoManager.peekLastCompressedUpdate doc_id, (error, lastCompressedUpdate, lastVersion) ->
# lastCompressedUpdate is the most recent update in Mongo.
# lastCompressedUpdate is the most recent update in Mongo, and
# lastVersion is its sharejs version number.
#
# The peekLastCompressedUpdate method may pass it back as 'null'
# to force the start of a new compressed update, even when there
# was a previous compressed update in Mongo. In this case it
# passes back the lastVersion from the update to check
# consistency.
# when lastVersion is not provided, default to lastCompressedUpdate.v
lastVersion ?= lastCompressedUpdate?.v
# The peekLastCompressedUpdate method may pass the update back
# as 'null' (for example if the previous compressed update has
# been archived). In this case it can still pass back the
# lastVersion from the update to allow us to check consistency.
return callback(error) if error?
# Ensure that raw updates start where lastVersion left off

23
services/track-changes/test/acceptance/coffee/ArchivingUpdatesTests.coffee
View file

@ -71,27 +71,27 @@ describe "Archiving updates", ->
throw error if error?
done()
it "should remain one doc change", (done) ->
it "should remain zero doc change", (done) ->
db.docHistory.count { doc_id: ObjectId(@doc_id) }, (error, count) ->
throw error if error?
count.should.equal 1
count.should.equal 0
done()
it "should remained doc marked as inS3", (done) ->
db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
it "should have docHistoryStats marked as inS3", (done) ->
db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
throw error if error?
doc.inS3.should.equal true
done()
it "should remained doc have last version", (done) ->
db.docHistory.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
it "should have docHistoryStats with the last version", (done) ->
db.docHistoryStats.findOne { doc_id: ObjectId(@doc_id) }, (error, doc) ->
throw error if error?
doc.v.should.equal 20
doc.lastVersion.should.equal 20
done()
it "should store nineteen doc changes in S3", (done) ->
it "should store twenty doc changes in S3", (done) ->
TrackChangesClient.getS3Doc @project_id, @doc_id, (error, res, doc) =>
doc.length.should.equal 19
doc.length.should.equal 20
done()
describe "unarchiving a doc's updates", ->
@ -107,7 +107,8 @@ describe "Archiving updates", ->
done()
it "should remove doc marked as inS3", (done) ->
db.docHistory.count { doc_id: ObjectId(@doc_id), inS3 : true }, (error, count) ->
db.docHistoryStats.findOne {doc_id: ObjectId(@doc_id)}, (error, doc) ->
throw error if error?
count.should.equal 0
doc.should.not.contain.key('inS3')
doc.should.not.contain.key('lastVersion')
done()

8
services/track-changes/test/unit/coffee/DocArchive/DocArchiveManager.coffee
View file

@ -61,8 +61,8 @@ describe "DocArchiveManager", ->
beforeEach ->
@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
@MongoManager.getDocChangesCount = sinon.stub().callsArg(1)
@MongoManager.getArchivedDocChanges = sinon.stub().callsArgWith(1, null, 0)
@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, 0)
@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update, @update.v)
@MongoAWS.archiveDocHistory = sinon.stub().callsArg(3)
@MongoManager.markDocHistoryAsArchiveInProgress = sinon.stub().callsArg(2)
@MongoManager.markDocHistoryAsArchived = sinon.stub().callsArg(2)
@ -71,7 +71,7 @@ describe "DocArchiveManager", ->
it "should run markDocHistoryAsArchived with doc_id and update", ->
@MongoManager.markDocHistoryAsArchived
.calledWith(
@doc_id, @update
@doc_id, @update.v
)
.should.equal true
it "should call the callback", ->
@ -107,7 +107,7 @@ describe "DocArchiveManager", ->
describe "unArchiveDocChanges", ->
beforeEach ->
@MongoManager.getArchivedDocChanges = sinon.stub().callsArg(1)
@MongoManager.getArchivedDocStatus = sinon.stub().callsArgWith(1, null, {inS3: true})
@MongoAWS.unArchiveDocHistory = sinon.stub().callsArg(2)
@MongoManager.markDocHistoryAsUnarchived = sinon.stub().callsArg(1)
@DocArchiveManager.unArchiveDocChanges @project_id, @doc_id, @callback

43
services/track-changes/test/unit/coffee/MongoManager/MongoManagerTests.coffee
View file

@ -57,6 +57,8 @@ describe "MongoManager", ->
describe "peekLastCompressedUpdate", ->
describe "when there is no last update", ->
beforeEach ->
@db.docHistoryStats = {}
@db.docHistoryStats.findOne = sinon.stub().callsArgWith(2, null, null)
@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, null)
@MongoManager.peekLastCompressedUpdate @doc_id, @callback
@ -84,8 +86,10 @@ describe "MongoManager", ->
describe "when there is a last update in S3", ->
beforeEach ->
@update = { _id: Object(), v: 12345, inS3: true }
@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @update)
@update = { _id: Object(), v: 12345}
@db.docHistoryStats = {}
@db.docHistoryStats.findOne = sinon.stub().callsArgWith(2, null, {inS3:true, lastVersion: @update.v})
@MongoManager.getLastCompressedUpdate = sinon.stub().callsArgWith(1, null)
@MongoManager.peekLastCompressedUpdate @doc_id, @callback
it "should get the last update", ->
@ -402,21 +406,20 @@ describe "MongoManager", ->
@db.docHistory.count
.calledWith({
doc_id: ObjectId(@doc_id)
inS3 : { $exists : false }
})
.should.equal true
it "should call the callback", ->
@callback.called.should.equal true
describe "getArchivedDocChanges", ->
describe "getArchivedDocStatus", ->
beforeEach ->
@db.docHistory =
count: sinon.stub().callsArg(1)
@MongoManager.getArchivedDocChanges @doc_id, @callback
@db.docHistoryStats =
findOne: sinon.stub().callsArg(2)
@MongoManager.getArchivedDocStatus @doc_id, @callback
it "should return if there is any archived doc changes", ->
@db.docHistory.count
@db.docHistoryStats.findOne
.calledWith({
doc_id: ObjectId(@doc_id)
inS3: {$exists: true}
@ -429,15 +432,16 @@ describe "MongoManager", ->
describe "markDocHistoryAsArchived", ->
beforeEach ->
@update = { _id: ObjectId(), op: "op", meta: "meta", v: "v"}
@db.docHistoryStats =
update: sinon.stub().callsArg(3)
@db.docHistory =
update: sinon.stub().callsArg(2)
remove: sinon.stub().callsArg(1)
@MongoManager.markDocHistoryAsArchived @doc_id, @update, @callback
@MongoManager.markDocHistoryAsArchived @doc_id, @update.v, @callback
it "should update last doc change with inS3 flag", ->
@db.docHistory.update
it "should update doc status with inS3 flag", ->
@db.docHistoryStats.update
.calledWith({
_id: ObjectId(@update._id)
doc_id: ObjectId(@doc_id)
},{
$set : { inS3 : true }
})
@ -447,8 +451,7 @@ describe "MongoManager", ->
@db.docHistory.remove
.calledWith({
doc_id: ObjectId(@doc_id)
inS3 : { $exists : false }
v: { $lt : @update.v }
v: { $lte : @update.v }
expiresAt: {$exists : false}
})
.should.equal true
@ -458,18 +461,16 @@ describe "MongoManager", ->
describe "markDocHistoryAsUnarchived", ->
beforeEach ->
@db.docHistory =
update: sinon.stub().callsArg(3)
@db.docHistoryStats =
update: sinon.stub().callsArg(2)
@MongoManager.markDocHistoryAsUnarchived @doc_id, @callback
it "should remove any doc changes inS3 flag", ->
@db.docHistory.update
@db.docHistoryStats.update
.calledWith({
doc_id: ObjectId(@doc_id)
},{
$unset : { inS3 : true }
},{
multi: true
$unset : { inS3 : true, lastVersion: true }
})
.should.equal true

2
services/track-changes/test/unit/coffee/UpdatesManager/UpdatesManagerTests.coffee
View file

@ -69,7 +69,7 @@ describe "UpdatesManager", ->
@lastCompressedUpdate = { v: 11, op: "compressed-op-11" }
@compressedUpdates = [ { v: 12, op: "compressed-op-11+12" }, { v: 13, op: "compressed-op-12" } ]
@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate)
@MongoManager.peekLastCompressedUpdate = sinon.stub().callsArgWith(1, null, @lastCompressedUpdate, @lastCompressedUpdate.v)
@MongoManager.modifyCompressedUpdate = sinon.stub().callsArg(2)
@MongoManager.insertCompressedUpdates = sinon.stub().callsArg(4)
@UpdateCompressor.compressRawUpdates = sinon.stub().returns(@compressedUpdates)