mirror of
https://github.com/overleaf/overleaf.git
synced 2024-09-16 02:52:31 -04:00
3d9dfeccc3
remove the op-specific code remove tests for ops, now only packing remove unused packing code work in progress store index for completed packs only support archiving and unarchiving of individual packs remove support for archiving whole document history split out ArchiveManager, IndexManager remove old DocArchive code remove docHistoryStats collection comment about archiving added method to look at index when last pack has been archived added start of iterator for project results use a proper iterator added heap module getting it working increase pack size since bulk operations no longer needed remove unused MongoAWSexternal cleanup added doc iterator remove old query code added missing files cleanup clean upclean up started adding pack worker for archiving work in progress work in progress getting pack worker working updating worker getting packworker working added lock use correct key name for track changes aws access use correct key name for track changes aws access always send back users array fix up comparison of retrieved objects handle op ids inside packs log when s3 download completes comments cleanup, remove finalisation ideacleanup, remove finalisation idea remove logging
66 lines
2.1 KiB
CoffeeScript
66 lines
2.1 KiB
CoffeeScript
async = require "async"
|
|
_ = require "underscore"
|
|
{db, ObjectId, BSON} = require "./mongojs"
|
|
logger = require "logger-sharelatex"
|
|
Heap = require "heap"
|
|
|
|
module.exports = ProjectIterator =
|
|
|
|
class ProjectIterator
|
|
constructor: (packs, @before, @getPackByIdFn) ->
|
|
byEndTs = (a,b) -> (b.meta.end_ts - a.meta.end_ts) || (a.fromIndex - b.fromIndex)
|
|
@packs = packs.slice().sort byEndTs
|
|
@queue = new Heap(byEndTs)
|
|
|
|
next: (callback) ->
|
|
# what's up next
|
|
#console.log ">>> top item", iterator.packs[0]
|
|
iterator = this
|
|
before = @before
|
|
queue = iterator.queue
|
|
opsToReturn = []
|
|
nextPack = iterator.packs[0]
|
|
lowWaterMark = nextPack?.meta.end_ts || 0
|
|
nextItem = queue.peek()
|
|
|
|
#console.log "queue empty?", queue.empty()
|
|
#console.log "nextItem", nextItem
|
|
#console.log "nextItem.meta.end_ts", nextItem?.meta.end_ts
|
|
#console.log "lowWaterMark", lowWaterMark
|
|
|
|
while before? and nextPack?.meta.start_ts > before
|
|
# discard pack that is outside range
|
|
iterator.packs.shift()
|
|
nextPack = iterator.packs[0]
|
|
lowWaterMark = nextPack?.meta.end_ts || 0
|
|
|
|
if (queue.empty() or nextItem?.meta.end_ts <= lowWaterMark) and nextPack?
|
|
# retrieve the next pack and populate the queue
|
|
return @getPackByIdFn nextPack.project_id, nextPack.doc_id, nextPack._id, (err, pack) ->
|
|
return callback(err) if err?
|
|
iterator.packs.shift() # have now retrieved this pack, remove it
|
|
#console.log "got pack", pack
|
|
for op in pack.pack when (not before? or op.meta.end_ts < before)
|
|
#console.log "adding op", op
|
|
op.doc_id = nextPack.doc_id
|
|
op.project_id = nextPack.project_id
|
|
queue.push op
|
|
# now try again
|
|
return iterator.next(callback)
|
|
|
|
#console.log "nextItem", nextItem, "lowWaterMark", lowWaterMark
|
|
while nextItem? and (nextItem?.meta.end_ts > lowWaterMark)
|
|
opsToReturn.push nextItem
|
|
queue.pop()
|
|
nextItem = queue.peek()
|
|
|
|
#console.log "queue empty?", queue.empty()
|
|
#console.log "nextPack", nextPack?
|
|
|
|
if queue.empty() and not nextPack? # got everything
|
|
iterator._done = true
|
|
|
|
callback(null, opsToReturn)
|
|
|
|
done: () ->
|
|
return @_done
|