diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index ca4914c733..798126724b 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -5,10 +5,12 @@ S3S = require 's3-streams' {db, ObjectId} = require "./mongojs" JSONStream = require "JSONStream" ReadlineStream = require "readline-stream" +BSON=db.bson.BSON module.exports = MongoAWS = - bulkLimit: 10 + MAX_SIZE: 1024*1024 # almost max size + MAX_COUNT: 1024 # almost max count archiveDocHistory: (project_id, doc_id, _callback = (error) ->) -> @@ -61,6 +63,7 @@ module.exports = MongoAWS = lineStream = new ReadlineStream(); ops = [] + sz = 0 download .on 'open', (obj) -> @@ -71,11 +74,13 @@ module.exports = MongoAWS = .on 'data', (line) -> if line.length > 2 ops.push(JSON.parse(line)) - if ops.length == MongoAWS.bulkLimit + sz += BSON.calculateObjectSize(ops[ops.length-1]) + if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE download.pause() MongoAWS.handleBulk ops.slice(0), () -> download.resume() ops.splice(0,ops.length) + sz = 0 .on 'end', () -> MongoAWS.handleBulk ops, callback .on 'error', (err) -> @@ -95,7 +100,7 @@ module.exports = MongoAWS = if err? logger.error err:err, "error bulking ReadlineStream" else - logger.log count:ops.length, result:result, "bulked ReadlineStream" + logger.log count:ops.length, result:result, size: BSON.calculateObjectSize(ops), "bulked ReadlineStream" cb(err) else cb() diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index b7a6787f0f..7bbb654cd1 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -21,11 +21,11 @@ describe "MongoAWS", -> "aws-sdk": @awssdk = {} "fs": @fs = {} "s3-streams": @s3streams = {} - "./mongojs" : { db: @db = {}, ObjectId: ObjectId } + "./mongojs" : { db: @db = { bson: { BSON:{} } }, ObjectId: ObjectId } "JSONStream": @JSONStream = {} "readline-stream": @readline = sinon.stub() - @bulkLimit = @MongoAWS.bulkLimit + @db.bson.BSON.calculateObjectSize = sinon.stub().returns true @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @callback = sinon.stub()