From 3f712c452a38fe5dea8fb0723d1949f79e947b69 Mon Sep 17 00:00:00 2001
From: Henrique Dias <heukirne@gmail.com>
Date: Thu, 17 Sep 2015 09:23:13 -0300
Subject: [PATCH] add size bulk limit

---
 services/track-changes/app/coffee/MongoAWS.coffee     | 11 ++++++++---
 .../test/unit/coffee/DocArchive/MongoAWS.coffee       |  4 ++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee
index ca4914c733..798126724b 100644
--- a/services/track-changes/app/coffee/MongoAWS.coffee
+++ b/services/track-changes/app/coffee/MongoAWS.coffee
@@ -5,10 +5,12 @@ S3S = require 's3-streams'
 {db, ObjectId} = require "./mongojs"
 JSONStream = require "JSONStream"
 ReadlineStream = require "readline-stream"
+BSON=db.bson.BSON
 
 module.exports = MongoAWS =
 
-	bulkLimit: 10
+	MAX_SIZE:  1024*1024 # almost max size
+	MAX_COUNT: 1024 # almost max count
 
 	archiveDocHistory: (project_id, doc_id, _callback = (error) ->) ->
 
@@ -61,6 +63,7 @@ module.exports = MongoAWS =
 
 		lineStream = new ReadlineStream();
 		ops = [] 
+		sz = 0
 
 		download
 			.on 'open', (obj) ->
@@ -71,11 +74,13 @@ module.exports = MongoAWS =
 			.on 'data', (line) ->
 				if line.length > 2
 					ops.push(JSON.parse(line))
-				if ops.length == MongoAWS.bulkLimit
+					sz += BSON.calculateObjectSize(ops[ops.length-1])
+				if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE
 					download.pause()
 					MongoAWS.handleBulk ops.slice(0), () ->
 						download.resume()
 					ops.splice(0,ops.length)
+					sz = 0
 			.on 'end', () ->
 				MongoAWS.handleBulk ops, callback
 			.on 'error', (err) ->
@@ -95,7 +100,7 @@ module.exports = MongoAWS =
 				if err?
 					logger.error err:err, "error bulking ReadlineStream"
 				else
-					logger.log count:ops.length, result:result, "bulked ReadlineStream"
+					logger.log count:ops.length, result:result, size: BSON.calculateObjectSize(ops), "bulked ReadlineStream"
 				cb(err)
 		else
 			cb()
diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee
index b7a6787f0f..7bbb654cd1 100644
--- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee
+++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee
@@ -21,11 +21,11 @@ describe "MongoAWS", ->
 			"aws-sdk": @awssdk = {}
 			"fs": @fs = {}
 			"s3-streams": @s3streams = {}
-			"./mongojs" : { db: @db = {}, ObjectId: ObjectId }
+			"./mongojs" : { db: @db = { bson: { BSON:{} } }, ObjectId: ObjectId }
 			"JSONStream": @JSONStream = {}
 			"readline-stream": @readline = sinon.stub()
 
-		@bulkLimit = @MongoAWS.bulkLimit
+		@db.bson.BSON.calculateObjectSize = sinon.stub().returns true
 		@project_id = ObjectId().toString()
 		@doc_id = ObjectId().toString()
 		@callback = sinon.stub()