From aa66c5ee8c74e15628263b1f23dcb192263e5d35 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 17 Sep 2015 10:41:53 -0300 Subject: [PATCH] improve size function --- services/track-changes/app/coffee/MongoAWS.coffee | 11 +++++------ .../test/unit/coffee/DocArchive/MongoAWS.coffee | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/services/track-changes/app/coffee/MongoAWS.coffee b/services/track-changes/app/coffee/MongoAWS.coffee index 798126724b..f2ad22bdf4 100644 --- a/services/track-changes/app/coffee/MongoAWS.coffee +++ b/services/track-changes/app/coffee/MongoAWS.coffee @@ -5,7 +5,6 @@ S3S = require 's3-streams' {db, ObjectId} = require "./mongojs" JSONStream = require "JSONStream" ReadlineStream = require "readline-stream" -BSON=db.bson.BSON module.exports = MongoAWS = @@ -74,19 +73,19 @@ module.exports = MongoAWS = .on 'data', (line) -> if line.length > 2 ops.push(JSON.parse(line)) - sz += BSON.calculateObjectSize(ops[ops.length-1]) + sz += line.length if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE download.pause() - MongoAWS.handleBulk ops.slice(0), () -> + MongoAWS.handleBulk ops.slice(0), sz, () -> download.resume() ops.splice(0,ops.length) sz = 0 .on 'end', () -> - MongoAWS.handleBulk ops, callback + MongoAWS.handleBulk ops, sz, callback .on 'error', (err) -> return callback(err) - handleBulk: (ops, cb) -> + handleBulk: (ops, size, cb) -> bulk = db.docHistory.initializeUnorderedBulkOp(); for op in ops @@ -100,7 +99,7 @@ module.exports = MongoAWS = if err? logger.error err:err, "error bulking ReadlineStream" else - logger.log count:ops.length, result:result, size: BSON.calculateObjectSize(ops), "bulked ReadlineStream" + logger.log count:ops.length, result:result, size: size, "bulked ReadlineStream" cb(err) else cb() diff --git a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee index 7bbb654cd1..0bad498eb5 100644 --- a/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee +++ b/services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee @@ -21,11 +21,10 @@ describe "MongoAWS", -> "aws-sdk": @awssdk = {} "fs": @fs = {} "s3-streams": @s3streams = {} - "./mongojs" : { db: @db = { bson: { BSON:{} } }, ObjectId: ObjectId } + "./mongojs" : { db: @db = {}, ObjectId: ObjectId } "JSONStream": @JSONStream = {} "readline-stream": @readline = sinon.stub() - @db.bson.BSON.calculateObjectSize = sinon.stub().returns true @project_id = ObjectId().toString() @doc_id = ObjectId().toString() @callback = sinon.stub() @@ -102,7 +101,7 @@ describe "MongoAWS", -> execute: sinon.stub().callsArgWith(0, null, {}) @db.docHistory = {} @db.docHistory.initializeUnorderedBulkOp = sinon.stub().returns @bulk - @MongoAWS.handleBulk @bulkOps, @callback + @MongoAWS.handleBulk @bulkOps, @bulkOps.length, @callback it "should call updateOne for each operation", -> @bulk.find.calledWith({_id:@bulkOps[0]._id}).should.equal true