github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-12-24 03:23:10 +00:00
Code Issues Projects Releases Packages Wiki Activity

add size bulk limit

Browse source
This commit is contained in:
Henrique Dias 2015-09-17 09:23:13 -03:00
parent 01cbfd5338
commit 3f712c452a
2 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
services/track-changes/app/coffee/MongoAWS.coffee
View file

@ -5,10 +5,12 @@ S3S = require 's3-streams'
{db, ObjectId} = require "./mongojs"
JSONStream = require "JSONStream"
ReadlineStream = require "readline-stream"
BSON=db.bson.BSON
module.exports = MongoAWS =
bulkLimit: 10
MAX_SIZE: 1024*1024 # almost max size
MAX_COUNT: 1024 # almost max count
archiveDocHistory: (project_id, doc_id, _callback = (error) ->) ->
@ -61,6 +63,7 @@ module.exports = MongoAWS =
lineStream = new ReadlineStream();
ops = []
sz = 0
download
.on 'open', (obj) ->
@ -71,11 +74,13 @@ module.exports = MongoAWS =
.on 'data', (line) ->
if line.length > 2
ops.push(JSON.parse(line))
if ops.length == MongoAWS.bulkLimit
sz += BSON.calculateObjectSize(ops[ops.length-1])
if ops.length >= MongoAWS.MAX_COUNT || sz >= MongoAWS.MAX_SIZE
download.pause()
MongoAWS.handleBulk ops.slice(0), () ->
download.resume()
ops.splice(0,ops.length)
sz = 0
.on 'end', () ->
MongoAWS.handleBulk ops, callback
.on 'error', (err) ->
@ -95,7 +100,7 @@ module.exports = MongoAWS =
if err?
logger.error err:err, "error bulking ReadlineStream"
else
logger.log count:ops.length, result:result, "bulked ReadlineStream"
logger.log count:ops.length, result:result, size: BSON.calculateObjectSize(ops), "bulked ReadlineStream"
cb(err)
else
cb()

4
services/track-changes/test/unit/coffee/DocArchive/MongoAWS.coffee
View file

@ -21,11 +21,11 @@ describe "MongoAWS", ->
"aws-sdk": @awssdk = {}
"fs": @fs = {}
"s3-streams": @s3streams = {}
"./mongojs" : { db: @db = {}, ObjectId: ObjectId }
"./mongojs" : { db: @db = { bson: { BSON:{} } }, ObjectId: ObjectId }
"JSONStream": @JSONStream = {}
"readline-stream": @readline = sinon.stub()
@bulkLimit = @MongoAWS.bulkLimit
@db.bson.BSON.calculateObjectSize = sinon.stub().returns true
@project_id = ObjectId().toString()
@doc_id = ObjectId().toString()
@callback = sinon.stub()