github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #36 from sharelatex/bg-improve-archiving

Browse source 
make archiving less noisy
This commit is contained in:
Brian Gough 2017-05-10 14:18:18 +01:00 committed by GitHub
commit 756563fe42
2 changed files with 15 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/track-changes/app.coffee
View file

@ -61,7 +61,7 @@ app.post "/pack", (req, res, next) ->
else else
logger.log "running pack" logger.log "running pack"
packWorker = child_process.fork(__dirname + '/app/js/PackWorker.js', packWorker = child_process.fork(__dirname + '/app/js/PackWorker.js',
[req.query.limit, req.query.delay, req.query.timeout]) [req.query.limit || 1000, req.query.delay || 1000, req.query.timeout || 30*60*1000])
packWorker.on 'exit', (code, signal) -> packWorker.on 'exit', (code, signal) ->
logger.log {code, signal}, "history auto pack exited" logger.log {code, signal}, "history auto pack exited"
packWorker = null packWorker = null

21
services/track-changes/app/coffee/PackWorker.coffee
View file

@ -21,8 +21,10 @@ PackManager = require "./PackManager"
source = process.argv[2] source = process.argv[2]
DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000 DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000
TIMEOUT = Number(process.argv[4]) || 30*60*1000 TIMEOUT = Number(process.argv[4]) || 30*60*1000
COUNT = 0 # number processed
TOTAL = 0 # total number to process
if source.match(/[^0-9]/) if !source.match(/^[0-9]+$/)
file = fs.readFileSync source file = fs.readFileSync source
result = for line in file.toString().split('\n') result = for line in file.toString().split('\n')
[project_id, doc_id] = line.split(' ') [project_id, doc_id] = line.split(' ')
@ -37,10 +39,11 @@ shutDownTimer = setTimeout () ->
# start the shutdown on the next pack # start the shutdown on the next pack
shutDownRequested = true shutDownRequested = true
# do a hard shutdown after a further 5 minutes # do a hard shutdown after a further 5 minutes
setTimeout () -> hardTimeout = setTimeout () ->
logger.error "HARD TIMEOUT in pack archive worker" logger.error "HARD TIMEOUT in pack archive worker"
process.exit() process.exit()
, 5*60*1000 , 5*60*1000
hardTimeout.unref()
, TIMEOUT , TIMEOUT
logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}" logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}"
@ -61,7 +64,7 @@ finish = () ->
db.close () -> db.close () ->
logger.log 'closing LockManager Redis Connection' logger.log 'closing LockManager Redis Connection'
LockManager.close () -> LockManager.close () ->
logger.log 'ready to exit from pack archive worker' logger.log {processedCount: COUNT, allCount: TOTAL}, 'ready to exit from pack archive worker'
hardTimeout = setTimeout () -> hardTimeout = setTimeout () ->
logger.error 'hard exit from pack archive worker' logger.error 'hard exit from pack archive worker'
process.exit(1) process.exit(1)
@ -74,7 +77,8 @@ process.on 'exit', (code) ->
processUpdates = (pending) -> processUpdates = (pending) ->
async.eachSeries pending, (result, callback) -> async.eachSeries pending, (result, callback) ->
{_id, project_id, doc_id} = result {_id, project_id, doc_id} = result
logger.log {project_id, doc_id}, "processing" COUNT++
logger.log {project_id, doc_id}, "processing #{COUNT}/#{TOTAL}"
if not project_id? or not doc_id? if not project_id? or not doc_id?
logger.log {project_id, doc_id}, "skipping pack, missing project/doc id" logger.log {project_id, doc_id}, "skipping pack, missing project/doc id"
return callback() return callback()
@ -87,7 +91,7 @@ processUpdates = (pending) ->
logger.error {err, result}, "error in pack archive worker" logger.error {err, result}, "error in pack archive worker"
return callback(err) return callback(err)
if shutDownRequested if shutDownRequested
logger.error "shutting down pack archive worker" logger.warn "shutting down pack archive worker"
return callback(new Error("shutdown")) return callback(new Error("shutdown"))
setTimeout () -> setTimeout () ->
callback(err, result) callback(err, result)
@ -115,11 +119,13 @@ if pending?
logger.log "got #{pending.length} entries from #{source}" logger.log "got #{pending.length} entries from #{source}"
processUpdates pending processUpdates pending
else else
oneWeekAgo = new Date(Date.now() - 7 * DAYS)
db.docHistory.find({ db.docHistory.find({
expiresAt: {$exists: false} expiresAt: {$exists: false}
project_id: {$exists: true} project_id: {$exists: true}
v_end: {$exists: true} v_end: {$exists: true}
_id: {$lt: ObjectIdFromDate(new Date(Date.now() - 7 * DAYS))} _id: {$lt: ObjectIdFromDate(oneWeekAgo)}
last_checked: {$lt: oneWeekAgo}
}, {_id:1, doc_id:1, project_id:1}).sort({ }, {_id:1, doc_id:1, project_id:1}).sort({
last_checked:1 last_checked:1
}).limit LIMIT, (err, results) -> }).limit LIMIT, (err, results) ->
@ -128,5 +134,6 @@ else
finish() finish()
return return
pending = _.uniq results, false, (result) -> result.doc_id.toString() pending = _.uniq results, false, (result) -> result.doc_id.toString()
logger.log "found #{pending.length} documents to archive" TOTAL = pending.length
logger.log "found #{TOTAL} documents to archive"
processUpdates pending processUpdates pending