From 9936710a1a2aa726f4a545bae1b7aede7ce43dad Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 May 2017 13:58:40 +0100 Subject: [PATCH 1/5] replace error with warning for archiving timeout --- services/track-changes/app/coffee/PackWorker.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/PackWorker.coffee b/services/track-changes/app/coffee/PackWorker.coffee index be5c511d1f..40520be73a 100644 --- a/services/track-changes/app/coffee/PackWorker.coffee +++ b/services/track-changes/app/coffee/PackWorker.coffee @@ -87,7 +87,7 @@ processUpdates = (pending) -> logger.error {err, result}, "error in pack archive worker" return callback(err) if shutDownRequested - logger.error "shutting down pack archive worker" + logger.warn "shutting down pack archive worker" return callback(new Error("shutdown")) setTimeout () -> callback(err, result) From c6e83c6cb8a417097e2b84fa226d293e953cf254 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 May 2017 13:59:09 +0100 Subject: [PATCH 2/5] allow archiving to exit properly on hard timeout --- services/track-changes/app/coffee/PackWorker.coffee | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/PackWorker.coffee b/services/track-changes/app/coffee/PackWorker.coffee index 40520be73a..e5927d5b45 100644 --- a/services/track-changes/app/coffee/PackWorker.coffee +++ b/services/track-changes/app/coffee/PackWorker.coffee @@ -37,10 +37,11 @@ shutDownTimer = setTimeout () -> # start the shutdown on the next pack shutDownRequested = true # do a hard shutdown after a further 5 minutes - setTimeout () -> + hardTimeout = setTimeout () -> logger.error "HARD TIMEOUT in pack archive worker" process.exit() , 5*60*1000 + hardTimeout.unref() , TIMEOUT logger.log "checking for updates, limit=#{LIMIT}, delay=#{DOCUMENT_PACK_DELAY}, timeout=#{TIMEOUT}" From cac7556ad54ddae75522126d379079a1c46635a0 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 May 2017 11:32:24 +0100 Subject: [PATCH 3/5] only check packs for archiving once each week --- services/track-changes/app/coffee/PackWorker.coffee | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/track-changes/app/coffee/PackWorker.coffee b/services/track-changes/app/coffee/PackWorker.coffee index e5927d5b45..563973d13a 100644 --- a/services/track-changes/app/coffee/PackWorker.coffee +++ b/services/track-changes/app/coffee/PackWorker.coffee @@ -116,11 +116,13 @@ if pending? logger.log "got #{pending.length} entries from #{source}" processUpdates pending else + oneWeekAgo = new Date(Date.now() - 7 * DAYS) db.docHistory.find({ expiresAt: {$exists: false} project_id: {$exists: true} v_end: {$exists: true} - _id: {$lt: ObjectIdFromDate(new Date(Date.now() - 7 * DAYS))} + _id: {$lt: ObjectIdFromDate(oneWeekAgo)} + last_checked: {$lt: oneWeekAgo} }, {_id:1, doc_id:1, project_id:1}).sort({ last_checked:1 }).limit LIMIT, (err, results) -> From c26bccd39006a7b3e1c454edc1f0a9f658b5dc56 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 May 2017 11:33:21 +0100 Subject: [PATCH 4/5] make arguments handling for packworker more robust --- services/track-changes/app.coffee | 2 +- services/track-changes/app/coffee/PackWorker.coffee | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/track-changes/app.coffee b/services/track-changes/app.coffee index 43cddca498..a83d78c7f1 100644 --- a/services/track-changes/app.coffee +++ b/services/track-changes/app.coffee @@ -61,7 +61,7 @@ app.post "/pack", (req, res, next) -> else logger.log "running pack" packWorker = child_process.fork(__dirname + '/app/js/PackWorker.js', - [req.query.limit, req.query.delay, req.query.timeout]) + [req.query.limit || 1000, req.query.delay || 1000, req.query.timeout || 30*60*1000]) packWorker.on 'exit', (code, signal) -> logger.log {code, signal}, "history auto pack exited" packWorker = null diff --git a/services/track-changes/app/coffee/PackWorker.coffee b/services/track-changes/app/coffee/PackWorker.coffee index 563973d13a..12c4a37aff 100644 --- a/services/track-changes/app/coffee/PackWorker.coffee +++ b/services/track-changes/app/coffee/PackWorker.coffee @@ -22,7 +22,7 @@ source = process.argv[2] DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000 TIMEOUT = Number(process.argv[4]) || 30*60*1000 -if source.match(/[^0-9]/) +if !source.match(/^[0-9]+$/) file = fs.readFileSync source result = for line in file.toString().split('\n') [project_id, doc_id] = line.split(' ') From 2781d9fd80fa8efd9ebc982951f92433969e54ee Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 May 2017 13:43:21 +0100 Subject: [PATCH 5/5] keep track of processed/total docs in archiving --- services/track-changes/app/coffee/PackWorker.coffee | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/track-changes/app/coffee/PackWorker.coffee b/services/track-changes/app/coffee/PackWorker.coffee index 12c4a37aff..e4c31c3b4a 100644 --- a/services/track-changes/app/coffee/PackWorker.coffee +++ b/services/track-changes/app/coffee/PackWorker.coffee @@ -21,6 +21,8 @@ PackManager = require "./PackManager" source = process.argv[2] DOCUMENT_PACK_DELAY = Number(process.argv[3]) || 1000 TIMEOUT = Number(process.argv[4]) || 30*60*1000 +COUNT = 0 # number processed +TOTAL = 0 # total number to process if !source.match(/^[0-9]+$/) file = fs.readFileSync source @@ -62,7 +64,7 @@ finish = () -> db.close () -> logger.log 'closing LockManager Redis Connection' LockManager.close () -> - logger.log 'ready to exit from pack archive worker' + logger.log {processedCount: COUNT, allCount: TOTAL}, 'ready to exit from pack archive worker' hardTimeout = setTimeout () -> logger.error 'hard exit from pack archive worker' process.exit(1) @@ -75,7 +77,8 @@ process.on 'exit', (code) -> processUpdates = (pending) -> async.eachSeries pending, (result, callback) -> {_id, project_id, doc_id} = result - logger.log {project_id, doc_id}, "processing" + COUNT++ + logger.log {project_id, doc_id}, "processing #{COUNT}/#{TOTAL}" if not project_id? or not doc_id? logger.log {project_id, doc_id}, "skipping pack, missing project/doc id" return callback() @@ -131,5 +134,6 @@ else finish() return pending = _.uniq results, false, (result) -> result.doc_id.toString() - logger.log "found #{pending.length} documents to archive" + TOTAL = pending.length + logger.log "found #{TOTAL} documents to archive" processUpdates pending