github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

improve logging and interrupt handling

Browse source
This commit is contained in:
Brian Gough 2016-02-04 16:31:38 +00:00
parent eb88347e1d
commit 1c24a889fb
1 changed files with 30 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
migrations/3_pack_docHistory_collection.coffee
View file

@ -3,7 +3,7 @@ fs = require("fs")
mongojs = require("mongojs") mongojs = require("mongojs")
ObjectId = mongojs.ObjectId ObjectId = mongojs.ObjectId
db = mongojs(Settings.mongo.url, ['docs','docHistory', 'docHistoryStats']) db = mongojs(Settings.mongo.url, ['docs','docHistory', 'docHistoryStats'])
_ = require("lodash") _ = require("underscore")
async = require("async") async = require("async")
exec = require("child_process").exec exec = require("child_process").exec
BSON = db.bson.BSON BSON = db.bson.BSON
@ -13,6 +13,14 @@ logger = {
err: -> err: ->
} }
needToExit = false
handleExit = () ->
needToExit = true
console.log('Got signal. Shutting down.')
process.on 'SIGINT', handleExit
process.on 'SIGHUP', handleExit
finished_docs_path = "/tmp/finished-docs-3" finished_docs_path = "/tmp/finished-docs-3"
all_docs_path = "/tmp/all-docs-3" all_docs_path = "/tmp/all-docs-3"
unmigrated_docs_path = "/tmp/unmigrated-docs-3" unmigrated_docs_path = "/tmp/unmigrated-docs-3"
@ -56,23 +64,24 @@ markDocAsUnmigrated = (doc_id, callback)->
markDocAsProcessed doc_id, (err)-> markDocAsProcessed doc_id, (err)->
fs.appendFile unmigrated_docs_path, "#{doc_id}\n", callback fs.appendFile unmigrated_docs_path, "#{doc_id}\n", callback
printProgress = ->
count = Object.keys(finished_docs).length
console.log "completed", count
checkIfDocHasBeenProccessed = (doc_id, callback)-> checkIfDocHasBeenProccessed = (doc_id, callback)->
callback(null, finished_docs[doc_id]) callback(null, finished_docs[doc_id])
processNext = (doc_id, callback)-> processNext = (doc_id, callback)->
if !doc_id? or doc_id.length == 0 if !doc_id? or doc_id.length == 0
return callback() return callback()
if needToExit
return callback(new Error("graceful shutdown"))
checkIfDocHasBeenProccessed doc_id, (err, hasBeenProcessed)-> checkIfDocHasBeenProccessed doc_id, (err, hasBeenProcessed)->
if hasBeenProcessed if hasBeenProcessed
console.log "#{doc_id} already processed, skipping" console.log "#{doc_id} already processed, skipping"
return callback() return callback()
console.log "#{doc_id} processing"
PackManager._packDocHistory doc_id, {}, (err) -> PackManager._packDocHistory doc_id, {}, (err) ->
markDocAsProcessed doc_id, callback if err?
console.log "error processing #{doc_id}"
markDocAsUnmigrated doc_id, callback
else
markDocAsProcessed doc_id, callback
updateIndexes = (callback) -> updateIndexes = (callback) ->
async.series [ async.series [
@ -91,15 +100,27 @@ updateIndexes = (callback) ->
exports.migrate = (client, done = ->)-> exports.migrate = (client, done = ->)->
getDocIds (err, ids)-> getDocIds (err, ids)->
totalDocCount = ids.length
alreadyFinishedCount = Object.keys(finished_docs).length
t0 = Date.now()
printProgress = () ->
count = Object.keys(finished_docs).length
processedFraction = (count-alreadyFinishedCount)/totalDocCount
remainingFraction = (totalDocCount-count)/totalDocCount
t = Date.now()
dt = (t-t0)*remainingFraction/processedFraction
estFinishTime = new Date(t + dt)
console.log "completed #{count}/#{totalDocCount} processed=#{processedFraction.toFixed(2)} remaining=#{remainingFraction.toFixed(2)} elapsed=#{(t-t0)/1000} est Finish=#{estFinishTime}"
interval = setInterval printProgress, 3*1000 interval = setInterval printProgress, 3*1000
jobs = _.map ids, (id)->
jobs = _.map _.filter(ids, (id) -> not finished_docs[id]), (id)->
return (cb)-> return (cb)->
processNext(id, cb) processNext(id, cb)
async.series jobs, (err)-> async.series jobs, (err)->
if err? if err?
console.error err, "at end of jobs" console.error err, "at end of jobs"
else else
console.log "finished" console.log "finished at #{new Date}"
clearInterval interval clearInterval interval
done(err) done(err)