diff --git a/libraries/metrics/memory.coffee b/libraries/metrics/memory.coffee new file mode 100644 index 0000000000..5b3c8f55ae --- /dev/null +++ b/libraries/metrics/memory.coffee @@ -0,0 +1,86 @@ +# record memory usage each minute and run a periodic gc(), keeping cpu +# usage within allowable range of 1ms per minute. Also, dynamically +# adjust the period between gc()'s to reach a target of the gc saving +# 4 megabytes each time. + +oneMinute = 60 * 1000 +oneMegaByte = 1024 * 1024 + +CpuTimeBucket = 100 # current cpu time allowance in milliseconds +CpuTimeBucketMax = 100 # maximum amount of cpu time allowed in bucket +CpuTimeBucketRate = 1 # add this many milliseconds per minute + +gcInterval = 1 # how many minutes between gc (parameter is dynamically adjusted) +countSinceLastGc = 0 # how many minutes since last gc +MemoryChunkSize = 4 * oneMegaByte # how much we need to free to consider gc worth doing + +readyToGc = () -> + # update allowed cpu time + CpuTimeBucket = CpuTimeBucket + CpuTimeBucketRate + CpuTimeBucket = if CpuTimeBucket < CpuTimeBucketMax then CpuTimeBucket else CpuTimeBucketMax + # update counts since last gc + countSinceLastGc = countSinceLastGc + 1 + # check there is enough time since last gc and we have enough cpu + return (countSinceLastGc > gcInterval) && (CpuTimeBucket > 0) + +executeAndTime = (fn) -> + # time the execution of fn() and subtract from cpu allowance + t0 = process.hrtime() + fn() + dt = process.hrtime(t0) + timeTaken = (dt[0] + dt[1]*1e-9) * 1e3 # in milliseconds + CpuTimeBucket -= Math.ceil timeTaken + return timeTaken + +inMegaBytes = (obj) -> + # convert process.memoryUsage hash {rss,heapTotal,heapFreed} into megabytes + result = {} + for k, v of obj + result[k] = (v / oneMegaByte).toFixed(2) + return result + +updateMemoryStats = (oldMem, newMem) -> + countSinceLastGc = 0 + delta = {} + for k of newMem + delta[k] = (newMem[k] - oldMem[k]).toFixed(2) + # take the max of all memory measures + savedMemory = Math.max -delta.rss, -delta.heapTotal, -delta.heapUsed + delta.megabytesFreed = savedMemory + # did it do any good? + if savedMemory < MemoryChunkSize + gcInterval = gcInterval + 1 # no, so wait longer next time + else + gcInterval = Math.max gcInterval - 1, 1 # yes, wait less time + return delta + +module.exports = MemoryMonitor = + monitor: (logger) -> + # use setTimeout add some randomisation of the start time + setTimeout () -> + interval = setInterval () -> + MemoryMonitor.Check(logger) + , oneMinute + , Math.floor Math.random()*oneMinute + Metrics = require "./metrics" + Metrics.registerDestructor () -> + clearInterval(interval) + + Check: (logger) -> + Metrics = require "./metrics" + memBeforeGc = mem = inMegaBytes process.memoryUsage() + Metrics.gauge("memory.rss", mem.rss) + Metrics.gauge("memory.heaptotal", mem.heapTotal) + Metrics.gauge("memory.heapused", mem.heapUsed) + Metrics.gauge("memory.gc-interval", gcInterval) + Metrics.gauge("memory.cpu-time-bucket", CpuTimeBucket) + + if global.gc? && readyToGc() + gcTime = (executeAndTime global.gc).toFixed(2) + memAfterGc = inMegaBytes process.memoryUsage() + deltaMem = updateMemoryStats(memBeforeGc, memAfterGc) + logger.log {gcTime, memBeforeGc, memAfterGc, deltaMem, gcInterval, CpuTimeBucket}, "global.gc() forced" + Metrics.timing("memory.gc-time", gcTime) + Metrics.gauge("memory.gc-rss-freed", -deltaMem.rss) + Metrics.gauge("memory.gc-heaptotal-freed", -deltaMem.heapTotal) + Metrics.gauge("memory.gc-heapused-freed", -deltaMem.heapUsed) diff --git a/libraries/metrics/metrics.coffee b/libraries/metrics/metrics.coffee index a0cd3070fb..e382c8a8cd 100644 --- a/libraries/metrics/metrics.coffee +++ b/libraries/metrics/metrics.coffee @@ -40,6 +40,7 @@ module.exports = http: require "./http" open_sockets: require "./open_sockets" event_loop: require "./event_loop" + memory: require "./memory" close: () -> for func in destructors