mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Brian Gough
GitHub
commit
919d101aa9
11 changed files with 6 additions and 424 deletions
|
|
@ -1,7 +1,4 @@
|
|||
if process.env["USE_PROM_METRICS"] != "true"
|
||||
return module.exports = require("./statsd/metrics")
|
||||
else
|
||||
console.log("using prometheus")
|
||||
console.log("using prometheus")
|
||||
|
||||
prom = require('./prom_wrapper')
|
||||
|
||||
|
|
@ -29,7 +26,7 @@ module.exports = Metrics =
|
|||
traceAgent = require('@google-cloud/trace-agent')
|
||||
|
||||
traceOpts =
|
||||
ignoreUrls: [/^\/status/, /^\/health_check/]
|
||||
ignoreUrls: [/^\/status/, /^\/health_check/]
|
||||
traceAgent.start(traceOpts)
|
||||
|
||||
console.log("ENABLE_DEBUG_AGENT set to #{process.env['ENABLE_DEBUG_AGENT']}")
|
||||
|
|
@ -61,7 +58,7 @@ module.exports = Metrics =
|
|||
destructors.push func
|
||||
|
||||
injectMetricsRoute: (app) ->
|
||||
app.get('/metrics', (req, res) ->
|
||||
app.get('/metrics', (req, res) ->
|
||||
res.set('Content-Type', prom.registry.contentType)
|
||||
res.end(prom.registry.metrics())
|
||||
)
|
||||
|
|
@ -125,7 +122,7 @@ module.exports = Metrics =
|
|||
prom.metric('gauge', key).set({app: appname, host: hostname, status: opts?.status}, this.sanitizeValue(value))
|
||||
if process.env['DEBUG_METRICS']
|
||||
console.log("doing gauge", key, opts)
|
||||
|
||||
|
||||
globalGauge: (key, value, sampleRate = 1, opts)->
|
||||
key = Metrics.buildPromKey(key)
|
||||
prom.metric('gauge', key).set({app: appname, status: opts?.status},this.sanitizeValue(value))
|
||||
|
|
|
|||
21
libraries/metrics/package-lock.json
generated
21
libraries/metrics/package-lock.json
generated
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "metrics-sharelatex",
|
||||
"version": "2.6.2",
|
||||
"version": "3.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
|
|
@ -1317,20 +1317,6 @@
|
|||
"yallist": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"lynx": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz",
|
||||
"integrity": "sha1-Mxjc7xaQi4KG6Bisz9sxzXQkj50=",
|
||||
"requires": {
|
||||
"mersenne": "~0.0.3",
|
||||
"statsd-parser": "~0.0.4"
|
||||
}
|
||||
},
|
||||
"mersenne": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz",
|
||||
"integrity": "sha1-QB/ex+whzbngPNPTAhOY2iGycIU="
|
||||
},
|
||||
"methods": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
|
||||
|
|
@ -1785,11 +1771,6 @@
|
|||
"resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.9.tgz",
|
||||
"integrity": "sha1-qPbq7KkGdMMz58Q5U/J1tFFRBpU="
|
||||
},
|
||||
"statsd-parser": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz",
|
||||
"integrity": "sha1-y9JDlTzELv/VSLXSI4jtaJ7GOb0="
|
||||
},
|
||||
"stream-shift": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "metrics-sharelatex",
|
||||
"version": "2.7.0",
|
||||
"version": "3.0.0",
|
||||
"description": "A drop-in metrics and monitoring module for node.js apps",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
|
@ -11,7 +11,6 @@
|
|||
"@google-cloud/profiler": "^0.2.3",
|
||||
"@google-cloud/trace-agent": "^3.2.0",
|
||||
"coffee-script": "1.6.0",
|
||||
"lynx": "~0.1.1",
|
||||
"prom-client": "^11.1.3",
|
||||
"underscore": "~1.6.0",
|
||||
"yn": "^3.1.1"
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
module.exports = EventLoopMonitor =
|
||||
monitor: (logger, interval = 1000, log_threshold = 100) ->
|
||||
Metrics = require "./metrics"
|
||||
# check for logger on startup to avoid exceptions later if undefined
|
||||
throw new Error("logger is undefined") if !logger?
|
||||
# monitor delay in setInterval to detect event loop blocking
|
||||
previous = Date.now()
|
||||
intervalId = setInterval () ->
|
||||
now = Date.now()
|
||||
offset = now - previous - interval
|
||||
if offset > log_threshold
|
||||
logger.warn {offset: offset}, "slow event loop"
|
||||
previous = now
|
||||
Metrics.timing("event-loop-millsec", offset)
|
||||
, interval
|
||||
|
||||
Metrics.registerDestructor () ->
|
||||
clearInterval(intervalId)
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
os = require("os")
|
||||
yn = require("yn")
|
||||
|
||||
STACKDRIVER_LOGGING = yn(process.env['STACKDRIVER_LOGGING'])
|
||||
|
||||
module.exports.monitor = (logger) ->
|
||||
return (req, res, next) ->
|
||||
Metrics = require("./metrics")
|
||||
startTime = process.hrtime()
|
||||
end = res.end
|
||||
res.end = () ->
|
||||
end.apply(this, arguments)
|
||||
responseTime = process.hrtime(startTime)
|
||||
responseTimeMs = Math.round(responseTime[0] * 1000 + responseTime[1] / 1000)
|
||||
if req.route?.path?
|
||||
routePath = req.route.path.toString().replace(/\//g, '_').replace(/\:/g, '').slice(1)
|
||||
key = "http-requests.#{routePath}.#{req.method}.#{res.statusCode}"
|
||||
Metrics.timing(key, responseTimeMs)
|
||||
remoteIp = req.ip || req.socket?.socket?.remoteAddress || req.socket?.remoteAddress
|
||||
reqUrl = req.originalUrl || req.url
|
||||
referrer = req.headers['referer'] || req.headers['referrer']
|
||||
if STACKDRIVER_LOGGING
|
||||
info =
|
||||
httpRequest:
|
||||
requestMethod: req.method
|
||||
requestUrl: reqUrl
|
||||
requestSize: req.headers["content-length"]
|
||||
status: res.statusCode
|
||||
responseSize: res._headers?["content-length"]
|
||||
userAgent: req.headers["user-agent"]
|
||||
remoteIp: remoteIp
|
||||
referer: referrer
|
||||
latency:
|
||||
seconds: responseTime[0]
|
||||
nanos: responseTime[1]
|
||||
protocol: req.protocol
|
||||
else
|
||||
info =
|
||||
req:
|
||||
url: reqUrl
|
||||
method: req.method
|
||||
referrer: referrer
|
||||
"remote-addr": remoteIp
|
||||
"user-agent": req.headers["user-agent"]
|
||||
"content-length": req.headers["content-length"]
|
||||
res:
|
||||
"content-length": res._headers?["content-length"]
|
||||
statusCode: res.statusCode
|
||||
"response-time": responseTimeMs
|
||||
if res.statusCode >= 500
|
||||
logger.error(info, "%s %s", req.method, reqUrl)
|
||||
else if res.statusCode >= 400 and res.statusCode < 500
|
||||
logger.warn(info, "%s %s", req.method, reqUrl)
|
||||
else
|
||||
logger.info(info, "%s %s", req.method, reqUrl)
|
||||
next()
|
||||
|
|
@ -1,85 +0,0 @@
|
|||
# record memory usage each minute and run a periodic gc(), keeping cpu
|
||||
# usage within allowable range of 1ms per minute. Also, dynamically
|
||||
# adjust the period between gc()'s to reach a target of the gc saving
|
||||
# 4 megabytes each time.
|
||||
|
||||
oneMinute = 60 * 1000
|
||||
oneMegaByte = 1024 * 1024
|
||||
|
||||
CpuTimeBucket = 100 # current cpu time allowance in milliseconds
|
||||
CpuTimeBucketMax = 100 # maximum amount of cpu time allowed in bucket
|
||||
CpuTimeBucketRate = 10 # add this many milliseconds per minute
|
||||
|
||||
gcInterval = 1 # how many minutes between gc (parameter is dynamically adjusted)
|
||||
countSinceLastGc = 0 # how many minutes since last gc
|
||||
MemoryChunkSize = 4 # how many megabytes we need to free to consider gc worth doing
|
||||
|
||||
readyToGc = () ->
|
||||
# update allowed cpu time
|
||||
CpuTimeBucket = CpuTimeBucket + CpuTimeBucketRate
|
||||
CpuTimeBucket = if CpuTimeBucket < CpuTimeBucketMax then CpuTimeBucket else CpuTimeBucketMax
|
||||
# update counts since last gc
|
||||
countSinceLastGc = countSinceLastGc + 1
|
||||
# check there is enough time since last gc and we have enough cpu
|
||||
return (countSinceLastGc > gcInterval) && (CpuTimeBucket > 0)
|
||||
|
||||
executeAndTime = (fn) ->
|
||||
# time the execution of fn() and subtract from cpu allowance
|
||||
t0 = process.hrtime()
|
||||
fn()
|
||||
dt = process.hrtime(t0)
|
||||
timeTaken = (dt[0] + dt[1]*1e-9) * 1e3 # in milliseconds
|
||||
CpuTimeBucket -= Math.ceil timeTaken
|
||||
return timeTaken
|
||||
|
||||
inMegaBytes = (obj) ->
|
||||
# convert process.memoryUsage hash {rss,heapTotal,heapFreed} into megabytes
|
||||
result = {}
|
||||
for k, v of obj
|
||||
result[k] = (v / oneMegaByte).toFixed(2)
|
||||
return result
|
||||
|
||||
updateMemoryStats = (oldMem, newMem) ->
|
||||
countSinceLastGc = 0
|
||||
delta = {}
|
||||
for k of newMem
|
||||
delta[k] = (newMem[k] - oldMem[k]).toFixed(2)
|
||||
# take the max of all memory measures
|
||||
savedMemory = Math.max -delta.rss, -delta.heapTotal, -delta.heapUsed
|
||||
delta.megabytesFreed = savedMemory
|
||||
# did it do any good?
|
||||
if savedMemory < MemoryChunkSize
|
||||
gcInterval = gcInterval + 1 # no, so wait longer next time
|
||||
else
|
||||
gcInterval = Math.max gcInterval - 1, 1 # yes, wait less time
|
||||
return delta
|
||||
|
||||
module.exports = MemoryMonitor =
|
||||
monitor: (logger) ->
|
||||
interval = setInterval () ->
|
||||
MemoryMonitor.Check(logger)
|
||||
, oneMinute
|
||||
Metrics = require "./metrics"
|
||||
Metrics.registerDestructor () ->
|
||||
clearInterval(interval)
|
||||
|
||||
Check: (logger) ->
|
||||
Metrics = require "./metrics"
|
||||
memBeforeGc = mem = inMegaBytes process.memoryUsage()
|
||||
Metrics.gauge("memory.rss", mem.rss)
|
||||
Metrics.gauge("memory.heaptotal", mem.heapTotal)
|
||||
Metrics.gauge("memory.heapused", mem.heapUsed)
|
||||
Metrics.gauge("memory.gc-interval", gcInterval)
|
||||
#Metrics.gauge("memory.cpu-time-bucket", CpuTimeBucket)
|
||||
|
||||
logger.log mem, "process.memoryUsage()"
|
||||
|
||||
if global.gc? && readyToGc()
|
||||
gcTime = (executeAndTime global.gc).toFixed(2)
|
||||
memAfterGc = inMegaBytes process.memoryUsage()
|
||||
deltaMem = updateMemoryStats(memBeforeGc, memAfterGc)
|
||||
logger.log {gcTime, memBeforeGc, memAfterGc, deltaMem, gcInterval, CpuTimeBucket}, "global.gc() forced"
|
||||
#Metrics.timing("memory.gc-time", gcTime)
|
||||
Metrics.gauge("memory.gc-rss-freed", -deltaMem.rss)
|
||||
Metrics.gauge("memory.gc-heaptotal-freed", -deltaMem.heapTotal)
|
||||
Metrics.gauge("memory.gc-heapused-freed", -deltaMem.heapUsed)
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
console.log("using statsd")
|
||||
|
||||
StatsD = require('lynx')
|
||||
statsd = new StatsD(process.env["STATSD_HOST"] or "localhost", 8125, {on_error:->})
|
||||
|
||||
name = "unknown"
|
||||
hostname = require('os').hostname()
|
||||
|
||||
buildKey = (key)-> "#{name}.#{hostname}.#{key}"
|
||||
buildGlobalKey = (key)-> "#{name}.global.#{key}"
|
||||
|
||||
destructors = []
|
||||
|
||||
require "./uv_threadpool_size"
|
||||
|
||||
module.exports = Metrics =
|
||||
initialize: (_name) ->
|
||||
name = _name
|
||||
|
||||
registerDestructor: (func) ->
|
||||
destructors.push func
|
||||
|
||||
set : (key, value, sampleRate = 1)->
|
||||
statsd.set buildKey(key), value, sampleRate
|
||||
|
||||
inc : (key, sampleRate = 1)->
|
||||
statsd.increment buildKey(key), sampleRate
|
||||
|
||||
count : (key, count, sampleRate = 1)->
|
||||
statsd.count buildKey(key), count, sampleRate
|
||||
|
||||
summary : (key, value)->
|
||||
# not supported
|
||||
|
||||
timing: (key, timeSpan, sampleRate)->
|
||||
statsd.timing(buildKey(key), timeSpan, sampleRate)
|
||||
|
||||
Timer : class
|
||||
constructor :(key, sampleRate = 1)->
|
||||
this.start = new Date()
|
||||
this.key = key
|
||||
this.sampleRate = sampleRate
|
||||
done:->
|
||||
timeSpan = new Date - this.start
|
||||
statsd.timing(buildKey(this.key), timeSpan, this.sampleRate)
|
||||
return timeSpan
|
||||
|
||||
gauge : (key, value, sampleRate = 1)->
|
||||
statsd.gauge buildKey(key), value, sampleRate
|
||||
|
||||
globalGauge: (key, value, sampleRate = 1)->
|
||||
statsd.gauge buildGlobalKey(key), value, sampleRate
|
||||
|
||||
mongodb: require "./mongodb"
|
||||
http: require "./http"
|
||||
open_sockets: require "./open_sockets"
|
||||
event_loop: require "./event_loop"
|
||||
memory: require "./memory"
|
||||
|
||||
timeAsyncMethod: require('./timeAsyncMethod')
|
||||
|
||||
injectMetricsRoute: (app) ->
|
||||
app.get('/metrics', (req, res) ->
|
||||
res.send("not implemented in statsd")
|
||||
)
|
||||
|
||||
close: () ->
|
||||
for func in destructors
|
||||
func()
|
||||
statsd.close()
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
module.exports =
|
||||
monitor: (mongodb_require_path, logger) ->
|
||||
|
||||
try
|
||||
# for the v1 driver the methods to wrap are in the mongodb
|
||||
# module in lib/mongodb/db.js
|
||||
mongodb = require("#{mongodb_require_path}")
|
||||
|
||||
try
|
||||
# for the v2 driver the relevant methods are in the mongodb-core
|
||||
# module in lib/topologies/{server,replset,mongos}.js
|
||||
v2_path = mongodb_require_path.replace(/\/mongodb$/, '/mongodb-core')
|
||||
mongodbCore = require(v2_path)
|
||||
|
||||
Metrics = require("./metrics")
|
||||
|
||||
monitorMethod = (base, method, type) ->
|
||||
return unless base?
|
||||
return unless (_method = base[method])?
|
||||
arglen = _method.length
|
||||
|
||||
mongo_driver_v1_wrapper = (db_command, options, callback) ->
|
||||
if (typeof callback == 'undefined')
|
||||
callback = options
|
||||
options = {}
|
||||
|
||||
collection = db_command.collectionName
|
||||
if collection.match(/\$cmd$/)
|
||||
# Ignore noisy command methods like authenticating, ismaster and ping
|
||||
return _method.call this, db_command, options, callback
|
||||
|
||||
key = "mongo-requests.#{collection}.#{type}"
|
||||
if db_command.query?
|
||||
query = Object.keys(db_command.query).sort().join("_")
|
||||
key += "." + query
|
||||
|
||||
timer = new Metrics.Timer(key)
|
||||
start = new Date()
|
||||
_method.call this, db_command, options, () ->
|
||||
timer.done()
|
||||
time = new Date() - start
|
||||
logger.log
|
||||
query: db_command.query
|
||||
query_type: type
|
||||
collection: collection
|
||||
"response-time": new Date() - start
|
||||
"mongo request"
|
||||
callback.apply this, arguments
|
||||
|
||||
mongo_driver_v2_wrapper = (ns, ops, options, callback) ->
|
||||
if (typeof callback == 'undefined')
|
||||
callback = options
|
||||
options = {}
|
||||
|
||||
if ns.match(/\$cmd$/)
|
||||
# Ignore noisy command methods like authenticating, ismaster and ping
|
||||
return _method.call this, ns, ops, options, callback
|
||||
|
||||
key = "mongo-requests.#{ns}.#{type}"
|
||||
if ops[0].q? # ops[0].q
|
||||
query = Object.keys(ops[0].q).sort().join("_")
|
||||
key += "." + query
|
||||
|
||||
timer = new Metrics.Timer(key)
|
||||
start = new Date()
|
||||
_method.call this, ns, ops, options, () ->
|
||||
timer.done()
|
||||
time = new Date() - start
|
||||
logger.log
|
||||
query: ops[0].q
|
||||
query_type: type
|
||||
collection: ns
|
||||
"response-time": new Date() - start
|
||||
"mongo request"
|
||||
callback.apply this, arguments
|
||||
|
||||
if arglen == 3
|
||||
base[method] = mongo_driver_v1_wrapper
|
||||
else if arglen == 4
|
||||
base[method] = mongo_driver_v2_wrapper
|
||||
|
||||
monitorMethod(mongodb?.Db.prototype, "_executeQueryCommand", "query")
|
||||
monitorMethod(mongodb?.Db.prototype, "_executeRemoveCommand", "remove")
|
||||
monitorMethod(mongodb?.Db.prototype, "_executeInsertCommand", "insert")
|
||||
monitorMethod(mongodb?.Db.prototype, "_executeUpdateCommand", "update")
|
||||
|
||||
monitorMethod(mongodbCore?.Server.prototype, "command", "command")
|
||||
monitorMethod(mongodbCore?.Server.prototype, "remove", "remove")
|
||||
monitorMethod(mongodbCore?.Server.prototype, "insert", "insert")
|
||||
monitorMethod(mongodbCore?.Server.prototype, "update", "update")
|
||||
|
||||
monitorMethod(mongodbCore?.ReplSet.prototype, "command", "command")
|
||||
monitorMethod(mongodbCore?.ReplSet.prototype, "remove", "remove")
|
||||
monitorMethod(mongodbCore?.ReplSet.prototype, "insert", "insert")
|
||||
monitorMethod(mongodbCore?.ReplSet.prototype, "update", "update")
|
||||
|
||||
monitorMethod(mongodbCore?.Mongos.prototype, "command", "command")
|
||||
monitorMethod(mongodbCore?.Mongos.prototype, "remove", "remove")
|
||||
monitorMethod(mongodbCore?.Mongos.prototype, "insert", "insert")
|
||||
monitorMethod(mongodbCore?.Mongos.prototype, "update", "update")
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
URL = require "url"
|
||||
seconds = 1000
|
||||
|
||||
# In Node 0.10 the default is 5, which means only 5 open connections at one.
|
||||
# Node 0.12 has a default of Infinity. Make sure we have no limit set,
|
||||
# regardless of Node version.
|
||||
require("http").globalAgent.maxSockets = Infinity
|
||||
require("https").globalAgent.maxSockets = Infinity
|
||||
|
||||
module.exports = OpenSocketsMonitor =
|
||||
monitor: (logger) ->
|
||||
interval = setInterval () ->
|
||||
OpenSocketsMonitor.gaugeOpenSockets()
|
||||
, 5 * seconds
|
||||
Metrics = require "./metrics"
|
||||
Metrics.registerDestructor () ->
|
||||
clearInterval(interval)
|
||||
|
||||
gaugeOpenSockets: () ->
|
||||
Metrics = require "./metrics"
|
||||
for url, agents of require('http').globalAgent.sockets
|
||||
url = URL.parse("http://#{url}")
|
||||
hostname = url.hostname?.replace(/\./g, "_")
|
||||
Metrics.gauge "open_connections.http.#{hostname}", agents.length
|
||||
for url, agents of require('https').globalAgent.sockets
|
||||
url = URL.parse("https://#{url}")
|
||||
hostname = url.hostname?.replace(/\./g, "_")
|
||||
Metrics.gauge "open_connections.https.#{hostname}", agents.length
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
|
||||
module.exports = (obj, methodName, prefix, logger) ->
|
||||
metrics = require('./metrics')
|
||||
|
||||
if typeof obj[methodName] != 'function'
|
||||
throw new Error("[Metrics] expected object property '#{methodName}' to be a function")
|
||||
|
||||
realMethod = obj[methodName]
|
||||
key = "#{prefix}.#{methodName}"
|
||||
|
||||
obj[methodName] = (originalArgs...) ->
|
||||
|
||||
[firstArgs..., callback] = originalArgs
|
||||
|
||||
if !callback? || typeof callback != 'function'
|
||||
if logger?
|
||||
logger.log "[Metrics] expected wrapped method '#{methodName}' to be invoked with a callback"
|
||||
return realMethod.apply this, originalArgs
|
||||
|
||||
timer = new metrics.Timer(key)
|
||||
|
||||
realMethod.call this, firstArgs..., (callbackArgs...) ->
|
||||
elapsedTime = timer.done()
|
||||
possibleError = callbackArgs[0]
|
||||
if possibleError?
|
||||
metrics.inc "#{key}.failure"
|
||||
else
|
||||
metrics.inc "#{key}.success"
|
||||
if logger?
|
||||
loggableArgs = {}
|
||||
try
|
||||
for arg, idx in firstArgs
|
||||
if arg.toString().match(/^[0-9a-f]{24}$/)
|
||||
loggableArgs["#{idx}"] = arg
|
||||
logger.log {key, args: loggableArgs, elapsedTime}, "[Metrics] timed async method call"
|
||||
callback.apply this, callbackArgs
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
process.env.UV_THREADPOOL_SIZE=16
|
||||
console.log "Set UV_THREADPOOL_SIZE=#{process.env.UV_THREADPOOL_SIZE}"
|
||||
Loading…
Reference in a new issue