merge multiple repositories into an existing monorepo

- merged using: 'monorepo_add.sh libraries-metrics:libraries/metrics'
- see https://github.com/shopsys/monorepo-tools
This commit is contained in:
Jakob Ackermann 2021-08-05 08:34:37 +01:00
commit 02010d9f6e
No known key found for this signature in database
GPG key ID: 30C56800FCA3828A
21 changed files with 7092 additions and 0 deletions

View file

@ -0,0 +1,11 @@
version: 2.1
orbs:
node: circleci/node@3.0.0
workflows:
test:
jobs:
- node/test:
version: "10.22"
override-ci-command: npm install

View file

@ -0,0 +1,55 @@
{
"extends": [
"standard",
"prettier",
"prettier/standard"
],
"parserOptions": {
"ecmaVersion": 2018
},
"plugins": [
"mocha",
"chai-expect",
"chai-friendly"
],
"env": {
"node": true
},
"rules": {
// Swap the no-unused-expressions rule with a more chai-friendly one
"no-unused-expressions": 0,
"chai-friendly/no-unused-expressions": "error"
},
"overrides": [
{
// Test specific rules
"files": ["test/**/*.js"],
"env": {
"mocha": true
},
"globals": {
"expect": true
},
"rules": {
// mocha-specific rules
"mocha/handle-done-callback": "error",
"mocha/no-exclusive-tests": "error",
"mocha/no-global-tests": "error",
"mocha/no-identical-title": "error",
"mocha/no-nested-tests": "error",
"mocha/no-pending-tests": "error",
"mocha/no-skipped-tests": "error",
"mocha/no-mocha-arrows": "error",
// chai-specific rules
"chai-expect/missing-assertion": "error",
"chai-expect/terminating-properties": "error",
// prefer-arrow-callback applies to all callbacks, not just ones in mocha tests.
// we don't enforce this at the top-level - just in tests to manage `this` scope
// based on mocha's context mechanism
"mocha/prefer-arrow-callback": "error"
}
}
]
}

4
libraries/metrics/.gitignore vendored Normal file
View file

@ -0,0 +1,4 @@
node_modules
.npmrc
Dockerfile

View file

@ -0,0 +1,4 @@
/.circleci
/.eslintrc
/.nvmrc
/.prettierrc

View file

@ -0,0 +1,4 @@
{
"semi": false,
"singleQuote": true
}

21
libraries/metrics/LICENSE Normal file
View file

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014 ShareLaTeX
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View file

@ -0,0 +1,30 @@
overleaf/metrics-module
=======================
Wrappers the [prom-client](https://github.com/siimon/prom-client) npm module to provide [Prometheus](https://prometheus.io/) metrics at `/metrics`.
Use:
```
const metrics = require('@overleaf/metrics')
metrics.initialize('myapp')
const express = require('express')
const app = express()
metrics.injectMetricsRoute(app)
```
Request logging can be enabled:
```
const logger = require('logger-sharelatex')
...
app.use(metrics.http.monitor(logger))
```
The metrics module can be configured through the following environment variables:
* `DEBUG_METRICS` - enables display of debugging messages to the console.
* `ENABLE_TRACE_AGENT` - enables @google-cloud/trace-agent on Google Cloud
* `ENABLE_DEBUG_AGENT` - enables @google-cloud/debug-agent on Google Cloud
* `ENABLE_PROFILE_AGENT` - enables @google-cloud/profiler on Google Cloud
* `METRICS_COMPRESSION_LEVEL` - sets the [compression level](https://www.npmjs.com/package/compression#level) for `/metrics`
* `STACKDRIVER_LOGGING` - toggles the request logging format
* `UV_THREADPOOL_SIZE` - sets the libuv [thread pool](http://docs.libuv.org/en/v1.x/threadpool.html) size

View file

@ -0,0 +1,34 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
module.exports = {
monitor(logger, interval, logThreshold) {
if (interval == null) {
interval = 1000
}
if (logThreshold == null) {
logThreshold = 100
}
const Metrics = require('./index')
// check for logger on startup to avoid exceptions later if undefined
if (logger == null) {
throw new Error('logger is undefined')
}
// monitor delay in setInterval to detect event loop blocking
let previous = Date.now()
const intervalId = setInterval(function() {
const now = Date.now()
const offset = now - previous - interval
if (offset > logThreshold) {
logger.warn({ offset }, 'slow event loop')
}
previous = now
return Metrics.timing('event-loop-millsec', offset)
}, interval)
return Metrics.registerDestructor(() => clearInterval(intervalId))
}
}

104
libraries/metrics/http.js Normal file
View file

@ -0,0 +1,104 @@
const yn = require('yn')
const STACKDRIVER_LOGGING = yn(process.env.STACKDRIVER_LOGGING)
module.exports.monitor = logger =>
function(req, res, next) {
const Metrics = require('./index')
const startTime = process.hrtime()
const { end } = res
res.end = function() {
end.apply(this, arguments)
const responseTime = process.hrtime(startTime)
const responseTimeMs = Math.round(
responseTime[0] * 1000 + responseTime[1] / 1000000
)
const requestSize = parseInt(req.headers['content-length'], 10)
const routePath = getRoutePath(req)
const remoteIp = getRemoteIp(req)
const reqUrl = req.originalUrl || req.url
const referrer = req.headers.referer || req.headers.referrer
if (routePath != null) {
Metrics.timing('http_request', responseTimeMs, null, {
method: req.method,
status_code: res.statusCode,
path: routePath
})
if (requestSize) {
Metrics.summary('http_request_size_bytes', requestSize, {
method: req.method,
status_code: res.statusCode,
path: routePath
})
}
}
let info
if (STACKDRIVER_LOGGING) {
info = {
httpRequest: {
requestMethod: req.method,
requestUrl: reqUrl,
requestSize,
status: res.statusCode,
responseSize: res.getHeader('content-length'),
userAgent: req.headers['user-agent'],
remoteIp,
referer: referrer,
latency: {
seconds: responseTime[0],
nanos: responseTime[1]
},
protocol: req.protocol
}
}
} else {
info = {
req: {
url: reqUrl,
method: req.method,
referrer,
'remote-addr': remoteIp,
'user-agent': req.headers['user-agent'],
'content-length': req.headers['content-length']
},
res: {
'content-length': res.getHeader('content-length'),
statusCode: res.statusCode
},
'response-time': responseTimeMs
}
}
logger.info(info, '%s %s', req.method, reqUrl)
}
next()
}
function getRoutePath(req) {
if (req.route && req.route.path != null) {
return req.route.path
.toString()
.replace(/\//g, '_')
.replace(/:/g, '')
.slice(1)
}
if (req.swagger && req.swagger.apiPath != null) {
return req.swagger.apiPath
}
return null
}
function getRemoteIp(req) {
if (req.ip) {
return req.ip
}
if (req.socket) {
if (req.socket.socket && req.socket.socket.remoteAddress) {
return req.socket.socket.remoteAddress
} else if (req.socket.remoteAddress) {
return req.socket.remoteAddress
}
}
return null
}

198
libraries/metrics/index.js Normal file
View file

@ -0,0 +1,198 @@
const os = require('os')
const ExpressCompression = require('compression')
const promClient = require('prom-client')
const promWrapper = require('./prom_wrapper')
const DEFAULT_APP_NAME = 'unknown'
const { collectDefaultMetrics } = promWrapper
const destructors = []
require('./uv_threadpool_size')
/**
* Configure the metrics module
*/
function configure(opts = {}) {
const appName = opts.appName || DEFAULT_APP_NAME
const hostname = os.hostname()
promClient.register.setDefaultLabels({ app: appName, host: hostname })
if (opts.ttlInMinutes) {
promWrapper.ttlInMinutes = opts.ttlInMinutes
}
}
/**
* Configure the metrics module and start the default metrics collectors and
* profiling agents.
*/
function initialize(appName, opts = {}) {
appName = appName || DEFAULT_APP_NAME
configure({ ...opts, appName })
collectDefaultMetrics({ timeout: 5000, prefix: '' })
promWrapper.setupSweeping()
console.log(`ENABLE_TRACE_AGENT set to ${process.env.ENABLE_TRACE_AGENT}`)
if (process.env.ENABLE_TRACE_AGENT === 'true') {
console.log('starting google trace agent')
const traceAgent = require('@google-cloud/trace-agent')
const traceOpts = { ignoreUrls: [/^\/status/, /^\/health_check/] }
traceAgent.start(traceOpts)
}
console.log(`ENABLE_DEBUG_AGENT set to ${process.env.ENABLE_DEBUG_AGENT}`)
if (process.env.ENABLE_DEBUG_AGENT === 'true') {
console.log('starting google debug agent')
const debugAgent = require('@google-cloud/debug-agent')
debugAgent.start({
allowExpressions: true,
serviceContext: {
service: appName,
version: process.env.BUILD_VERSION
}
})
}
console.log(`ENABLE_PROFILE_AGENT set to ${process.env.ENABLE_PROFILE_AGENT}`)
if (process.env.ENABLE_PROFILE_AGENT === 'true') {
console.log('starting google profile agent')
const profiler = require('@google-cloud/profiler')
profiler.start({
serviceContext: {
service: appName,
version: process.env.BUILD_VERSION
}
})
}
inc('process_startup')
}
function registerDestructor(func) {
destructors.push(func)
}
function injectMetricsRoute(app) {
app.get(
'/metrics',
ExpressCompression({
level: parseInt(process.env.METRICS_COMPRESSION_LEVEL || '1', 10)
}),
function(req, res) {
res.set('Content-Type', promWrapper.registry.contentType)
res.end(promWrapper.registry.metrics())
}
)
}
function buildPromKey(key) {
return key.replace(/[^a-zA-Z0-9]/g, '_')
}
function sanitizeValue(value) {
return parseFloat(value)
}
function set(key, value, sampleRate = 1) {
console.log('counts are not currently supported')
}
function inc(key, sampleRate = 1, opts = {}) {
key = buildPromKey(key)
promWrapper.metric('counter', key).inc(opts)
if (process.env.DEBUG_METRICS) {
console.log('doing inc', key, opts)
}
}
function count(key, count, sampleRate = 1, opts = {}) {
key = buildPromKey(key)
promWrapper.metric('counter', key).inc(opts, count)
if (process.env.DEBUG_METRICS) {
console.log('doing count/inc', key, opts)
}
}
function summary(key, value, opts = {}) {
key = buildPromKey(key)
promWrapper.metric('summary', key).observe(opts, value)
if (process.env.DEBUG_METRICS) {
console.log('doing summary', key, value, opts)
}
}
function timing(key, timeSpan, sampleRate = 1, opts = {}) {
key = buildPromKey('timer_' + key)
promWrapper.metric('summary', key).observe(opts, timeSpan)
if (process.env.DEBUG_METRICS) {
console.log('doing timing', key, opts)
}
}
class Timer {
constructor(key, sampleRate = 1, opts = {}) {
this.start = new Date()
key = buildPromKey(key)
this.key = key
this.sampleRate = sampleRate
this.opts = opts
}
done() {
const timeSpan = new Date() - this.start
timing(this.key, timeSpan, this.sampleRate, this.opts)
return timeSpan
}
}
function gauge(key, value, sampleRate = 1, opts = {}) {
key = buildPromKey(key)
promWrapper
.metric('gauge', key)
.set({ status: opts.status }, sanitizeValue(value))
if (process.env.DEBUG_METRICS) {
console.log('doing gauge', key, opts)
}
}
function globalGauge(key, value, sampleRate = 1, opts = {}) {
key = buildPromKey(key)
promWrapper
.metric('gauge', key)
.set({ host: 'global', status: opts.status }, sanitizeValue(value))
}
function close() {
for (const func of destructors) {
func()
}
}
module.exports = {
configure,
initialize,
registerDestructor,
injectMetricsRoute,
buildPromKey,
sanitizeValue,
set,
inc,
count,
summary,
timing,
Timer,
gauge,
globalGauge,
close,
prom: promClient,
register: promWrapper.registry,
mongodb: require('./mongodb'),
http: require('./http'),
open_sockets: require('./open_sockets'),
event_loop: require('./event_loop'),
memory: require('./memory'),
timeAsyncMethod: require('./timeAsyncMethod')
}

113
libraries/metrics/memory.js Normal file
View file

@ -0,0 +1,113 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
// record memory usage each minute and run a periodic gc(), keeping cpu
// usage within allowable range of 1ms per minute. Also, dynamically
// adjust the period between gc()'s to reach a target of the gc saving
// 4 megabytes each time.
let MemoryMonitor
const oneMinute = 60 * 1000
const oneMegaByte = 1024 * 1024
let CpuTimeBucket = 100 // current cpu time allowance in milliseconds
const CpuTimeBucketMax = 100 // maximum amount of cpu time allowed in bucket
const CpuTimeBucketRate = 10 // add this many milliseconds per minute
let gcInterval = 1 // how many minutes between gc (parameter is dynamically adjusted)
let countSinceLastGc = 0 // how many minutes since last gc
const MemoryChunkSize = 4 // how many megabytes we need to free to consider gc worth doing
const readyToGc = function() {
// update allowed cpu time
CpuTimeBucket = CpuTimeBucket + CpuTimeBucketRate
CpuTimeBucket =
CpuTimeBucket < CpuTimeBucketMax ? CpuTimeBucket : CpuTimeBucketMax
// update counts since last gc
countSinceLastGc = countSinceLastGc + 1
// check there is enough time since last gc and we have enough cpu
return countSinceLastGc > gcInterval && CpuTimeBucket > 0
}
const executeAndTime = function(fn) {
// time the execution of fn() and subtract from cpu allowance
const t0 = process.hrtime()
fn()
const dt = process.hrtime(t0)
const timeTaken = (dt[0] + dt[1] * 1e-9) * 1e3 // in milliseconds
CpuTimeBucket -= Math.ceil(timeTaken)
return timeTaken
}
const inMegaBytes = function(obj) {
// convert process.memoryUsage hash {rss,heapTotal,heapFreed} into megabytes
const result = {}
for (const k in obj) {
const v = obj[k]
result[k] = (v / oneMegaByte).toFixed(2)
}
return result
}
const updateMemoryStats = function(oldMem, newMem) {
countSinceLastGc = 0
const delta = {}
for (const k in newMem) {
delta[k] = (newMem[k] - oldMem[k]).toFixed(2)
}
// take the max of all memory measures
const savedMemory = Math.max(-delta.rss, -delta.heapTotal, -delta.heapUsed)
delta.megabytesFreed = savedMemory
// did it do any good?
if (savedMemory < MemoryChunkSize) {
gcInterval = gcInterval + 1 // no, so wait longer next time
} else {
gcInterval = Math.max(gcInterval - 1, 1) // yes, wait less time
}
return delta
}
module.exports = MemoryMonitor = {
monitor(logger) {
const interval = setInterval(() => MemoryMonitor.Check(logger), oneMinute)
const Metrics = require('./index')
return Metrics.registerDestructor(() => clearInterval(interval))
},
Check(logger) {
let mem
const Metrics = require('./index')
const memBeforeGc = (mem = inMegaBytes(process.memoryUsage()))
Metrics.gauge('memory.rss', mem.rss)
Metrics.gauge('memory.heaptotal', mem.heapTotal)
Metrics.gauge('memory.heapused', mem.heapUsed)
Metrics.gauge('memory.gc-interval', gcInterval)
// Metrics.gauge("memory.cpu-time-bucket", CpuTimeBucket)
logger.log(mem, 'process.memoryUsage()')
if (global.gc != null && readyToGc()) {
const gcTime = executeAndTime(global.gc).toFixed(2)
const memAfterGc = inMegaBytes(process.memoryUsage())
const deltaMem = updateMemoryStats(memBeforeGc, memAfterGc)
logger.log(
{
gcTime,
memBeforeGc,
memAfterGc,
deltaMem,
gcInterval,
CpuTimeBucket
},
'global.gc() forced'
)
// Metrics.timing("memory.gc-time", gcTime)
Metrics.gauge('memory.gc-rss-freed', -deltaMem.rss)
Metrics.gauge('memory.gc-heaptotal-freed', -deltaMem.heapTotal)
return Metrics.gauge('memory.gc-heapused-freed', -deltaMem.heapUsed)
}
}
}

View file

@ -0,0 +1,200 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
module.exports = {
monitor(mongodbRequirePath, logger) {
let mongodb, mongodbCore
try {
// for the v1 driver the methods to wrap are in the mongodb
// module in lib/mongodb/db.js
mongodb = require(mongodbRequirePath)
} catch (error) {}
try {
// for the v2 driver the relevant methods are in the mongodb-core
// module in lib/topologies/{server,replset,mongos}.js
const v2Path = mongodbRequirePath.replace(/\/mongodb$/, '/mongodb-core')
mongodbCore = require(v2Path)
} catch (error1) {}
const Metrics = require('./index')
const monitorMethod = function(base, method, type) {
let _method
if (base == null) {
return
}
if ((_method = base[method]) == null) {
return
}
const arglen = _method.length
const mongoDriverV1Wrapper = function(dbCommand, options, callback) {
let query
if (typeof callback === 'undefined') {
callback = options
options = {}
}
const collection = dbCommand.collectionName
if (collection.match(/\$cmd$/)) {
// Ignore noisy command methods like authenticating, ismaster and ping
return _method.call(this, dbCommand, options, callback)
}
if (dbCommand.query != null) {
query = Object.keys(dbCommand.query)
.sort()
.join('_')
}
const timer = new Metrics.Timer('mongo', { collection, query })
const start = new Date()
return _method.call(this, dbCommand, options, function() {
timer.done()
logger.log(
{
query: dbCommand.query,
query_type: type,
collection,
'response-time': new Date() - start
},
'mongo request'
)
return callback.apply(this, arguments)
})
}
const mongoDriverV2Wrapper = function(ns, ops, options, callback) {
let query
if (typeof callback === 'undefined') {
callback = options
options = {}
}
if (ns.match(/\$cmd$/)) {
// Ignore noisy command methods like authenticating, ismaster and ping
return _method.call(this, ns, ops, options, callback)
}
let key = `mongo-requests.${ns}.${type}`
if (ops[0].q != null) {
// ops[0].q
query = Object.keys(ops[0].q)
.sort()
.join('_')
key += '.' + query
}
const timer = new Metrics.Timer(key)
const start = new Date()
return _method.call(this, ns, ops, options, function() {
timer.done()
logger.log(
{
query: ops[0].q,
query_type: type,
collection: ns,
'response-time': new Date() - start
},
'mongo request'
)
return callback.apply(this, arguments)
})
}
if (arglen === 3) {
return (base[method] = mongoDriverV1Wrapper)
} else if (arglen === 4) {
return (base[method] = mongoDriverV2Wrapper)
}
}
monitorMethod(
mongodb != null ? mongodb.Db.prototype : undefined,
'_executeQueryCommand',
'query'
)
monitorMethod(
mongodb != null ? mongodb.Db.prototype : undefined,
'_executeRemoveCommand',
'remove'
)
monitorMethod(
mongodb != null ? mongodb.Db.prototype : undefined,
'_executeInsertCommand',
'insert'
)
monitorMethod(
mongodb != null ? mongodb.Db.prototype : undefined,
'_executeUpdateCommand',
'update'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Server.prototype : undefined,
'command',
'command'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Server.prototype : undefined,
'remove',
'remove'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Server.prototype : undefined,
'insert',
'insert'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Server.prototype : undefined,
'update',
'update'
)
monitorMethod(
mongodbCore != null ? mongodbCore.ReplSet.prototype : undefined,
'command',
'command'
)
monitorMethod(
mongodbCore != null ? mongodbCore.ReplSet.prototype : undefined,
'remove',
'remove'
)
monitorMethod(
mongodbCore != null ? mongodbCore.ReplSet.prototype : undefined,
'insert',
'insert'
)
monitorMethod(
mongodbCore != null ? mongodbCore.ReplSet.prototype : undefined,
'update',
'update'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Mongos.prototype : undefined,
'command',
'command'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Mongos.prototype : undefined,
'remove',
'remove'
)
monitorMethod(
mongodbCore != null ? mongodbCore.Mongos.prototype : undefined,
'insert',
'insert'
)
return monitorMethod(
mongodbCore != null ? mongodbCore.Mongos.prototype : undefined,
'update',
'update'
)
}
}

View file

@ -0,0 +1,52 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS205: Consider reworking code to avoid use of IIFEs
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OpenSocketsMonitor
const seconds = 1000
// In Node 0.10 the default is 5, which means only 5 open connections at one.
// Node 0.12 has a default of Infinity. Make sure we have no limit set,
// regardless of Node version.
require('http').globalAgent.maxSockets = Infinity
require('https').globalAgent.maxSockets = Infinity
const SOCKETS_HTTP = require('http').globalAgent.sockets
const SOCKETS_HTTPS = require('https').globalAgent.sockets
// keep track of set gauges and reset them in the next collection cycle
const SEEN_HOSTS_HTTP = new Set()
const SEEN_HOSTS_HTTPS = new Set()
function collectOpenConnections(sockets, seenHosts, prefix) {
const Metrics = require('./index')
Object.keys(sockets).forEach(host => seenHosts.add(host))
seenHosts.forEach(host => {
// host: 'HOST:PORT:'
const hostname = host.split(':')[0]
const openConnections = (sockets[host] || []).length
if (!openConnections) {
seenHosts.delete(host)
}
Metrics.gauge(`open_connections.${prefix}.${hostname}`, openConnections)
})
}
module.exports = OpenSocketsMonitor = {
monitor(logger) {
const interval = setInterval(
() => OpenSocketsMonitor.gaugeOpenSockets(),
5 * seconds
)
const Metrics = require('./index')
return Metrics.registerDestructor(() => clearInterval(interval))
},
gaugeOpenSockets() {
collectOpenConnections(SOCKETS_HTTP, SEEN_HOSTS_HTTP, 'http')
collectOpenConnections(SOCKETS_HTTPS, SEEN_HOSTS_HTTPS, 'https')
}
}

5472
libraries/metrics/package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,44 @@
{
"name": "@overleaf/metrics",
"version": "3.5.1",
"description": "A drop-in metrics and monitoring module for node.js apps",
"repository": {
"type": "git",
"url": "https://github.com/overleaf/metrics-module.git"
},
"dependencies": {
"@google-cloud/debug-agent": "^5.1.2",
"@google-cloud/profiler": "^4.0.3",
"@google-cloud/trace-agent": "^5.1.1",
"compression": "^1.7.4",
"prom-client": "^11.1.3",
"underscore": "~1.6.0",
"yn": "^3.1.1"
},
"devDependencies": {
"bunyan": "^1.0.0",
"chai": "^4.2.0",
"eslint": "^7.8.1",
"eslint-config-prettier": "^6.11.0",
"eslint-config-standard": "^14.1.1",
"eslint-plugin-chai-expect": "^2.2.0",
"eslint-plugin-chai-friendly": "^0.6.0",
"eslint-plugin-import": "^2.22.0",
"eslint-plugin-mocha": "^8.0.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.1",
"mocha": "^8.0.1",
"prettier-eslint-cli": "^5.0.0",
"sandboxed-module": "^2.0.4",
"sinon": "^9.0.2"
},
"scripts": {
"lint": "eslint --max-warnings 0 .",
"test:unit": "mocha --reporter spec --recursive --exit --grep=$MOCHA_GREP test/unit",
"test:acceptance": "mocha --reporter spec --recursive --exit --grep=$MOCHA_GREP test/acceptance",
"test": "npm run test:unit && npm run test:acceptance",
"format": "prettier-eslint $PWD'/**/*.js' --list-different",
"format:fix": "prettier-eslint $PWD'/**/*.js' --write"
}
}

View file

@ -0,0 +1,168 @@
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS205: Consider reworking code to avoid use of IIFEs
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const prom = require('prom-client')
const registry = require('prom-client').register
const metrics = new Map()
const optsKey = function(opts) {
let keys = Object.keys(opts)
if (keys.length === 0) {
return ''
}
keys = keys.sort()
let hash = ''
for (const key of Array.from(keys)) {
if (hash.length) {
hash += ','
}
hash += `${key}:${opts[key]}`
}
return hash
}
const extendOpts = function(opts, labelNames) {
for (const label of Array.from(labelNames)) {
if (!opts[label]) {
opts[label] = ''
}
}
return opts
}
const optsAsArgs = function(opts, labelNames) {
const args = []
for (const label of Array.from(labelNames)) {
args.push(opts[label] || '')
}
return args
}
const PromWrapper = {
ttlInMinutes: 0,
registry,
metric(type, name) {
return metrics.get(name) || new MetricWrapper(type, name)
},
collectDefaultMetrics: prom.collectDefaultMetrics
}
class MetricWrapper {
constructor(type, name) {
metrics.set(name, this)
this.name = name
this.instances = new Map()
this.lastAccess = new Date()
this.metric = (() => {
switch (type) {
case 'counter':
return new prom.Counter({
name,
help: name,
labelNames: ['status', 'method', 'path']
})
case 'summary':
return new prom.Summary({
name,
help: name,
maxAgeSeconds: 60,
ageBuckets: 10,
labelNames: ['path', 'status_code', 'method', 'collection', 'query']
})
case 'gauge':
return new prom.Gauge({
name,
help: name,
labelNames: ['host', 'status']
})
}
})()
}
inc(opts, value) {
return this._execMethod('inc', opts, value)
}
observe(opts, value) {
return this._execMethod('observe', opts, value)
}
set(opts, value) {
return this._execMethod('set', opts, value)
}
sweep() {
const thresh = new Date(Date.now() - 1000 * 60 * PromWrapper.ttlInMinutes)
this.instances.forEach((instance, key) => {
if (thresh > instance.time) {
if (process.env.DEBUG_METRICS) {
console.log(
'Sweeping stale metric instance',
this.name,
{ opts: instance.opts },
key
)
}
return this.metric.remove(
...Array.from(optsAsArgs(instance.opts, this.metric.labelNames) || [])
)
}
})
if (thresh > this.lastAccess) {
if (process.env.DEBUG_METRICS) {
console.log('Sweeping stale metric', this.name, thresh, this.lastAccess)
}
metrics.delete(this.name)
return registry.removeSingleMetric(this.name)
}
}
_execMethod(method, opts, value) {
opts = extendOpts(opts, this.metric.labelNames)
const key = optsKey(opts)
if (key !== '') {
this.instances.set(key, { time: new Date(), opts })
}
this.lastAccess = new Date()
return this.metric[method](opts, value)
}
}
let sweepingInterval
PromWrapper.setupSweeping = function() {
if (sweepingInterval) {
clearInterval(sweepingInterval)
}
if (!PromWrapper.ttlInMinutes) {
if (process.env.DEBUG_METRICS) {
console.log('Not registering sweep method -- empty ttl')
}
return
}
if (process.env.DEBUG_METRICS) {
console.log('Registering sweep method')
}
sweepingInterval = setInterval(function() {
if (process.env.DEBUG_METRICS) {
console.log('Sweeping metrics')
}
return metrics.forEach((metric, key) => {
return metric.sweep()
})
}, 60000)
const Metrics = require('./index')
Metrics.registerDestructor(() => clearInterval(sweepingInterval))
}
module.exports = PromWrapper

View file

@ -0,0 +1,245 @@
const os = require('os')
const http = require('http')
const { expect } = require('chai')
const Metrics = require('../..')
const HOSTNAME = os.hostname()
const APP_NAME = 'test-app'
describe('Metrics module', function() {
before(function() {
Metrics.initialize(APP_NAME)
})
describe('at startup', function() {
it('increments the process_startup counter', async function() {
await expectMetricValue('process_startup', 1)
})
it('collects default metrics', async function() {
const metric = await getMetric('process_cpu_user_seconds_total')
expect(metric).to.exist
})
})
describe('inc()', function() {
it('increments counts by 1', async function() {
Metrics.inc('duck_count')
await expectMetricValue('duck_count', 1)
Metrics.inc('duck_count')
Metrics.inc('duck_count')
await expectMetricValue('duck_count', 3)
})
it('escapes special characters in the key', async function() {
Metrics.inc('show.me the $!!')
await expectMetricValue('show_me_the____', 1)
})
})
describe('count()', function() {
it('increments counts by the given count', async function() {
Metrics.count('rabbit_count', 5)
await expectMetricValue('rabbit_count', 5)
Metrics.count('rabbit_count', 6)
Metrics.count('rabbit_count', 7)
await expectMetricValue('rabbit_count', 18)
})
})
describe('summary()', function() {
it('collects observations', async function() {
Metrics.summary('oven_temp', 200)
Metrics.summary('oven_temp', 300)
Metrics.summary('oven_temp', 450)
const sum = await getSummarySum('oven_temp')
expect(sum).to.equal(950)
})
})
describe('timing()', function() {
it('collects timings', async function() {
Metrics.timing('sprint_100m', 10)
Metrics.timing('sprint_100m', 20)
Metrics.timing('sprint_100m', 30)
const sum = await getSummarySum('timer_sprint_100m')
expect(sum).to.equal(60)
})
})
describe('gauge()', function() {
it('records values', async function() {
Metrics.gauge('water_level', 1.5)
await expectMetricValue('water_level', 1.5)
Metrics.gauge('water_level', 4.2)
await expectMetricValue('water_level', 4.2)
})
})
describe('globalGauge()', function() {
it('records values without a host label', async function() {
Metrics.globalGauge('tire_pressure', 99.99)
const { value, labels } = await getMetricValue('tire_pressure')
expect(value).to.equal(99.99)
expect(labels.host).to.equal('global')
expect(labels.app).to.equal(APP_NAME)
})
})
describe('open_sockets', function() {
const keyServer1 = 'open_connections_http_127_42_42_1'
const keyServer2 = 'open_connections_http_127_42_42_2'
let finish1, finish2, emitResponse1, emitResponse2
function resetEmitResponse1() {
emitResponse1 = new Promise(resolve => (finish1 = resolve))
}
resetEmitResponse1()
function resetEmitResponse2() {
emitResponse2 = new Promise(resolve => (finish2 = resolve))
}
resetEmitResponse2()
let server1, server2
before(function setupServer1(done) {
server1 = http.createServer((req, res) => {
res.write('...')
emitResponse1.then(() => res.end())
})
server1.listen(0, '127.42.42.1', done)
})
before(function setupServer2(done) {
server2 = http.createServer((req, res) => {
res.write('...')
emitResponse2.then(() => res.end())
})
server2.listen(0, '127.42.42.2', done)
})
after(function cleanupPendingRequests() {
finish1()
finish2()
})
after(function shutdownServer1(done) {
if (server1) server1.close(done)
})
after(function shutdownServer2(done) {
if (server2) server2.close(done)
})
let urlServer1, urlServer2
before(function setUrls() {
urlServer1 = `http://127.42.42.1:${server1.address().port}/`
urlServer2 = `http://127.42.42.2:${server2.address().port}/`
})
describe('gaugeOpenSockets()', function() {
beforeEach(function runGaugeOpenSockets() {
Metrics.open_sockets.gaugeOpenSockets()
})
describe('without pending connections', function() {
it('emits no open_connections', async function() {
await expectNoMetricValue(keyServer1)
await expectNoMetricValue(keyServer2)
})
})
describe('with pending connections for server1', function() {
before(function(done) {
http.get(urlServer1)
http.get(urlServer1)
setTimeout(done, 10)
})
it('emits 2 open_connections for server1', async function() {
await expectMetricValue(keyServer1, 2)
})
it('emits no open_connections for server2', async function() {
await expectNoMetricValue(keyServer2)
})
})
describe('with pending connections for server1 and server2', function() {
before(function(done) {
http.get(urlServer2)
http.get(urlServer2)
setTimeout(done, 10)
})
it('emits 2 open_connections for server1', async function() {
await expectMetricValue(keyServer1, 2)
})
it('emits 2 open_connections for server1', async function() {
await expectMetricValue(keyServer1, 2)
})
})
describe('when requests finish for server1', function() {
before(function(done) {
finish1()
resetEmitResponse1()
http.get(urlServer1)
setTimeout(done, 10)
})
it('emits 1 open_connections for server1', async function() {
await expectMetricValue(keyServer1, 1)
})
it('emits 2 open_connections for server2', async function() {
await expectMetricValue(keyServer2, 2)
})
})
describe('when all requests complete', function() {
before(function(done) {
finish1()
finish2()
setTimeout(done, 10)
})
it('emits no open_connections', async function() {
await expectNoMetricValue(keyServer1)
await expectNoMetricValue(keyServer2)
})
})
})
})
})
function getMetric(key) {
return Metrics.register.getSingleMetric(key)
}
async function getSummarySum(key) {
const metric = getMetric(key)
const item = await metric.get()
for (const value of item.values) {
if (value.metricName === `${key}_sum`) {
return value.value
}
}
return null
}
async function getMetricValue(key) {
const metrics = await Metrics.register.getMetricsAsJSON()
const metric = metrics.find(m => m.name === key)
return metric.values[0]
}
async function expectMetricValue(key, expectedValue) {
const value = await getMetricValue(key)
expect(value.value).to.equal(expectedValue)
expect(value.labels.host).to.equal(HOSTNAME)
expect(value.labels.app).to.equal(APP_NAME)
}
async function expectNoMetricValue(key) {
const metric = getMetric(key)
if (!metric) return
await expectMetricValue(key, 0)
}

View file

@ -0,0 +1,44 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const chai = require('chai')
const { expect } = chai
const path = require('path')
const modulePath = path.join(__dirname, '../../../event_loop.js')
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
describe('event_loop', function() {
before(function() {
this.metrics = {
timing: sinon.stub(),
registerDestructor: sinon.stub()
}
this.logger = {
warn: sinon.stub()
}
return (this.event_loop = SandboxedModule.require(modulePath, {
requires: {
'./index': this.metrics
}
}))
})
describe('with a logger provided', function() {
before(function() {
return this.event_loop.monitor(this.logger)
})
return it('should register a destructor with metrics', function() {
return expect(this.metrics.registerDestructor.called).to.equal(true)
})
})
return describe('without a logger provided', function() {
return it('should throw an exception', function() {
return expect(this.event_loop.monitor).to.throw('logger is undefined')
})
})
})

View file

@ -0,0 +1,201 @@
/*
* decaffeinate suggestions:
* DS102: Remove unnecessary code created because of implicit returns
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const chai = require('chai')
const { expect } = chai
const path = require('path')
const modulePath = path.join(__dirname, '../../../timeAsyncMethod.js')
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
describe('timeAsyncMethod', function() {
beforeEach(function() {
this.Timer = { done: sinon.stub() }
this.TimerConstructor = sinon.stub().returns(this.Timer)
this.metrics = {
Timer: this.TimerConstructor,
inc: sinon.stub()
}
this.timeAsyncMethod = SandboxedModule.require(modulePath, {
requires: {
'./index': this.metrics
}
})
return (this.testObject = {
nextNumber(n, callback) {
return setTimeout(() => callback(null, n + 1), 100)
}
})
})
it('should have the testObject behave correctly before wrapping', function(done) {
return this.testObject.nextNumber(2, (err, result) => {
expect(err).to.not.exist
expect(result).to.equal(3)
return done()
})
})
it('should wrap method without error', function(done) {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
return done()
})
it('should transparently wrap method invocation in timer', function(done) {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
return this.testObject.nextNumber(2, (err, result) => {
expect(err).to.not.exist
expect(result).to.equal(3)
expect(this.TimerConstructor.callCount).to.equal(1)
expect(this.Timer.done.callCount).to.equal(1)
return done()
})
})
it('should increment success count', function(done) {
this.metrics.inc = sinon.stub()
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
return this.testObject.nextNumber(2, (err, result) => {
if (err) {
return done(err)
}
expect(this.metrics.inc.callCount).to.equal(1)
expect(
this.metrics.inc.calledWith('someContext_result', 1, {
method: 'TestObject_nextNumber',
status: 'success'
})
).to.equal(true)
return done()
})
})
describe('when base method produces an error', function() {
beforeEach(function() {
this.metrics.inc = sinon.stub()
return (this.testObject.nextNumber = function(n, callback) {
return setTimeout(() => callback(new Error('woops')), 100)
})
})
it('should propagate the error transparently', function(done) {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
return this.testObject.nextNumber(2, (err, result) => {
expect(err).to.exist
expect(err).to.be.instanceof(Error)
expect(result).to.not.exist
return done()
})
})
return it('should increment failure count', function(done) {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
return this.testObject.nextNumber(2, (err, result) => {
expect(err).to.exist
expect(this.metrics.inc.callCount).to.equal(1)
expect(
this.metrics.inc.calledWith('someContext_result', 1, {
method: 'TestObject_nextNumber',
status: 'failed'
})
).to.equal(true)
return done()
})
})
})
describe('when a logger is supplied', function() {
beforeEach(function() {
return (this.logger = { log: sinon.stub() })
})
return it('should also call logger.log', function(done) {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject',
this.logger
)
return this.testObject.nextNumber(2, (err, result) => {
expect(err).to.not.exist
expect(result).to.equal(3)
expect(this.TimerConstructor.callCount).to.equal(1)
expect(this.Timer.done.callCount).to.equal(1)
expect(this.logger.log.callCount).to.equal(1)
return done()
})
})
})
describe('when the wrapper cannot be applied', function() {
beforeEach(function() {})
return it('should raise an error', function() {
const badWrap = () => {
return this.timeAsyncMethod(
this.testObject,
'DEFINITELY_NOT_A_REAL_METHOD',
'someContext.TestObject'
)
}
return expect(badWrap).to.throw(
/^.*expected object property 'DEFINITELY_NOT_A_REAL_METHOD' to be a function.*$/
)
})
})
return describe('when the wrapped function is not using a callback', function() {
beforeEach(function() {
this.realMethod = sinon.stub().returns(42)
return (this.testObject.nextNumber = this.realMethod)
})
it('should not throw an error', function() {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
const badCall = () => {
return this.testObject.nextNumber(2)
}
return expect(badCall).to.not.throw(Error)
})
return it('should call the underlying method', function() {
this.timeAsyncMethod(
this.testObject,
'nextNumber',
'someContext.TestObject'
)
const result = this.testObject.nextNumber(12)
expect(this.realMethod.callCount).to.equal(1)
expect(this.realMethod.calledWith(12)).to.equal(true)
return expect(result).to.equal(42)
})
})
})

View file

@ -0,0 +1,84 @@
/*
* decaffeinate suggestions:
* DS101: Remove unnecessary use of Array.from
* DS102: Remove unnecessary code created because of implicit returns
* DS201: Simplify complex destructure assignments
* DS207: Consider shorter variations of null checks
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
module.exports = function(obj, methodName, prefix, logger) {
let modifedMethodName
const metrics = require('./index')
if (typeof obj[methodName] !== 'function') {
throw new Error(
`[Metrics] expected object property '${methodName}' to be a function`
)
}
const key = `${prefix}.${methodName}`
const realMethod = obj[methodName]
const splitPrefix = prefix.split('.')
const startPrefix = splitPrefix[0]
if (splitPrefix[1] != null) {
modifedMethodName = `${splitPrefix[1]}_${methodName}`
} else {
modifedMethodName = methodName
}
return (obj[methodName] = function(...originalArgs) {
const adjustedLength = Math.max(originalArgs.length, 1)
const firstArgs = originalArgs.slice(0, adjustedLength - 1)
const callback = originalArgs[adjustedLength - 1]
if (callback == null || typeof callback !== 'function') {
if (logger != null) {
logger.log(
`[Metrics] expected wrapped method '${methodName}' to be invoked with a callback`
)
}
return realMethod.apply(this, originalArgs)
}
const timer = new metrics.Timer(startPrefix, 1, {
method: modifedMethodName
})
return realMethod.call(this, ...Array.from(firstArgs), function(
...callbackArgs
) {
const elapsedTime = timer.done()
const possibleError = callbackArgs[0]
if (possibleError != null) {
metrics.inc(`${startPrefix}_result`, 1, {
status: 'failed',
method: modifedMethodName
})
} else {
metrics.inc(`${startPrefix}_result`, 1, {
status: 'success',
method: modifedMethodName
})
}
if (logger != null) {
const loggableArgs = {}
try {
for (let idx = 0; idx < firstArgs.length; idx++) {
const arg = firstArgs[idx]
if (arg.toString().match(/^[0-9a-f]{24}$/)) {
loggableArgs[`${idx}`] = arg
}
}
} catch (error) {}
logger.log(
{ key, args: loggableArgs, elapsedTime },
'[Metrics] timed async method call'
)
}
return callback.apply(this, callbackArgs)
})
})
}

View file

@ -0,0 +1,4 @@
if (!process.env.UV_THREADPOOL_SIZE) {
process.env.UV_THREADPOOL_SIZE = 16
console.log(`Set UV_THREADPOOL_SIZE=${process.env.UV_THREADPOOL_SIZE}`)
}