From 5638f06b3af05ad1107cf361cb6d88e27c5f94ec Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Mon, 22 May 2023 11:36:51 +0100 Subject: [PATCH] Merge pull request #13055 from overleaf/jpa-monitor-sockets-everywhere [metrics] collect HTTP socket stats for all the services GitOrigin-RevId: 52872f4ca46d8cf351f7f4b4df90f9d6b05010b3 --- libraries/metrics/open_sockets.js | 38 ++++++++++++++----- .../metrics/test/acceptance/metrics_tests.js | 2 +- services/chat/app/js/server.js | 1 + services/clsi/app.js | 2 +- services/contacts/app/js/server.js | 1 + services/docstore/app.js | 1 + services/document-updater/app.js | 1 + services/filestore/app.js | 2 +- services/history-v1/app.js | 1 + services/notifications/app.js | 1 + services/project-history/app/js/server.js | 1 + services/real-time/app.js | 1 + services/spelling/app/js/server.js | 1 + services/track-changes/app.js | 1 + services/web/app.js | 1 + 15 files changed, 43 insertions(+), 12 deletions(-) diff --git a/libraries/metrics/open_sockets.js b/libraries/metrics/open_sockets.js index a89736077b..eb120bcf91 100644 --- a/libraries/metrics/open_sockets.js +++ b/libraries/metrics/open_sockets.js @@ -25,7 +25,13 @@ const SEEN_HOSTS_HTTPS = new Set() const FREE_SEEN_HOSTS_HTTP = new Set() const FREE_SEEN_HOSTS_HTTPS = new Set() -function collectConnectionsCount(sockets, seenHosts, status, https) { +function collectConnectionsCount( + sockets, + seenHosts, + status, + https, + emitLegacyMetric +) { const Metrics = require('./index') Object.keys(sockets).forEach(host => seenHosts.add(host)) seenHosts.forEach(host => { @@ -40,7 +46,7 @@ function collectConnectionsCount(sockets, seenHosts, status, https) { method: https, status, }) - if (status === 'open') { + if (status === 'open' && emitLegacyMetric) { // Emit legacy metric to keep old time series intact. Metrics.gauge( `${status}_connections.${https}.${hostname}`, @@ -51,29 +57,43 @@ function collectConnectionsCount(sockets, seenHosts, status, https) { } module.exports = OpenSocketsMonitor = { - monitor(logger) { + monitor(emitLegacyMetric) { const interval = setInterval( - () => OpenSocketsMonitor.gaugeOpenSockets(), + () => OpenSocketsMonitor.gaugeOpenSockets(emitLegacyMetric), 5 * seconds ) const Metrics = require('./index') return Metrics.registerDestructor(() => clearInterval(interval)) }, - gaugeOpenSockets() { - collectConnectionsCount(SOCKETS_HTTP, SEEN_HOSTS_HTTP, 'open', 'http') - collectConnectionsCount(SOCKETS_HTTPS, SEEN_HOSTS_HTTPS, 'open', 'https') + gaugeOpenSockets(emitLegacyMetric) { + collectConnectionsCount( + SOCKETS_HTTP, + SEEN_HOSTS_HTTP, + 'open', + 'http', + emitLegacyMetric + ) + collectConnectionsCount( + SOCKETS_HTTPS, + SEEN_HOSTS_HTTPS, + 'open', + 'https', + emitLegacyMetric + ) collectConnectionsCount( FREE_SOCKETS_HTTP, FREE_SEEN_HOSTS_HTTP, 'free', - 'http' + 'http', + false ) collectConnectionsCount( FREE_SOCKETS_HTTPS, FREE_SEEN_HOSTS_HTTPS, 'free', - 'https' + 'https', + false ) }, } diff --git a/libraries/metrics/test/acceptance/metrics_tests.js b/libraries/metrics/test/acceptance/metrics_tests.js index 6401e55ab0..42235eba73 100644 --- a/libraries/metrics/test/acceptance/metrics_tests.js +++ b/libraries/metrics/test/acceptance/metrics_tests.js @@ -189,7 +189,7 @@ describe('Metrics module', function () { }) describe('gaugeOpenSockets()', function () { beforeEach(function runGaugeOpenSockets() { - Metrics.open_sockets.gaugeOpenSockets() + Metrics.open_sockets.gaugeOpenSockets(true) }) describe('without pending connections', function () { diff --git a/services/chat/app/js/server.js b/services/chat/app/js/server.js index ede4b19a40..28f7b6f1b6 100644 --- a/services/chat/app/js/server.js +++ b/services/chat/app/js/server.js @@ -11,6 +11,7 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url)) metrics.initialize('chat') logger.initialize('chat') +metrics.open_sockets.monitor() export async function createServer() { const app = express() diff --git a/services/clsi/app.js b/services/clsi/app.js index fb8d335999..44e1e5c4d4 100644 --- a/services/clsi/app.js +++ b/services/clsi/app.js @@ -16,7 +16,7 @@ const Errors = require('./app/js/Errors') const Path = require('path') -Metrics.open_sockets.monitor(logger) +Metrics.open_sockets.monitor(true) Metrics.memory.monitor(logger) const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager') diff --git a/services/contacts/app/js/server.js b/services/contacts/app/js/server.js index 5fe888c1fe..e0b5a0e010 100644 --- a/services/contacts/app/js/server.js +++ b/services/contacts/app/js/server.js @@ -8,6 +8,7 @@ import * as Errors from './Errors.js' Metrics.initialize('contacts') logger.initialize('contacts') Metrics.event_loop?.monitor(logger) +Metrics.open_sockets.monitor() export const app = express() app.use(Metrics.http.monitor(logger)) diff --git a/services/docstore/app.js b/services/docstore/app.js index 8cde97ca27..f8e83fb4ec 100644 --- a/services/docstore/app.js +++ b/services/docstore/app.js @@ -23,6 +23,7 @@ logger.initialize('docstore') if (Metrics.event_loop != null) { Metrics.event_loop.monitor(logger) } +Metrics.open_sockets.monitor() const app = express() diff --git a/services/document-updater/app.js b/services/document-updater/app.js index de826c887e..68ad3ebd4a 100644 --- a/services/document-updater/app.js +++ b/services/document-updater/app.js @@ -23,6 +23,7 @@ const async = require('async') const bodyParser = require('body-parser') Metrics.event_loop.monitor(logger, 100) +Metrics.open_sockets.monitor() const app = express() app.use(bodyParser.json({ limit: Settings.maxJsonRequestSize })) diff --git a/services/filestore/app.js b/services/filestore/app.js index 48cbd1071d..a0406fac1c 100644 --- a/services/filestore/app.js +++ b/services/filestore/app.js @@ -22,7 +22,7 @@ if (settings.sentry && settings.sentry.dsn) { logger.initializeErrorReporting(settings.sentry.dsn) } -Metrics.open_sockets.monitor(logger) +Metrics.open_sockets.monitor(true) Metrics.memory.monitor(logger) if (Metrics.event_loop) { Metrics.event_loop.monitor(logger) diff --git a/services/history-v1/app.js b/services/history-v1/app.js index f8be20328e..9f1033c041 100644 --- a/services/history-v1/app.js +++ b/services/history-v1/app.js @@ -26,6 +26,7 @@ const app = express() module.exports = app logger.initialize('history-v1') +Metrics.open_sockets.monitor() Metrics.injectMetricsRoute(app) app.use(Metrics.http.monitor(logger)) diff --git a/services/notifications/app.js b/services/notifications/app.js index d798b43199..6f75629be5 100644 --- a/services/notifications/app.js +++ b/services/notifications/app.js @@ -18,6 +18,7 @@ const { mongoClient } = require('./app/js/mongodb') const controller = require('./app/js/NotificationsController') metrics.memory.monitor(logger) +metrics.open_sockets.monitor() const HealthCheckController = require('./app/js/HealthCheckController') diff --git a/services/project-history/app/js/server.js b/services/project-history/app/js/server.js index 7aae484f4b..d7b83975a4 100644 --- a/services/project-history/app/js/server.js +++ b/services/project-history/app/js/server.js @@ -16,6 +16,7 @@ if (Settings.sentry.dsn) { Metrics.initialize('project-history') Metrics.event_loop.monitor(logger) Metrics.memory.monitor(logger) +Metrics.open_sockets.monitor() // log updates as truncated strings function truncateFn(updates) { diff --git a/services/real-time/app.js b/services/real-time/app.js index 0893d2d7d0..295cdede39 100644 --- a/services/real-time/app.js +++ b/services/real-time/app.js @@ -6,6 +6,7 @@ const async = require('async') const logger = require('@overleaf/logger') logger.initialize('real-time') Metrics.event_loop.monitor(logger) +Metrics.open_sockets.monitor() const express = require('express') const session = require('express-session') diff --git a/services/spelling/app/js/server.js b/services/spelling/app/js/server.js index fcd89fb72e..50ac2420d1 100644 --- a/services/spelling/app/js/server.js +++ b/services/spelling/app/js/server.js @@ -12,6 +12,7 @@ if (Settings.sentry?.dsn != null) { logger.initializeErrorReporting(Settings.sentry.dsn) } metrics.memory.monitor(logger) +metrics.open_sockets.monitor() export const app = express() diff --git a/services/track-changes/app.js b/services/track-changes/app.js index 5be3447040..862994f7a0 100644 --- a/services/track-changes/app.js +++ b/services/track-changes/app.js @@ -41,6 +41,7 @@ TrackChangesLogger.addSerializers({ const Path = require('path') Metrics.memory.monitor(logger) +Metrics.open_sockets.monitor() const childProcess = require('child_process') diff --git a/services/web/app.js b/services/web/app.js index 2128fa46dc..28068c90bb 100644 --- a/services/web/app.js +++ b/services/web/app.js @@ -34,6 +34,7 @@ http.globalAgent.maxSockets = Settings.limits.httpGlobalAgentMaxSockets https.globalAgent.maxSockets = Settings.limits.httpsGlobalAgentMaxSockets metrics.memory.monitor(logger) +metrics.open_sockets.monitor() const Server = require('./app/src/infrastructure/Server') const QueueWorkers = require('./app/src/infrastructure/QueueWorkers')