Merge pull request #13055 from overleaf/jpa-monitor-sockets-everywhere

[metrics] collect HTTP socket stats for all the services

GitOrigin-RevId: 52872f4ca46d8cf351f7f4b4df90f9d6b05010b3

libraries/metrics/open_sockets.js

@@ -25,7 +25,13 @@ const SEEN_HOSTS_HTTPS = new Set()
 const FREE_SEEN_HOSTS_HTTP = new Set()
 const FREE_SEEN_HOSTS_HTTPS = new Set()
 
-function collectConnectionsCount(sockets, seenHosts, status, https) {
+function collectConnectionsCount(
+  sockets,
+  seenHosts,
+  status,
+  https,
+  emitLegacyMetric
+) {
   const Metrics = require('./index')
   Object.keys(sockets).forEach(host => seenHosts.add(host))
   seenHosts.forEach(host => {
@@ -40,7 +46,7 @@ function collectConnectionsCount(sockets, seenHosts, status, https) {
       method: https,
       status,
     })
-    if (status === 'open') {
+    if (status === 'open' && emitLegacyMetric) {
       // Emit legacy metric to keep old time series intact.
       Metrics.gauge(
         `${status}_connections.${https}.${hostname}`,
@@ -51,29 +57,43 @@ function collectConnectionsCount(sockets, seenHosts, status, https) {
 }
 
 module.exports = OpenSocketsMonitor = {
-  monitor(logger) {
+  monitor(emitLegacyMetric) {
     const interval = setInterval(
-      () => OpenSocketsMonitor.gaugeOpenSockets(),
+      () => OpenSocketsMonitor.gaugeOpenSockets(emitLegacyMetric),
       5 * seconds
     )
     const Metrics = require('./index')
     return Metrics.registerDestructor(() => clearInterval(interval))
   },
 
-  gaugeOpenSockets() {
+  gaugeOpenSockets(emitLegacyMetric) {
-    collectConnectionsCount(SOCKETS_HTTP, SEEN_HOSTS_HTTP, 'open', 'http')
+    collectConnectionsCount(
-    collectConnectionsCount(SOCKETS_HTTPS, SEEN_HOSTS_HTTPS, 'open', 'https')
+      SOCKETS_HTTP,
+      SEEN_HOSTS_HTTP,
+      'open',
+      'http',
+      emitLegacyMetric
+    )
+    collectConnectionsCount(
+      SOCKETS_HTTPS,
+      SEEN_HOSTS_HTTPS,
+      'open',
+      'https',
+      emitLegacyMetric
+    )
     collectConnectionsCount(
       FREE_SOCKETS_HTTP,
       FREE_SEEN_HOSTS_HTTP,
       'free',
-      'http'
+      'http',
+      false
     )
     collectConnectionsCount(
       FREE_SOCKETS_HTTPS,
       FREE_SEEN_HOSTS_HTTPS,
       'free',
-      'https'
+      'https',
+      false
     )
   },
 }

libraries/metrics/test/acceptance/metrics_tests.js

@@ -189,7 +189,7 @@ describe('Metrics module', function () {
     })
     describe('gaugeOpenSockets()', function () {
       beforeEach(function runGaugeOpenSockets() {
-        Metrics.open_sockets.gaugeOpenSockets()
+        Metrics.open_sockets.gaugeOpenSockets(true)
       })
 
       describe('without pending connections', function () {

services/chat/app/js/server.js

@@ -11,6 +11,7 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
 
 metrics.initialize('chat')
 logger.initialize('chat')
+metrics.open_sockets.monitor()
 
 export async function createServer() {
   const app = express()

services/clsi/app.js

@@ -16,7 +16,7 @@ const Errors = require('./app/js/Errors')
 
 const Path = require('path')
 
-Metrics.open_sockets.monitor(logger)
+Metrics.open_sockets.monitor(true)
 Metrics.memory.monitor(logger)
 
 const ProjectPersistenceManager = require('./app/js/ProjectPersistenceManager')

services/contacts/app/js/server.js

@@ -8,6 +8,7 @@ import * as Errors from './Errors.js'
 Metrics.initialize('contacts')
 logger.initialize('contacts')
 Metrics.event_loop?.monitor(logger)
+Metrics.open_sockets.monitor()
 
 export const app = express()
 app.use(Metrics.http.monitor(logger))

services/docstore/app.js

@@ -23,6 +23,7 @@ logger.initialize('docstore')
 if (Metrics.event_loop != null) {
   Metrics.event_loop.monitor(logger)
 }
+Metrics.open_sockets.monitor()
 
 const app = express()
 

services/document-updater/app.js

@@ -23,6 +23,7 @@ const async = require('async')
 const bodyParser = require('body-parser')
 
 Metrics.event_loop.monitor(logger, 100)
+Metrics.open_sockets.monitor()
 
 const app = express()
 app.use(bodyParser.json({ limit: Settings.maxJsonRequestSize }))

services/filestore/app.js

@@ -22,7 +22,7 @@ if (settings.sentry && settings.sentry.dsn) {
   logger.initializeErrorReporting(settings.sentry.dsn)
 }
 
-Metrics.open_sockets.monitor(logger)
+Metrics.open_sockets.monitor(true)
 Metrics.memory.monitor(logger)
 if (Metrics.event_loop) {
   Metrics.event_loop.monitor(logger)

services/history-v1/app.js

@@ -26,6 +26,7 @@ const app = express()
 module.exports = app
 
 logger.initialize('history-v1')
+Metrics.open_sockets.monitor()
 Metrics.injectMetricsRoute(app)
 app.use(Metrics.http.monitor(logger))
 

services/notifications/app.js

@@ -18,6 +18,7 @@ const { mongoClient } = require('./app/js/mongodb')
 const controller = require('./app/js/NotificationsController')
 
 metrics.memory.monitor(logger)
+metrics.open_sockets.monitor()
 
 const HealthCheckController = require('./app/js/HealthCheckController')
 

services/project-history/app/js/server.js

@@ -16,6 +16,7 @@ if (Settings.sentry.dsn) {
 Metrics.initialize('project-history')
 Metrics.event_loop.monitor(logger)
 Metrics.memory.monitor(logger)
+Metrics.open_sockets.monitor()
 
 // log updates as truncated strings
 function truncateFn(updates) {

services/real-time/app.js

@@ -6,6 +6,7 @@ const async = require('async')
 const logger = require('@overleaf/logger')
 logger.initialize('real-time')
 Metrics.event_loop.monitor(logger)
+Metrics.open_sockets.monitor()
 
 const express = require('express')
 const session = require('express-session')

services/spelling/app/js/server.js

@@ -12,6 +12,7 @@ if (Settings.sentry?.dsn != null) {
   logger.initializeErrorReporting(Settings.sentry.dsn)
 }
 metrics.memory.monitor(logger)
+metrics.open_sockets.monitor()
 
 export const app = express()
 

services/track-changes/app.js

@@ -41,6 +41,7 @@ TrackChangesLogger.addSerializers({
 const Path = require('path')
 
 Metrics.memory.monitor(logger)
+Metrics.open_sockets.monitor()
 
 const childProcess = require('child_process')
 

services/web/app.js

@@ -34,6 +34,7 @@ http.globalAgent.maxSockets = Settings.limits.httpGlobalAgentMaxSockets
 https.globalAgent.maxSockets = Settings.limits.httpsGlobalAgentMaxSockets
 
 metrics.memory.monitor(logger)
+metrics.open_sockets.monitor()
 
 const Server = require('./app/src/infrastructure/Server')
 const QueueWorkers = require('./app/src/infrastructure/QueueWorkers')