diff --git a/services/web/app/src/Features/HealthCheck/HealthCheckController.js b/services/web/app/src/Features/HealthCheck/HealthCheckController.js index f4ace088a7..030272383f 100644 --- a/services/web/app/src/Features/HealthCheck/HealthCheckController.js +++ b/services/web/app/src/Features/HealthCheck/HealthCheckController.js @@ -57,6 +57,26 @@ module.exports = HealthCheckController = { }) }, + checkActiveHandles(req, res, next) { + if (!(settings.maxActiveHandles > 0) || !process._getActiveHandles) { + return next() + } + const activeHandlesCount = (process._getActiveHandles() || []).length + if (activeHandlesCount > settings.maxActiveHandles) { + logger.err( + { activeHandlesCount, maxActiveHandles: settings.maxActiveHandles }, + 'exceeded max active handles, failing health check' + ) + return res.sendStatus(500) + } else { + logger.debug( + { activeHandlesCount, maxActiveHandles: settings.maxActiveHandles }, + 'active handles are below maximum' + ) + next() + } + }, + checkApi(req, res, next) { rclient.healthCheck(err => { if (err) { diff --git a/services/web/app/src/router.js b/services/web/app/src/router.js index 8096cf1fea..e94946542f 100644 --- a/services/web/app/src/router.js +++ b/services/web/app/src/router.js @@ -981,8 +981,16 @@ function initialize(webRouter, privateApiRouter, publicApiRouter) { // used by kubernetes health-check and acceptance tests webRouter.get('/dev/csrf', (req, res) => res.send(res.locals.csrfToken)) - publicApiRouter.get('/health_check', HealthCheckController.check) - privateApiRouter.get('/health_check', HealthCheckController.checkApi) + publicApiRouter.get( + '/health_check', + HealthCheckController.checkActiveHandles, + HealthCheckController.check + ) + privateApiRouter.get( + '/health_check', + HealthCheckController.checkActiveHandles, + HealthCheckController.checkApi + ) publicApiRouter.get('/health_check/redis', HealthCheckController.checkRedis) privateApiRouter.get('/health_check/redis', HealthCheckController.checkRedis) diff --git a/services/web/config/settings.defaults.coffee b/services/web/config/settings.defaults.coffee index 331b05adcb..eebfe4e9b6 100644 --- a/services/web/config/settings.defaults.coffee +++ b/services/web/config/settings.defaults.coffee @@ -231,6 +231,9 @@ module.exports = settings = maxUploadSize: 50 * 1024 * 1024 # 50 MB + # start failing the health check if active handles exceeds this limit + maxActiveHandles: if process.env['MAX_ACTIVE_HANDLES'] then parseInt(process.env['MAX_ACTIVE_HANDLES'], 10) + # Security # -------- security: