From 17e30684993a46801895d074bfb13323578ab19b Mon Sep 17 00:00:00 2001
From: Brian Gough <brian.gough@overleaf.com>
Date: Fri, 19 Mar 2021 16:27:27 +0000
Subject: [PATCH] close real-time via a status file

---
 services/real-time/app.js                     | 15 +++++-
 .../real-time/app/js/DeploymentManager.js     | 53 +++++++++++++++++++
 .../real-time/config/settings.defaults.js     |  6 +++
 3 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 services/real-time/app/js/DeploymentManager.js

diff --git a/services/real-time/app.js b/services/real-time/app.js
index e65561cf43..832bf52a53 100644
--- a/services/real-time/app.js
+++ b/services/real-time/app.js
@@ -22,6 +22,7 @@ const CookieParser = require('cookie-parser')
 
 const DrainManager = require('./app/js/DrainManager')
 const HealthCheckManager = require('./app/js/HealthCheckManager')
+const DeploymentManager = require('./app/js/DeploymentManager')
 
 // NOTE: debug is invoked for every blob that is put on the wire
 const socketIoLogger = {
@@ -36,6 +37,9 @@ const socketIoLogger = {
   log() {}
 }
 
+// monitor status file to take dark deployments out of the load-balancer
+DeploymentManager.initialise()
+
 // Set up socket.io server
 const app = express()
 
@@ -79,13 +83,20 @@ io.configure(function () {
 })
 
 // a 200 response on '/' is required for load balancer health checks
-app.get('/', (req, res) => res.send('real-time-sharelatex is alive'))
+// these operate separately from kubernetes readiness checks
+app.get('/', function (req, res) {
+  if (Settings.serviceIsClosed || Settings.shutDownInProgress) {
+    res.sendStatus(503) // Service unavailable
+  } else {
+    res.send('real-time is open')
+  }
+})
 
 app.get('/status', function (req, res) {
   if (Settings.shutDownInProgress) {
     res.sendStatus(503) // Service unavailable
   } else {
-    res.send('real-time-sharelatex is alive')
+    res.send('real-time is alive')
   }
 })
 
diff --git a/services/real-time/app/js/DeploymentManager.js b/services/real-time/app/js/DeploymentManager.js
new file mode 100644
index 0000000000..a86db02dd1
--- /dev/null
+++ b/services/real-time/app/js/DeploymentManager.js
@@ -0,0 +1,53 @@
+const logger = require('logger-sharelatex')
+const settings = require('settings-sharelatex')
+const fs = require('fs')
+
+// Monitor a status file (e.g. /etc/real_time_status) periodically and close the
+// service if the file contents don't contain the matching deployment colour.
+
+const FILE_CHECK_INTERVAL = 5000
+const statusFile = settings.deploymentFile
+const deploymentColour = settings.deploymentColour
+
+function updateDeploymentStatus(fileContent) {
+  const closed = fileContent && fileContent.indexOf(deploymentColour) === -1
+  if (closed && !settings.serviceIsClosed) {
+    settings.serviceIsClosed = true
+    logger.warn({ fileContent }, 'closing service')
+  } else if (!closed && settings.serviceIsClosed) {
+    settings.serviceIsClosed = false
+    logger.warn({ fileContent }, 'opening service')
+  }
+}
+
+function pollStatusFile() {
+  fs.readFile(statusFile, { encoding: 'utf8' }, (err, fileContent) => {
+    if (err) {
+      logger.error(
+        { file: statusFile, fsErr: err },
+        'error reading service status file'
+      )
+      return
+    }
+    updateDeploymentStatus(fileContent)
+  })
+}
+
+function checkStatusFileSync() {
+  // crash on start up if file does not exist
+  const content = fs.readFileSync(statusFile, { encoding: 'utf8' })
+  updateDeploymentStatus(content)
+}
+
+module.exports = {
+  initialise() {
+    if (statusFile && deploymentColour) {
+      logger.log(
+        { statusFile, deploymentColour, interval: FILE_CHECK_INTERVAL },
+        'monitoring deployment status file'
+      )
+      checkStatusFileSync() // perform an initial synchronous check at start up
+      setInterval(pollStatusFile, FILE_CHECK_INTERVAL) // continue checking periodically
+    }
+  }
+}
diff --git a/services/real-time/config/settings.defaults.js b/services/real-time/config/settings.defaults.js
index 486b686083..c15ca57f6f 100644
--- a/services/real-time/config/settings.defaults.js
+++ b/services/real-time/config/settings.defaults.js
@@ -148,6 +148,12 @@ const settings = {
 
   statusCheckInterval: parseInt(process.env.STATUS_CHECK_INTERVAL || '0'),
 
+  // The deployment colour for this app (if any). Used for blue green deploys.
+  deploymentColour: process.env.DEPLOYMENT_COLOUR,
+  // Load balancer health checks will return 200 only when this file contains
+  // the deployment colour for this app.
+  deploymentFile: process.env.DEPLOYMENT_FILE,
+
   sentry: {
     dsn: process.env.SENTRY_DSN
   },