From 2435099766bb78d81f4d85e8be7b704fd8789164 Mon Sep 17 00:00:00 2001
From: Brian Gough <brian.gough@overleaf.com>
Date: Thu, 19 Sep 2019 15:05:32 +0100
Subject: [PATCH 1/2] make health check more robust

---
 libraries/redis-wrapper/health.coffee | 13 ++++++++++
 libraries/redis-wrapper/index.coffee  | 36 +++++++++++++++++++++++----
 2 files changed, 44 insertions(+), 5 deletions(-)
 create mode 100644 libraries/redis-wrapper/health.coffee

diff --git a/libraries/redis-wrapper/health.coffee b/libraries/redis-wrapper/health.coffee
new file mode 100644
index 0000000000..feb02a7f88
--- /dev/null
+++ b/libraries/redis-wrapper/health.coffee
@@ -0,0 +1,13 @@
+# execute this script with a redis container running to test the health check
+# starting and stopping redis with this script running is a good test
+
+redis = require "./index.coffee"
+
+rclient = redis.createClient({host:"localhost",port:"6379"})
+setInterval () ->
+    rclient.healthCheck (err) ->
+        if err?
+            console.log "HEALTH CHECK FAILED", err
+        else
+            console.log "HEALTH CHECK OK"
+, 1000
\ No newline at end of file
diff --git a/libraries/redis-wrapper/index.coffee b/libraries/redis-wrapper/index.coffee
index 18aa47c65f..674dae5594 100644
--- a/libraries/redis-wrapper/index.coffee
+++ b/libraries/redis-wrapper/index.coffee
@@ -1,5 +1,13 @@
 _ = require("underscore")
 async = require "async"
+os = require('os')
+crypto = require('crypto')
+
+# generate unique values for health check
+HOST = os.hostname()
+PID = process.pid
+RND = crypto.randomBytes(4).toString('hex')
+COUNT = 0
 
 module.exports = RedisSharelatex =
 	createClient: (opts = {port: 6379, host: "localhost"})->
@@ -44,19 +52,37 @@ module.exports = RedisSharelatex =
 	
 	_checkClient: (client, callback) ->
 		callback = _.once(callback)
+		# check the redis connection by storing and retrieving a unique key/value pair
+		uniqueToken = "host=#{HOST}:pid=#{PID}:random=#{RND}:time=#{Date.now()}:count=#{COUNT++}"
 		timer = setTimeout () ->
-			error = new Error("redis client ping check timed out #{client?.options?.host}")
+			error = new Error("redis client health check timed out #{client?.options?.host}")
 			console.error {
 				err: error,
 				key: client.options?.key # only present for cluster
 				clientOptions: client.options
+				uniqueToken: uniqueToken
 			}, "client timed out"
 			callback(error)
 		, RedisSharelatex.HEARTBEAT_TIMEOUT
-		client.ping (err) ->
-			clearTimeout timer
-			callback(err)
-		
+		healthCheckKey = "_redis-wrapper:healthCheckKey:{#{uniqueToken}}"
+		healthCheckValue = "_redis-wrapper:healthCheckValue:{#{uniqueToken}}"
+		# set the unique key/value pair
+		multi = client.multi()
+		multi.set healthCheckKey, healthCheckValue, "EX", 60
+		multi.exec (err, reply) ->
+			if err?
+				clearTimeout timer
+				return callback(err)
+			# check that we can retrieve the unique key/value pair
+			multi = client.multi()
+			multi.get healthCheckKey
+			multi.del healthCheckKey
+			multi.exec (err, reply) ->
+				clearTimeout timer
+				return callback(err) if err?
+				return callback(new Error("bad response from redis health check")) if reply?[0] isnt healthCheckValue
+				return callback()
+
 	_monkeyPatchIoredisExec: (client) ->
 		_multi = client.multi
 		client.multi = (args...) ->

From d9214b6eebe00c6964c51370e2ca3251bf0513e5 Mon Sep 17 00:00:00 2001
From: Brian Gough <brian.gough@overleaf.com>
Date: Thu, 19 Sep 2019 15:41:43 +0100
Subject: [PATCH 2/2] check full reply in health check

---
 libraries/redis-wrapper/index.coffee | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/redis-wrapper/index.coffee b/libraries/redis-wrapper/index.coffee
index 674dae5594..05f5fa9fbb 100644
--- a/libraries/redis-wrapper/index.coffee
+++ b/libraries/redis-wrapper/index.coffee
@@ -80,7 +80,7 @@ module.exports = RedisSharelatex =
 			multi.exec (err, reply) ->
 				clearTimeout timer
 				return callback(err) if err?
-				return callback(new Error("bad response from redis health check")) if reply?[0] isnt healthCheckValue
+				return callback(new Error("bad response from redis health check")) if reply?[0] isnt healthCheckValue or reply?[1] isnt 1
 				return callback()
 
 	_monkeyPatchIoredisExec: (client) ->