From 54c0fc1180b20ce52d943459d3deadab561af304 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 11 Jul 2017 14:25:59 +0100 Subject: [PATCH] put a 5 second timeout in getting redis lock --- services/document-updater/app/coffee/LockManager.coffee | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/document-updater/app/coffee/LockManager.coffee b/services/document-updater/app/coffee/LockManager.coffee index 716cac2291..cd6df46878 100644 --- a/services/document-updater/app/coffee/LockManager.coffee +++ b/services/document-updater/app/coffee/LockManager.coffee @@ -14,6 +14,8 @@ PID = process.pid RND = crypto.randomBytes(4).toString('hex') COUNT = 0 +MAX_REDIS_REQUEST_LENGTH = 5000 # 5 seconds + module.exports = LockManager = LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock MAX_TEST_INTERVAL: 1000 # back off to 1s between each test of the lock @@ -37,8 +39,11 @@ module.exports = LockManager = return callback(err) if err? if gotLock == "OK" metrics.inc "doc-not-blocking" - profile.log("got lock").end() - callback err, true, lockValue + timeTaken = profile.log("got lock").end() + if timeTaken > MAX_REDIS_REQUEST_LENGTH + callback err, false # took too long to get the lock, bail out + else + callback err, true, lockValue else metrics.inc "doc-blocking" profile.log("doc is locked").end()