From c7d8fbbb8ab29788385b510c97248cf2fbc2cf2d Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Tue, 9 May 2017 14:06:09 +0100 Subject: [PATCH] exponential backoff for lock up to maximum time --- services/document-updater/app/coffee/LockManager.coffee | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/document-updater/app/coffee/LockManager.coffee b/services/document-updater/app/coffee/LockManager.coffee index d237b51feb..0321a68841 100644 --- a/services/document-updater/app/coffee/LockManager.coffee +++ b/services/document-updater/app/coffee/LockManager.coffee @@ -14,6 +14,7 @@ COUNT = 0 module.exports = LockManager = LOCK_TEST_INTERVAL: 50 # 50ms between each test of the lock + MAX_TEST_INTERVAL: 1000 # back off to 1s between each test of the lock MAX_LOCK_WAIT_TIME: 10000 # 10s maximum time to spend trying to get the lock LOCK_TTL: 30 # seconds. Time until lock auto expires in redis. @@ -41,6 +42,7 @@ module.exports = LockManager = getLock: (doc_id, callback = (error, lockValue) ->) -> startTime = Date.now() + testInterval = LockManager.LOCK_TEST_INTERVAL do attempt = () -> if Date.now() - startTime > LockManager.MAX_LOCK_WAIT_TIME e = new Error("Timeout") @@ -52,7 +54,9 @@ module.exports = LockManager = if gotLock callback(null, lockValue) else - setTimeout attempt, LockManager.LOCK_TEST_INTERVAL + setTimeout attempt, testInterval + # back off when the lock is taken to avoid overloading + testInterval = Math.max(testInterval * 2, LockManager.MAX_TEST_INTERVAL) checkLock: (doc_id, callback = (err, isFree)->)-> key = keys.blockingKey(doc_id:doc_id)