From 6609c10d36ece32e5469f0517a49bf70a4abe6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Fern=C3=A1ndez=20Capel?= Date: Thu, 15 Mar 2018 15:11:11 +0000 Subject: [PATCH 1/4] Use exponential backoff to retry request to analytics --- .../Analytics/AnalyticsManager.coffee | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee index 469183d2fa..df7cb20800 100644 --- a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee +++ b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee @@ -9,11 +9,30 @@ makeRequest = (opts, callback)-> if settings.apis?.analytics?.url? urlPath = opts.url opts.url = "#{settings.apis.analytics.url}#{urlPath}" - request opts, callback + request(opts) + callback() # Do not wait for all the attempts else callback(new Errors.ServiceNotConfiguredError('Analytics service not configured')) +# Set an exponential backoff to retry calls to analytics. First retry will +# happen after 4s, then 8, 16, 32, 64... +exponentialBackoffStrategy = () -> + attempts = 1 # This won't be called until there has been 1 failure + + () -> + attempts += 1 + exponentialBackoffDelay(attempts) + +exponentialBackoffDelay = (attempts) -> + delay = 2 ** attempts * 1000 + + logger.warn "Error comunicating with the analytics service. " + + "Will try again attempt #{attempts} in #{delay}ms" + + delay + + module.exports = identifyUser: (user_id, old_user_id, callback = (error)->)-> @@ -37,8 +56,8 @@ module.exports = method:"POST" timeout:1000 url: "/user/#{user_id}/event" - maxAttempts: 20 - retryDelay: 5000 + delayStrategy: exponentialBackoffStrategy() + maxAttempts: 7 # Give up after ~ 8min if settings.overleaf? opts.qs = {fromV2: 1} makeRequest opts, callback @@ -56,8 +75,8 @@ module.exports = timeout: 1000 url: "/editingSession" qs: query - maxAttempts: 20 - retryDelay: 5000 + maxAttempts: 6 # Give up after ~ 4min + delayStrategy: exponentialBackoffStrategy() if settings.overleaf? opts.qs.fromV2 = 1 makeRequest opts, callback From 07b0f37579468ad8192a06257daa9897861da174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Fern=C3=A1ndez=20Capel?= Date: Mon, 19 Mar 2018 09:56:44 +0000 Subject: [PATCH 2/4] Log errors in requests to analytics --- .../app/coffee/Features/Analytics/AnalyticsManager.coffee | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee index df7cb20800..b2d29014a1 100644 --- a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee +++ b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee @@ -9,7 +9,10 @@ makeRequest = (opts, callback)-> if settings.apis?.analytics?.url? urlPath = opts.url opts.url = "#{settings.apis.analytics.url}#{urlPath}" - request(opts) + request opts, (err) -> + if err? + logger.err { err: err }, 'Request to analytics failed' + callback() # Do not wait for all the attempts else callback(new Errors.ServiceNotConfiguredError('Analytics service not configured')) From f2fc999ce47e361ce2fe4949a2104c86ae3a39f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Fern=C3=A1ndez=20Capel?= Date: Mon, 19 Mar 2018 10:09:45 +0000 Subject: [PATCH 3/4] Extract method to add common params --- .../Analytics/AnalyticsManager.coffee | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee index b2d29014a1..dd15c05d24 100644 --- a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee +++ b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee @@ -5,6 +5,20 @@ request = require "requestretry" Errors = require '../Errors/Errors' +makeFaultTolerantRequest = (userId, options, callback) -> + if userId+"" == settings.smokeTest?.userId+"" + return callback() + + options = Object.assign(options, { + delayStrategy: exponentialBackoffStrategy() + timeout: 1000 + }) + + if settings.overleaf? + options.qs = Object.assign({}, options.qs, { fromV2: 1 }) + + makeRequest(options, callback) + makeRequest = (opts, callback)-> if settings.apis?.analytics?.url? urlPath = opts.url @@ -49,40 +63,33 @@ module.exports = makeRequest opts, callback recordEvent: (user_id, event, segmentation = {}, callback = (error) ->) -> - if user_id+"" == settings.smokeTest?.userId+"" - return callback() opts = body: event:event segmentation:segmentation json:true method:"POST" - timeout:1000 url: "/user/#{user_id}/event" - delayStrategy: exponentialBackoffStrategy() maxAttempts: 7 # Give up after ~ 8min - if settings.overleaf? - opts.qs = {fromV2: 1} - makeRequest opts, callback + + makeFaultTolerantRequest user_id, opts, callback + updateEditingSession: (userId, projectId, countryCode, callback = (error) ->) -> - if userId+"" == settings.smokeTest?.userId+"" - return callback() query = userId: userId projectId: projectId + if countryCode query.countryCode = countryCode + opts = method: "PUT" - timeout: 1000 url: "/editingSession" qs: query maxAttempts: 6 # Give up after ~ 4min - delayStrategy: exponentialBackoffStrategy() - if settings.overleaf? - opts.qs.fromV2 = 1 - makeRequest opts, callback + + makeFaultTolerantRequest userId, opts, callback getLastOccurrence: (user_id, event, callback = (error) ->) -> From 132b31c39fa6cfb0b6a6f39ee3b8edd418dee8f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alberto=20Fern=C3=A1ndez=20Capel?= Date: Mon, 19 Mar 2018 14:46:18 +0000 Subject: [PATCH 4/4] Increase timeout --- .../web/app/coffee/Features/Analytics/AnalyticsManager.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee index dd15c05d24..c8cafc52d5 100644 --- a/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee +++ b/services/web/app/coffee/Features/Analytics/AnalyticsManager.coffee @@ -11,7 +11,7 @@ makeFaultTolerantRequest = (userId, options, callback) -> options = Object.assign(options, { delayStrategy: exponentialBackoffStrategy() - timeout: 1000 + timeout: 30000 }) if settings.overleaf?