From d5a49038dfcd14e81b7b506e5717e1dd1df6a8ed Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Wed, 4 Nov 2020 08:35:37 -0500 Subject: [PATCH] Merge pull request #3302 from overleaf/em-analytics-queues Send analytics events through a queue GitOrigin-RevId: b9eb12e469faf16e32aba5fae665c5f85dfbc52c --- .../Features/Analytics/AnalyticsController.js | 36 +- .../Features/Analytics/AnalyticsManager.js | 172 ++++----- .../CollaboratorsInviteController.js | 4 +- .../Project/ProjectCreationHandler.js | 6 +- .../app/src/infrastructure/ExpressLocals.js | 3 +- .../infrastructure/FaultTolerantRequest.js | 85 ----- .../web/app/src/infrastructure/Metrics.js | 7 + services/web/app/src/infrastructure/Queues.js | 23 ++ services/web/config/settings.defaults.coffee | 8 + services/web/package-lock.json | 326 +++++++++++++----- services/web/package.json | 3 +- .../src/Analytics/AnalyticsControllerTests.js | 4 +- .../src/Analytics/AnalyticsManagerTests.js | 118 +++---- .../FaultTolerantRequestTests.js | 121 ------- 14 files changed, 420 insertions(+), 496 deletions(-) delete mode 100644 services/web/app/src/infrastructure/FaultTolerantRequest.js create mode 100644 services/web/app/src/infrastructure/Metrics.js create mode 100644 services/web/app/src/infrastructure/Queues.js delete mode 100644 services/web/test/unit/src/infrastructure/FaultTolerantRequestTests.js diff --git a/services/web/app/src/Features/Analytics/AnalyticsController.js b/services/web/app/src/Features/Analytics/AnalyticsController.js index fe0ad397bc..063bb1b64f 100644 --- a/services/web/app/src/Features/Analytics/AnalyticsController.js +++ b/services/web/app/src/Features/Analytics/AnalyticsController.js @@ -1,5 +1,5 @@ +const metrics = require('@overleaf/metrics') const AnalyticsManager = require('./AnalyticsManager') -const Errors = require('../Errors/Errors') const AuthenticationController = require('../Authentication/AuthenticationController') const InstitutionsAPI = require('../Institutions/InstitutionsAPI') const GeoIpLookup = require('../../infrastructure/GeoIpLookup') @@ -8,7 +8,7 @@ const Features = require('../../infrastructure/Features') module.exports = { updateEditingSession(req, res, next) { if (!Features.hasFeature('analytics')) { - return res.sendStatus(204) + return res.sendStatus(202) } const userId = AuthenticationController.getLoggedInUserId(req) const { projectId } = req.params @@ -16,30 +16,25 @@ module.exports = { if (userId) { GeoIpLookup.getDetails(req.ip, function(err, geoDetails) { - if (!err && geoDetails && geoDetails.country_code) { + if (err) { + metrics.inc('analytics_geo_ip_lookup_errors') + } else if (geoDetails && geoDetails.country_code) { countryCode = geoDetails.country_code } - AnalyticsManager.updateEditingSession( - userId, - projectId, - countryCode, - error => respondWith(error, res, next) - ) + AnalyticsManager.updateEditingSession(userId, projectId, countryCode) }) - } else { - res.sendStatus(204) } + res.sendStatus(202) }, recordEvent(req, res, next) { if (!Features.hasFeature('analytics')) { - return res.sendStatus(204) + return res.sendStatus(202) } const userId = AuthenticationController.getLoggedInUserId(req) || req.sessionID - AnalyticsManager.recordEvent(userId, req.params.event, req.body, error => - respondWith(error, res, next) - ) + AnalyticsManager.recordEvent(userId, req.params.event, req.body) + res.sendStatus(202) }, licences(req, res, next) { @@ -72,14 +67,3 @@ module.exports = { ) } } - -var respondWith = function(error, res, next) { - if (error instanceof Errors.ServiceNotConfiguredError) { - // ignore, no-op - res.sendStatus(204) - } else if (error) { - next(error) - } else { - res.sendStatus(204) - } -} diff --git a/services/web/app/src/Features/Analytics/AnalyticsManager.js b/services/web/app/src/Features/Analytics/AnalyticsManager.js index 743cbdca6c..67afc29b20 100644 --- a/services/web/app/src/Features/Analytics/AnalyticsManager.js +++ b/services/web/app/src/Features/Analytics/AnalyticsManager.js @@ -1,128 +1,72 @@ -const settings = require('settings-sharelatex') -const FaultTolerantRequest = require('../../infrastructure/FaultTolerantRequest') -const Errors = require('../Errors/Errors') +const Settings = require('settings-sharelatex') +const Metrics = require('../../infrastructure/Metrics') +const Queues = require('../../infrastructure/Queues') -// check that the request should be made: ignore smoke test user and ensure the -// analytics service is configured -const checkAnalyticsRequest = function(userId) { - if ( - settings.smokeTest && - settings.smokeTest.userId && - settings.smokeTest.userId.toString() === userId.toString() - ) { - // ignore smoke test user - return { error: null, skip: true } +function identifyUser(userId, oldUserId) { + if (isAnalyticsDisabled() || isSmokeTestUser(userId)) { + return } - - if (!settings.apis.analytics) { - return { - error: new Errors.ServiceNotConfiguredError( - 'Analytics service not configured' - ), - skip: true - } - } - - return { error: null, skip: false } + Metrics.analyticsQueue.inc({ status: 'adding', event_type: 'identify' }) + Queues.analytics.events + .add('identify', { userId, oldUserId }) + .then(() => { + Metrics.analyticsQueue.inc({ status: 'added', event_type: 'identify' }) + }) + .catch(() => { + Metrics.analyticsQueue.inc({ status: 'error', event_type: 'identify' }) + }) } -// prepare the request: set `fromv2` param and full URL -const prepareAnalyticsRequest = function(options) { - if (settings.overleaf != null) { - options.qs = Object.assign({}, options.qs, { fromV2: 1 }) +function recordEvent(userId, event, segmentation) { + if (isAnalyticsDisabled() || isSmokeTestUser(userId)) { + return } - - const urlPath = options.url - options.url = `${settings.apis.analytics.url}${urlPath}` - - options.timeout = options.timeout || 30000 - - return options + Metrics.analyticsQueue.inc({ status: 'adding', event_type: 'event' }) + Queues.analytics.events + .add('event', { userId, event, segmentation }) + .then(() => { + Metrics.analyticsQueue.inc({ status: 'added', event_type: 'event' }) + }) + .catch(() => { + Metrics.analyticsQueue.inc({ status: 'error', event_type: 'event' }) + }) } -// make the request to analytics after checking and preparing it. -// request happens asynchronously in the background and will be retried on error -const makeAnalyticsBackgroundRequest = function(userId, options, callback) { - let { error, skip } = checkAnalyticsRequest(userId) - if (error || skip) { - return callback(error) +function updateEditingSession(userId, projectId, countryCode) { + if (isAnalyticsDisabled() || isSmokeTestUser(userId)) { + return } - prepareAnalyticsRequest(options) + Metrics.analyticsQueue.inc({ + status: 'adding', + event_type: 'editing-session' + }) + Queues.analytics.editingSessions + .add({ userId, projectId, countryCode }) + .then(() => { + Metrics.analyticsQueue.inc({ + status: 'added', + event_type: 'editing-session' + }) + }) + .catch(() => { + Metrics.analyticsQueue.inc({ + status: 'error', + event_type: 'editing-session' + }) + }) +} - // With the tweaked parameter values (BACKOFF_BASE=3000, BACKOFF_MULTIPLIER=3): - // - the 6th attempt (maxAttempts=6) will run after 5.5 to 11.5 minutes - // - the 9th attempt (maxAttempts=9) will run after 86 to 250 minutes - options.maxAttempts = options.maxAttempts || 9 - options.backoffBase = options.backoffBase || 3000 - options.backoffMultiplier = options.backoffMultiplier || 3 +function isSmokeTestUser(userId) { + const smokeTestUserId = Settings.smokeTest && Settings.smokeTest.userId + return smokeTestUserId != null && userId.toString() === smokeTestUserId +} - FaultTolerantRequest.backgroundRequest(options, callback) +function isAnalyticsDisabled() { + return !(Settings.analytics && Settings.analytics.enabled) } module.exports = { - identifyUser(userId, oldUserId, callback) { - if (!callback) { - // callback is optional - callback = () => {} - } - - const opts = { - body: { - old_user_id: oldUserId - }, - json: true, - method: 'POST', - url: `/user/${userId}/identify` - } - makeAnalyticsBackgroundRequest(userId, opts, callback) - }, - - recordEvent(userId, event, segmentation, callback) { - if (segmentation == null) { - // segmentation is optional - segmentation = {} - } - if (!callback) { - // callback is optional - callback = () => {} - } - - const opts = { - body: { - event, - segmentation - }, - json: true, - method: 'POST', - url: `/user/${userId}/event` - } - - makeAnalyticsBackgroundRequest(userId, opts, callback) - }, - - updateEditingSession(userId, projectId, countryCode, callback) { - if (!callback) { - // callback is optional - callback = () => {} - } - - const query = { - userId, - projectId - } - - if (countryCode) { - query.countryCode = countryCode - } - - const opts = { - method: 'PUT', - url: '/editingSession', - qs: query, - maxAttempts: 6 // dont retry for too long as session ping timestamp are - // recorded when the request is received on the analytics - } - - makeAnalyticsBackgroundRequest(userId, opts, callback) - } + identifyUser, + recordEvent, + updateEditingSession } diff --git a/services/web/app/src/Features/Collaborators/CollaboratorsInviteController.js b/services/web/app/src/Features/Collaborators/CollaboratorsInviteController.js index d7a3c427d8..1b44c52b1b 100644 --- a/services/web/app/src/Features/Collaborators/CollaboratorsInviteController.js +++ b/services/web/app/src/Features/Collaborators/CollaboratorsInviteController.js @@ -24,7 +24,7 @@ const Settings = require('settings-sharelatex') const EmailHelper = require('../Helpers/EmailHelper') const EditorRealTimeController = require('../Editor/EditorRealTimeController') const NotificationsBuilder = require('../Notifications/NotificationsBuilder') -const AnalyticsManger = require('../Analytics/AnalyticsManager') +const AnalyticsManager = require('../Analytics/AnalyticsManager') const AuthenticationController = require('../Authentication/AuthenticationController') const rateLimiter = require('../../infrastructure/RateLimiter') const request = require('request') @@ -377,7 +377,7 @@ module.exports = CollaboratorsInviteController = { 'project:membership:changed', { invites: true, members: true } ) - AnalyticsManger.recordEvent(currentUser._id, 'project-invite-accept', { + AnalyticsManager.recordEvent(currentUser._id, 'project-invite-accept', { projectId, userId: currentUser._id }) diff --git a/services/web/app/src/Features/Project/ProjectCreationHandler.js b/services/web/app/src/Features/Project/ProjectCreationHandler.js index 1f06fe660f..c9212c9e82 100644 --- a/services/web/app/src/Features/Project/ProjectCreationHandler.js +++ b/services/web/app/src/Features/Project/ProjectCreationHandler.js @@ -28,7 +28,7 @@ const fs = require('fs') const Path = require('path') const { promisify } = require('util') const _ = require('underscore') -const AnalyticsManger = require('../Analytics/AnalyticsManager') +const AnalyticsManager = require('../Analytics/AnalyticsManager') const ProjectCreationHandler = { createBlankProject(owner_id, projectName, attributes, callback) { @@ -56,7 +56,7 @@ const ProjectCreationHandler = { if (error != null) { return callback(error) } - AnalyticsManger.recordEvent(owner_id, 'project-imported', { + AnalyticsManager.recordEvent(owner_id, 'project-imported', { projectId: project._id, attributes }) @@ -79,7 +79,7 @@ const ProjectCreationHandler = { if (error != null) { return callback(error) } - AnalyticsManger.recordEvent(owner_id, 'project-created', { + AnalyticsManager.recordEvent(owner_id, 'project-created', { projectId: project._id, attributes }) diff --git a/services/web/app/src/infrastructure/ExpressLocals.js b/services/web/app/src/infrastructure/ExpressLocals.js index cb33cf9e34..4424847aa7 100644 --- a/services/web/app/src/infrastructure/ExpressLocals.js +++ b/services/web/app/src/infrastructure/ExpressLocals.js @@ -245,7 +245,8 @@ module.exports = function(webRouter, privateApiRouter, publicApiRouter) { }) webRouter.use(function(req, res, next) { - res.locals.gaToken = Settings.analytics && Settings.analytics.ga.token + res.locals.gaToken = + Settings.analytics && Settings.analytics.ga && Settings.analytics.ga.token res.locals.gaOptimizeId = _.get(Settings, ['analytics', 'gaOptimize', 'id']) next() }) diff --git a/services/web/app/src/infrastructure/FaultTolerantRequest.js b/services/web/app/src/infrastructure/FaultTolerantRequest.js deleted file mode 100644 index ed5ed398d3..0000000000 --- a/services/web/app/src/infrastructure/FaultTolerantRequest.js +++ /dev/null @@ -1,85 +0,0 @@ -const logger = require('logger-sharelatex') -const request = require('requestretry') - -const isProduction = - (process.env['NODE_ENV'] || '').toLowerCase() === 'production' -const isTest = process.env['MOCHA_GREP'] !== undefined - -const BACKOFF_MAX_TRIES = 3 -const BACKOFF_BASE = 500 -const BACKOFF_MULTIPLIER = 1.5 -const BACKOFF_RANDOM_FACTOR = 0.5 - -// -// Use an exponential backoff to retry requests -// -// This is based on what the Google HTTP client does: -// https://developers.google.com/api-client-library/java/google-http-java-client/reference/1.20.0/com/google/api/client/util/ExponentialBackOff -// -let FaultTolerantRequest -module.exports = FaultTolerantRequest = { - request: function(options, callback) { - options = Object.assign( - { - maxAttempts: BACKOFF_MAX_TRIES, - backoffBase: BACKOFF_BASE, - backoffMultiplier: BACKOFF_MULTIPLIER, - backoffRandomFactor: BACKOFF_RANDOM_FACTOR - }, - options - ) - - options.delayStrategy = FaultTolerantRequest.exponentialDelayStrategy( - options.backoffBase, - options.backoffMultiplier, - options.backoffRandomFactor - ) - - request(options, callback) - }, - - backgroundRequest: function(options, callback) { - FaultTolerantRequest.request(options, function(err) { - if (err) { - return logger.err( - { err, url: options.url, query: options.qs, body: options.body }, - 'Background request failed' - ) - } - }) - - callback() // Do not wait for all the attempts - }, - - exponentialDelayStrategy: function( - backoffBase, - backoffMultiplier, - backoffRandomFactor - ) { - let backoff = backoffBase - - return function() { - const delay = exponentialDelay(backoff, backoffRandomFactor) - backoff *= backoffMultiplier - return delay - } - } -} - -function exponentialDelay(backoff, backoffRandomFactor) { - // set delay to `backoff` initially - let delay = backoff - - // adds randomness - delay *= 1 - backoffRandomFactor + 2 * Math.random() * backoffRandomFactor - - // round value as it's already in milliseconds - delay = Math.round(delay) - - // log retries in production - if (isProduction && !isTest) { - logger.warn(`Background request failed. Will try again in ${delay}ms`) - } - - return delay -} diff --git a/services/web/app/src/infrastructure/Metrics.js b/services/web/app/src/infrastructure/Metrics.js new file mode 100644 index 0000000000..426bf3f8b7 --- /dev/null +++ b/services/web/app/src/infrastructure/Metrics.js @@ -0,0 +1,7 @@ +const Metrics = require('@overleaf/metrics') + +exports.analyticsQueue = new Metrics.prom.Counter({ + name: 'analytics_queue', + help: 'Number of events sent to the analytics queue', + labelNames: ['status', 'event_type'] +}) diff --git a/services/web/app/src/infrastructure/Queues.js b/services/web/app/src/infrastructure/Queues.js new file mode 100644 index 0000000000..cf6a6e5206 --- /dev/null +++ b/services/web/app/src/infrastructure/Queues.js @@ -0,0 +1,23 @@ +const Queue = require('bull') +const Settings = require('settings-sharelatex') + +function createQueue(queueName, defaultJobOptions) { + return new Queue(queueName, { + redis: Settings.redis.queues, + defaultJobOptions: { + removeOnComplete: true, + attempts: 11, + backoff: { + type: 'exponential', + delay: 3000 + } + } + }) +} + +module.exports = { + analytics: { + events: createQueue('analytics-events'), + editingSessions: createQueue('analytics-editing-sessions') + } +} diff --git a/services/web/config/settings.defaults.coffee b/services/web/config/settings.defaults.coffee index 5aaba1510a..783f9d518b 100644 --- a/services/web/config/settings.defaults.coffee +++ b/services/web/config/settings.defaults.coffee @@ -93,6 +93,11 @@ module.exports = settings = password: process.env["REDIS_PASSWORD"] or "" maxRetriesPerRequest: parseInt(process.env["REDIS_MAX_RETRIES_PER_REQUEST"] || '20') + queues: + host: process.env['QUEUES_REDIS_HOST'] || process.env['REDIS_HOST'] || 'localhost' + port: process.env['QUEUES_REDIS_PORT'] || process.env['REDIS_PORT'] || '6379' + password: process.env['QUEUES_REDIS_PASSWORD'] || process.env['REDIS_PASSWORD'] || '' + # Service locations # ----------------- @@ -616,6 +621,9 @@ module.exports = settings = everyone: process.env['RATE_LIMIT_AUTO_COMPILE_EVERYONE'] or 100 standard: process.env['RATE_LIMIT_AUTO_COMPILE_STANDARD'] or 25 + analytics: + enabled: process.env['ANALYTICS_ENABLED'] == 'true' + # currentImage: "texlive-full:2017.1" # imageRoot: "" # without any trailing slash diff --git a/services/web/package-lock.json b/services/web/package-lock.json index 05b0aa1ba8..26941630ed 100644 --- a/services/web/package-lock.json +++ b/services/web/package-lock.json @@ -7,7 +7,7 @@ "@auth0/thumbprint": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@auth0/thumbprint/-/thumbprint-0.0.6.tgz", - "integrity": "sha1-yrEGLGwEZizmxZLUgVfsQmiuhRg=", + "integrity": "sha512-+YciWHxNUOE78T+xoXI1fMI6G1WdyyAay8ioaMZhvGOJ+lReYzj0b7mpfNr5WtjGrmtWPvPOOxh0TO+5Y2M/Hw==", "dev": true }, "@babel/cli": { @@ -2889,9 +2889,9 @@ }, "dependencies": { "@types/node": { - "version": "13.13.29", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.29.tgz", - "integrity": "sha512-WPGpyEDx4/F4Rx1p1Zar8m+JsMxpSY/wNFPlyNXWV+UzJwkYt3LQg2be/qJgpsLdVJsfxTR5ipY6rv2579jStQ==" + "version": "13.13.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.30.tgz", + "integrity": "sha512-HmqFpNzp3TSELxU/bUuRK+xzarVOAsR00hzcvM0TXrMlt/+wcSLa5q6YhTb6/cA6wqDCZLDcfd8fSL95x5h7AA==" }, "protobufjs": { "version": "6.10.1", @@ -3817,9 +3817,9 @@ } }, "@overleaf/metrics": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.3.0.tgz", - "integrity": "sha512-y+t0ZUW3WC90PrDH6KTDig4S4wRtL/0Uay+DFG2iuxFTaxYvf0CA1ebZidj2QyRZm6Yf0FNOJnSRp55fcrAadA==", + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@overleaf/metrics/-/metrics-3.4.0.tgz", + "integrity": "sha512-MsD+5d7gpoz2VyJ5AnEP1VHGyHsfya7bfkdnMgi93kS+FPcYvdpcooBXPq4jmGemEVxXhdyP9lBTJDwcdjZeiQ==", "requires": { "@google-cloud/debug-agent": "^5.1.2", "@google-cloud/profiler": "^4.0.3", @@ -12154,6 +12154,175 @@ "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", "dev": true }, + "bull": { + "version": "3.18.0", + "resolved": "https://registry.npmjs.org/bull/-/bull-3.18.0.tgz", + "integrity": "sha512-nE/BKlg1dnJ/AcOy5D1nzthcmpAKqpUVXzQ43mJfnVC8ZM7mi4ZzP3spN7745UuikzmGGsbTe9px2TbEKhR+DQ==", + "requires": { + "cron-parser": "^2.13.0", + "debuglog": "^1.0.0", + "get-port": "^5.1.1", + "ioredis": "^4.14.1", + "lodash": "^4.17.19", + "p-timeout": "^3.2.0", + "promise.prototype.finally": "^3.1.2", + "semver": "^7.3.2", + "util.promisify": "^1.0.1", + "uuid": "^8.3.0" + }, + "dependencies": { + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "requires": { + "object-keys": "^1.0.12" + } + }, + "es-abstract": { + "version": "1.17.7", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.7.tgz", + "integrity": "sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==", + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" + }, + "is-callable": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.2.tgz", + "integrity": "sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==" + }, + "is-regex": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", + "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", + "requires": { + "has-symbols": "^1.0.1" + } + }, + "is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "requires": { + "has-symbols": "^1.0.1" + } + }, + "object-inspect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", + "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==" + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, + "object.assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.1.tgz", + "integrity": "sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.0", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + }, + "dependencies": { + "es-abstract": { + "version": "1.18.0-next.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.1.tgz", + "integrity": "sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==", + "requires": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.2", + "is-negative-zero": "^2.0.0", + "is-regex": "^1.1.1", + "object-inspect": "^1.8.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.1", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + } + } + }, + "object.getownpropertydescriptors": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz", + "integrity": "sha512-Z53Oah9A3TdLoblT7VKJaTDdXdT+lQO+cNpKVnya5JDe9uLvzu1YyY1yFDFrcxrlRgWrEFH0jJtD/IbuwjcEVg==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + } + }, + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "requires": { + "p-finally": "^1.0.0" + } + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" + }, + "util.promisify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.1.tgz", + "integrity": "sha512-g9JpC/3He3bm38zsLupWryXHoEcS22YHthuPQSJdMy6KNrzIRzWqcsHzD/WUnqe45whVou4VIsPew37DoXWNrA==", + "requires": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.2", + "has-symbols": "^1.0.1", + "object.getownpropertydescriptors": "^2.1.0" + } + }, + "uuid": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.1.tgz", + "integrity": "sha512-FOmRr+FmWEIG8uhZv6C2bTgEVXsHk08kE7mPlrBbEe+c3r9pjceVPgupIfNIhc4yx55H69OXANrUaSuu9eInKg==" + } + } + }, "bunyan": { "version": "1.8.14", "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.14.tgz", @@ -13897,6 +14066,15 @@ } } }, + "cron-parser": { + "version": "2.16.3", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-2.16.3.tgz", + "integrity": "sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg==", + "requires": { + "is-nan": "^1.3.0", + "moment-timezone": "^0.5.31" + } + }, "cross-spawn": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", @@ -14493,6 +14671,11 @@ } } }, + "debuglog": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz", + "integrity": "sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI=" + }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", @@ -17375,7 +17558,7 @@ "flowstate": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/flowstate/-/flowstate-0.4.1.tgz", - "integrity": "sha1-tfu4t/wte9xbVL5GyYMJ73NvTsA=", + "integrity": "sha512-U67AgveyMwXFIiDgs6Yz/PrUNrZGLJUUMDwJ9Q0fDFTQSzyDg8Jj9YDyZIUnFZKggQZONVueK9+grp/Gxa/scw==", "dev": true, "requires": { "clone": "^1.0.2", @@ -18378,11 +18561,11 @@ } }, "gcp-metadata": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.0.tgz", - "integrity": "sha512-vQZD57cQkqIA6YPGXM/zc+PIZfNRFdukWGsGZ5+LcJzesi5xp6Gn7a02wRJi4eXPyArNMIYpPET4QMxGqtlk6Q==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.1.tgz", + "integrity": "sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw==", "requires": { - "gaxios": "^3.0.0", + "gaxios": "^4.0.0", "json-bigint": "^1.0.0" }, "dependencies": { @@ -18391,23 +18574,6 @@ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz", "integrity": "sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA==" }, - "gaxios": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.2.0.tgz", - "integrity": "sha512-+6WPeVzPvOshftpxJwRi2Ozez80tn/hdtOUag7+gajDHRJvAblKxTFSSMPtr2hmnLy7p0mvYz0rMXLBl8pSO7Q==", - "requires": { - "abort-controller": "^3.0.0", - "extend": "^3.0.2", - "https-proxy-agent": "^5.0.0", - "is-stream": "^2.0.0", - "node-fetch": "^2.3.0" - } - }, - "is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" - }, "json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -18441,6 +18607,11 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true }, + "get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==" + }, "get-stream": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", @@ -20527,6 +20698,34 @@ "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==", "dev": true }, + "is-nan": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.0.tgz", + "integrity": "sha512-z7bbREymOqt2CCaZVly8aC4ML3Xhfi0ekuOnjO2L8vKdl+CttdVoGZQhd4adMFAsxQ5VeRVwORs4tU8RH+HFtQ==", + "requires": { + "define-properties": "^1.1.3" + }, + "dependencies": { + "define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "requires": { + "object-keys": "^1.0.12" + } + }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + } + } + }, + "is-negative-zero": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.0.tgz", + "integrity": "sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE=" + }, "is-npm": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", @@ -23660,7 +23859,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "", "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", "requires": { "minimist": "0.0.8" @@ -23835,6 +24034,14 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, + "moment-timezone": { + "version": "0.5.31", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.31.tgz", + "integrity": "sha512-+GgHNg8xRhMXfEbv81iDtrVeTcWt0kWmTEY1XQK14dICTXnWJnT0dxdlPspwqF3keKMVPXwayEsk1DI0AA/jdA==", + "requires": { + "moment": ">= 2.9.0" + } + }, "mongodb": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.2.tgz", @@ -25283,8 +25490,7 @@ "object-inspect": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", - "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", - "dev": true + "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==" }, "object-is": { "version": "1.1.2", @@ -27802,7 +28008,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/promise.prototype.finally/-/promise.prototype.finally-3.1.2.tgz", "integrity": "sha512-A2HuJWl2opDH0EafgdjwEw7HysI8ff/n4lW4QEVBCUXFk9QeGecBWv0Deph0UmLe3tTNYegz8MOjsVuE6SMoJA==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.0-next.0", @@ -27813,7 +28018,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "requires": { "object-keys": "^1.0.12" } @@ -27822,7 +28026,6 @@ "version": "1.17.6", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", - "dev": true, "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", @@ -27841,7 +28044,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, "requires": { "is-callable": "^1.1.4", "is-date-object": "^1.0.1", @@ -27852,7 +28054,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -27860,20 +28061,17 @@ "has-symbols": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", - "dev": true + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" }, "is-callable": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.1.tgz", - "integrity": "sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg==", - "dev": true + "integrity": "sha512-wliAfSzx6V+6WfMOmus1xy0XvSgf/dlStkvTfq7F0g4bOIW0PSUbnyse3NhDwdyYS1ozfUtAAySqTws3z9Eqgg==" }, "is-regex": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", "integrity": "sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==", - "dev": true, "requires": { "has-symbols": "^1.0.1" } @@ -27882,7 +28080,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", - "dev": true, "requires": { "has-symbols": "^1.0.1" } @@ -27890,8 +28087,7 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" } } }, @@ -33366,7 +33562,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5" @@ -33376,7 +33571,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "requires": { "object-keys": "^1.0.12" } @@ -33385,7 +33579,6 @@ "version": "1.17.5", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", - "dev": true, "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", @@ -33404,7 +33597,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, "requires": { "is-callable": "^1.1.4", "is-date-object": "^1.0.1", @@ -33415,7 +33607,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -33423,20 +33614,17 @@ "has-symbols": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", - "dev": true + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" }, "is-callable": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", - "dev": true + "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==" }, "is-regex": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", - "dev": true, "requires": { "has": "^1.0.3" } @@ -33445,7 +33633,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", - "dev": true, "requires": { "has-symbols": "^1.0.1" } @@ -33453,14 +33640,12 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, "string.prototype.trimleft": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5", @@ -33471,7 +33656,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5", @@ -33538,7 +33722,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5" @@ -33548,7 +33731,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "dev": true, "requires": { "object-keys": "^1.0.12" } @@ -33557,7 +33739,6 @@ "version": "1.17.5", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", - "dev": true, "requires": { "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", @@ -33576,7 +33757,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", - "dev": true, "requires": { "is-callable": "^1.1.4", "is-date-object": "^1.0.1", @@ -33587,7 +33767,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -33595,20 +33774,17 @@ "has-symbols": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", - "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", - "dev": true + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==" }, "is-callable": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", - "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", - "dev": true + "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==" }, "is-regex": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", - "dev": true, "requires": { "has": "^1.0.3" } @@ -33617,7 +33793,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", - "dev": true, "requires": { "has-symbols": "^1.0.1" } @@ -33625,14 +33800,12 @@ "object-keys": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, "string.prototype.trimleft": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5", @@ -33643,7 +33816,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", - "dev": true, "requires": { "define-properties": "^1.1.3", "es-abstract": "^1.17.5", @@ -35690,7 +35862,7 @@ "utils-flatten": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/utils-flatten/-/utils-flatten-1.0.0.tgz", - "integrity": "sha1-AfMNMZO+RkxAsxdV5nQNDbDO8kM=", + "integrity": "sha512-s21PUgUZ+XPvH8Wi8aj2FEqzZWeNEdemP7LB4u8u5wTDRO4xB+7czAYd3FY2O2rnu89U//khR0Ce8ka3//6M0w==", "dev": true }, "utils-merge": { @@ -37651,7 +37823,7 @@ "xml-name-validator": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-2.0.1.tgz", - "integrity": "sha1-TYuPHszTQZqjYgYb7O9RXh5VljU=", + "integrity": "sha512-jRKe/iQYMyVJpzPH+3HL97Lgu5HrCfii+qSo+TfjKHtOnvbnvdVfMYrn9Q34YV81M2e5sviJlI6Ko9y+nByzvA==", "dev": true }, "xml2js": { diff --git a/services/web/package.json b/services/web/package.json index bb8c29f90a..333add4749 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -41,7 +41,7 @@ "@babel/core": "^7.9.0", "@babel/preset-env": "^7.9.5", "@babel/preset-react": "^7.9.4", - "@overleaf/metrics": "^3.3.0", + "@overleaf/metrics": "^3.4.0", "@overleaf/o-error": "^3.0.0", "@pollyjs/adapter-node-http": "^4.2.1", "@pollyjs/core": "^4.2.1", @@ -58,6 +58,7 @@ "bcrypt": "^5.0.0", "body-parser": "^1.19.0", "bufferedstream": "1.6.0", + "bull": "^3.18.0", "celebrate": "^10.0.1", "classnames": "^2.2.6", "codemirror": "^5.33.0", diff --git a/services/web/test/unit/src/Analytics/AnalyticsControllerTests.js b/services/web/test/unit/src/Analytics/AnalyticsControllerTests.js index 292687f0a7..d215047b0b 100644 --- a/services/web/test/unit/src/Analytics/AnalyticsControllerTests.js +++ b/services/web/test/unit/src/Analytics/AnalyticsControllerTests.js @@ -11,8 +11,8 @@ describe('AnalyticsController', function() { this.AuthenticationController = { getLoggedInUserId: sinon.stub() } this.AnalyticsManager = { - updateEditingSession: sinon.stub().callsArgWith(3), - recordEvent: sinon.stub().callsArgWith(3) + updateEditingSession: sinon.stub(), + recordEvent: sinon.stub() } this.InstitutionsAPI = { diff --git a/services/web/test/unit/src/Analytics/AnalyticsManagerTests.js b/services/web/test/unit/src/Analytics/AnalyticsManagerTests.js index eb2cf40894..b44dae4a07 100644 --- a/services/web/test/unit/src/Analytics/AnalyticsManagerTests.js +++ b/services/web/test/unit/src/Analytics/AnalyticsManagerTests.js @@ -1,106 +1,96 @@ const SandboxedModule = require('sandboxed-module') const path = require('path') -const modulePath = path.join( +const sinon = require('sinon') + +const MODULE_PATH = path.join( __dirname, '../../../../app/src/Features/Analytics/AnalyticsManager' ) -const sinon = require('sinon') -const { expect } = require('chai') -const Errors = require('../../../../app/src/Features/Errors/Errors') describe('AnalyticsManager', function() { beforeEach(function() { this.fakeUserId = '123abc' - this.settings = { - overleaf: true, - apis: { analytics: { url: 'analytics.test' } } + this.Settings = { + analytics: { enabled: true } + } + + this.Queues = { + analytics: { + events: { + add: sinon.stub().resolves() + }, + editingSessions: { + add: sinon.stub().resolves() + } + } } this.backgroundRequest = sinon.stub().yields() this.request = sinon.stub().yields() - this.AnalyticsManager = SandboxedModule.require(modulePath, { + this.AnalyticsManager = SandboxedModule.require(MODULE_PATH, { globals: { console: console }, requires: { - 'settings-sharelatex': this.settings, - '../../infrastructure/FaultTolerantRequest': { - backgroundRequest: this.backgroundRequest - }, - '../Errors/Errors': Errors, - request: this.request, + 'settings-sharelatex': this.Settings, 'logger-sharelatex': { - log() {} - } + warn() {} + }, + '../../infrastructure/Queues': this.Queues } }) }) - describe('checkAnalyticsRequest', function() { - it('ignores smoke test user', function(done) { - this.settings.smokeTest = { userId: this.fakeUserId } - this.AnalyticsManager.identifyUser(this.fakeUserId, '', error => { - expect(error).to.not.exist - sinon.assert.notCalled(this.request) - done() - }) + describe('ignores when', function() { + it('user is smoke test user', function() { + this.Settings.smokeTest = { userId: this.fakeUserId } + this.AnalyticsManager.identifyUser(this.fakeUserId, '') + sinon.assert.notCalled(this.Queues.analytics.events.add) }) - it('return error if analytics service is not configured', function(done) { - this.settings.apis.analytics = null - this.AnalyticsManager.identifyUser(this.fakeUserId, '', error => { - expect(error).to.be.instanceof(Errors.ServiceNotConfiguredError) - sinon.assert.notCalled(this.request) - done() - }) + it('analytics service is disabled', function() { + this.Settings.analytics.enabled = false + this.AnalyticsManager.identifyUser(this.fakeUserId, '') + sinon.assert.notCalled(this.Queues.analytics.events.add) }) }) - describe('makes correct request to analytics', function() { - it('identifyUser', function(done) { + describe('queues the appropriate message for', function() { + it('identifyUser', function() { const oldUserId = '456def' - this.AnalyticsManager.identifyUser(this.fakeUserId, oldUserId, error => { - expect(error).to.not.exist - sinon.assert.calledWithMatch(this.backgroundRequest, { - body: { old_user_id: oldUserId }, - url: 'analytics.test/user/123abc/identify' - }) - done() - }) + this.AnalyticsManager.identifyUser(this.fakeUserId, oldUserId) + sinon.assert.calledWithMatch( + this.Queues.analytics.events.add, + 'identify', + { + userId: this.fakeUserId, + oldUserId + } + ) }) - it('recordEvent', function(done) { + it('recordEvent', function() { const event = 'fake-event' - this.AnalyticsManager.recordEvent(this.fakeUserId, event, null, error => { - expect(error).to.not.exist - sinon.assert.calledWithMatch(this.backgroundRequest, { - body: { event }, - qs: { fromV2: 1 }, - url: 'analytics.test/user/123abc/event', - timeout: 30000, - maxAttempts: 9, - backoffBase: 3000, - backoffMultiplier: 3 - }) - done() + this.AnalyticsManager.recordEvent(this.fakeUserId, event, null) + sinon.assert.calledWithMatch(this.Queues.analytics.events.add, 'event', { + event, + userId: this.fakeUserId, + segmentation: null }) }) - it('updateEditingSession', function(done) { + it('updateEditingSession', function() { const projectId = '789ghi' const countryCode = 'fr' this.AnalyticsManager.updateEditingSession( this.fakeUserId, projectId, - countryCode, - error => { - expect(error).to.not.exist - sinon.assert.calledWithMatch(this.backgroundRequest, { - qs: { userId: this.fakeUserId, projectId, countryCode, fromV2: 1 }, - url: 'analytics.test/editingSession' - }) - done() - } + countryCode ) + sinon.assert.calledWithMatch(this.Queues.analytics.editingSessions.add, { + userId: this.fakeUserId, + projectId, + countryCode + }) }) }) }) diff --git a/services/web/test/unit/src/infrastructure/FaultTolerantRequestTests.js b/services/web/test/unit/src/infrastructure/FaultTolerantRequestTests.js deleted file mode 100644 index 5f22bf876b..0000000000 --- a/services/web/test/unit/src/infrastructure/FaultTolerantRequestTests.js +++ /dev/null @@ -1,121 +0,0 @@ -const SandboxedModule = require('sandboxed-module') -const path = require('path') -const modulePath = path.join( - __dirname, - '../../../../app/src/infrastructure/FaultTolerantRequest' -) -const sinon = require('sinon') -const { expect } = require('chai') - -describe('FaultTolerantRequest', function() { - beforeEach(function() { - this.request = sinon.stub().yields() - this.logger = { - err: sinon.stub() - } - this.FaultTolerantRequest = SandboxedModule.require(modulePath, { - globals: { - console: console - }, - requires: { - requestretry: this.request, - 'logger-sharelatex': this.logger - } - }) - }) - - describe('exponentialBackoffStrategy', function() { - it('returns delays within expected range with default options', function() { - this.FaultTolerantRequest.request({}, () => {}) - const delayStrategy = this.request.lastCall.args[0].delayStrategy - expect(delayStrategy()).to.be.closeTo(500, 250) // attempt 1 - expect(delayStrategy()).to.be.closeTo(750, 375) // attempt 2 - expect(delayStrategy()).to.be.closeTo(1125, 563) // attempt 3 - expect(delayStrategy()).to.be.closeTo(1688, 844) // attempt 4 - expect(delayStrategy()).to.be.closeTo(2531, 1266) // attempt 5 - expect(delayStrategy()).to.be.closeTo(3797, 1899) // attempt 6 - expect(delayStrategy()).to.be.closeTo(5695, 2848) // attempt 7 - expect(delayStrategy()).to.be.closeTo(8543, 4272) // attempt 8 - expect(delayStrategy()).to.be.closeTo(12814, 6408) // attempt 9 - expect(delayStrategy()).to.be.closeTo(19222, 9610) // attempt 10 - }) - - it('returns delays within expected range with custom options', function() { - const delayStrategy = this.FaultTolerantRequest.exponentialDelayStrategy( - 3000, - 3, - 0.5 - ) - expect(delayStrategy()).to.be.closeTo(3000, 1500) // attempt 1 - expect(delayStrategy()).to.be.closeTo(9000, 4500) // attempt 2 - expect(delayStrategy()).to.be.closeTo(27000, 13500) // attempt 3 - expect(delayStrategy()).to.be.closeTo(81000, 40500) // attempt 4 - expect(delayStrategy()).to.be.closeTo(243000, 121500) // attempt 5 - expect(delayStrategy()).to.be.closeTo(729000, 364500) // attempt 6 - expect(delayStrategy()).to.be.closeTo(2187000, 1093500) // attempt 7 - expect(delayStrategy()).to.be.closeTo(6561000, 3280500) // attempt 8 - expect(delayStrategy()).to.be.closeTo(19683000, 9841500) // attempt 9 - expect(delayStrategy()).to.be.closeTo(59049000, 29524500) // attempt 10 - }) - }) - - describe('request', function() { - it('sets retry options', function(done) { - this.FaultTolerantRequest.request({}, error => { - expect(error).to.not.exist - sinon.assert.calledOnce(this.request) - - const { delayStrategy, maxAttempts } = this.request.lastCall.args[0] - expect(delayStrategy).to.be.a('function') - expect(maxAttempts).to.be.a('number') - done() - }) - }) - - it("don't overwrite retry options", function(done) { - const customMaxAttempts = Math.random() - const customBase = Math.random() - const customMultiplier = Math.random() - const customRandomFactor = Math.random() - this.FaultTolerantRequest.request( - { - maxAttempts: customMaxAttempts, - backoffBase: customBase, - backoffMultiplier: customMultiplier, - backoffRandomFactor: customRandomFactor - }, - error => { - expect(error).to.not.exist - - const { - maxAttempts, - backoffBase, - backoffMultiplier, - backoffRandomFactor - } = this.request.lastCall.args[0] - expect(maxAttempts).to.equal(customMaxAttempts) - expect(backoffBase).to.equal(customBase) - expect(backoffMultiplier).to.equal(customMultiplier) - expect(backoffRandomFactor).to.equal(customRandomFactor) - done() - } - ) - }) - }) - - describe('backgroundRequest', function() { - it('logs error in the background', function(done) { - this.request.yields(new Error('Nope')) - this.logger.err = (options, message) => { - expect(options.url).to.equal('test.url') - done() - } - this.FaultTolerantRequest.backgroundRequest( - { url: 'test.url' }, - error => { - expect(error).to.not.exist - } - ) - }) - }) -})