Merge pull request #9116 from overleaf/ab-script-group-membership-sync

[web] Script & cron job to synchronize group subscriptions memberships in BQ

GitOrigin-RevId: 0180f7586eb520f37d4d600bcb8d3eeea36a538a
This commit is contained in:
Alexandre Bourdin 2022-12-20 17:09:28 +01:00 committed by Copybot
parent efd30fae15
commit 971dabb1f8
4 changed files with 769 additions and 10 deletions

507
package-lock.json generated
View file

@ -25881,9 +25881,9 @@
} }
}, },
"node_modules/node-forge": { "node_modules/node-forge": {
"version": "1.2.1", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz",
"integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==", "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==",
"engines": { "engines": {
"node": ">= 6.13.0" "node": ">= 6.13.0"
} }
@ -38918,6 +38918,7 @@
"@codemirror/view": "^6.7.1", "@codemirror/view": "^6.7.1",
"@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-html-renderer": "^16.0.2",
"@contentful/rich-text-types": "^16.0.2", "@contentful/rich-text-types": "^16.0.2",
"@google-cloud/bigquery": "^6.0.1",
"@lezer/common": "^1.0.2", "@lezer/common": "^1.0.2",
"@lezer/highlight": "^1.1.3", "@lezer/highlight": "^1.1.3",
"@lezer/lr": "^1.2.5", "@lezer/lr": "^1.2.5",
@ -39170,6 +39171,97 @@
"worker-loader": "^3.0.8" "worker-loader": "^3.0.8"
} }
}, },
"services/web/node_modules/@google-cloud/bigquery": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz",
"integrity": "sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q==",
"dependencies": {
"@google-cloud/common": "^4.0.0",
"@google-cloud/paginator": "^4.0.0",
"@google-cloud/promisify": "^3.0.0",
"arrify": "^2.0.1",
"big.js": "^6.0.0",
"duplexify": "^4.0.0",
"extend": "^3.0.2",
"is": "^3.3.0",
"p-event": "^4.1.0",
"readable-stream": "^4.0.0",
"stream-events": "^1.0.5",
"uuid": "^8.0.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"services/web/node_modules/@google-cloud/bigquery/node_modules/readable-stream": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz",
"integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==",
"dependencies": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"services/web/node_modules/@google-cloud/bigquery/node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"services/web/node_modules/@google-cloud/common": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz",
"integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==",
"dependencies": {
"@google-cloud/projectify": "^3.0.0",
"@google-cloud/promisify": "^3.0.0",
"arrify": "^2.0.1",
"duplexify": "^4.1.1",
"ent": "^2.2.0",
"extend": "^3.0.2",
"google-auth-library": "^8.0.2",
"retry-request": "^5.0.0",
"teeny-request": "^8.0.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"services/web/node_modules/@google-cloud/paginator": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz",
"integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==",
"dependencies": {
"arrify": "^2.0.0",
"extend": "^3.0.2"
},
"engines": {
"node": ">=12.0.0"
}
},
"services/web/node_modules/@google-cloud/projectify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz",
"integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA==",
"engines": {
"node": ">=12.0.0"
}
},
"services/web/node_modules/@google-cloud/promisify": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz",
"integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA==",
"engines": {
"node": ">=12"
}
},
"services/web/node_modules/@hapi/address": { "services/web/node_modules/@hapi/address": {
"version": "2.1.4", "version": "2.1.4",
"resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@ -39263,6 +39355,14 @@
"lodash": "^4.17.15" "lodash": "^4.17.15"
} }
}, },
"services/web/node_modules/@tootallnate/once": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
"integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
"engines": {
"node": ">= 10"
}
},
"services/web/node_modules/@uppy/utils": { "services/web/node_modules/@uppy/utils": {
"version": "4.0.7", "version": "4.0.7",
"resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz", "resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz",
@ -39331,6 +39431,29 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}, },
"services/web/node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"services/web/node_modules/busboy": { "services/web/node_modules/busboy": {
"version": "1.6.0", "version": "1.6.0",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@ -39803,6 +39926,17 @@
"node": ">=0.3.1" "node": ">=0.3.1"
} }
}, },
"services/web/node_modules/duplexify": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz",
"integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==",
"dependencies": {
"end-of-stream": "^1.4.1",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1",
"stream-shift": "^1.0.0"
}
},
"services/web/node_modules/emoji-regex": { "services/web/node_modules/emoji-regex": {
"version": "9.2.2", "version": "9.2.2",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
@ -40006,6 +40140,32 @@
"universalify": "^0.1.0" "universalify": "^0.1.0"
} }
}, },
"services/web/node_modules/gaxios": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz",
"integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==",
"dependencies": {
"extend": "^3.0.2",
"https-proxy-agent": "^5.0.0",
"is-stream": "^2.0.0",
"node-fetch": "^2.6.7"
},
"engines": {
"node": ">=12"
}
},
"services/web/node_modules/gcp-metadata": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz",
"integrity": "sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g==",
"dependencies": {
"gaxios": "^5.0.0",
"json-bigint": "^1.0.0"
},
"engines": {
"node": ">=12"
}
},
"services/web/node_modules/glob-parent": { "services/web/node_modules/glob-parent": {
"version": "6.0.2", "version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@ -40018,6 +40178,63 @@
"node": ">=10.13.0" "node": ">=10.13.0"
} }
}, },
"services/web/node_modules/google-auth-library": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz",
"integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==",
"dependencies": {
"arrify": "^2.0.0",
"base64-js": "^1.3.0",
"ecdsa-sig-formatter": "^1.0.11",
"fast-text-encoding": "^1.0.0",
"gaxios": "^5.0.0",
"gcp-metadata": "^5.0.0",
"gtoken": "^6.1.0",
"jws": "^4.0.0",
"lru-cache": "^6.0.0"
},
"engines": {
"node": ">=12"
}
},
"services/web/node_modules/google-auth-library/node_modules/lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dependencies": {
"yallist": "^4.0.0"
},
"engines": {
"node": ">=10"
}
},
"services/web/node_modules/google-p12-pem": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz",
"integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==",
"dependencies": {
"node-forge": "^1.3.1"
},
"bin": {
"gp12-pem": "build/src/bin/gp12-pem.js"
},
"engines": {
"node": ">=12.0.0"
}
},
"services/web/node_modules/gtoken": {
"version": "6.1.2",
"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz",
"integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==",
"dependencies": {
"gaxios": "^5.0.1",
"google-p12-pem": "^4.0.0",
"jws": "^4.0.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"services/web/node_modules/http-errors": { "services/web/node_modules/http-errors": {
"version": "1.7.2", "version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@ -40033,6 +40250,19 @@
"node": ">= 0.6" "node": ">= 0.6"
} }
}, },
"services/web/node_modules/http-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
"integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
"dependencies": {
"@tootallnate/once": "2",
"agent-base": "6",
"debug": "4"
},
"engines": {
"node": ">= 6"
}
},
"services/web/node_modules/icss-utils": { "services/web/node_modules/icss-utils": {
"version": "5.1.0", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz",
@ -40045,6 +40275,25 @@
"postcss": "^8.1.0" "postcss": "^8.1.0"
} }
}, },
"services/web/node_modules/ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
]
},
"services/web/node_modules/inherits": { "services/web/node_modules/inherits": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
@ -41009,6 +41258,18 @@
"url": "https://github.com/sponsors/ljharb" "url": "https://github.com/sponsors/ljharb"
} }
}, },
"services/web/node_modules/retry-request": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz",
"integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==",
"dependencies": {
"debug": "^4.1.1",
"extend": "^3.0.2"
},
"engines": {
"node": ">=12"
}
},
"services/web/node_modules/sandboxed-module": { "services/web/node_modules/sandboxed-module": {
"version": "2.3.0", "version": "2.3.0",
"resolved": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz", "resolved": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz",
@ -41215,6 +41476,29 @@
"node": ">=10.13.0" "node": ">=10.13.0"
} }
}, },
"services/web/node_modules/teeny-request": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz",
"integrity": "sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg==",
"dependencies": {
"http-proxy-agent": "^5.0.0",
"https-proxy-agent": "^5.0.0",
"node-fetch": "^2.6.1",
"stream-events": "^1.0.5",
"uuid": "^9.0.0"
},
"engines": {
"node": ">=12"
}
},
"services/web/node_modules/teeny-request/node_modules/uuid": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
"integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"services/web/node_modules/toidentifier": { "services/web/node_modules/toidentifier": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@ -41271,8 +41555,7 @@
"services/web/node_modules/yallist": { "services/web/node_modules/yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
"dev": true
} }
}, },
"dependencies": { "dependencies": {
@ -49185,6 +49468,7 @@
"@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-html-renderer": "^16.0.2",
"@contentful/rich-text-types": "^16.0.2", "@contentful/rich-text-types": "^16.0.2",
"@cypress/react": "^6.2.0", "@cypress/react": "^6.2.0",
"@google-cloud/bigquery": "^6.0.1",
"@juggle/resize-observer": "^3.3.1", "@juggle/resize-observer": "^3.3.1",
"@lezer/common": "^1.0.2", "@lezer/common": "^1.0.2",
"@lezer/generator": "^1.1.3", "@lezer/generator": "^1.1.3",
@ -49431,6 +49715,78 @@
"yup": "^0.32.11" "yup": "^0.32.11"
}, },
"dependencies": { "dependencies": {
"@google-cloud/bigquery": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-6.0.3.tgz",
"integrity": "sha512-BP464228S9dqDCb4dR99h9D8+N498YZi/AZvoOJUaieg2H6qbiYBE1xlYuaMvyV1WEQT/2/yZTCJnCo5WiaY0Q==",
"requires": {
"@google-cloud/common": "^4.0.0",
"@google-cloud/paginator": "^4.0.0",
"@google-cloud/promisify": "^3.0.0",
"arrify": "^2.0.1",
"big.js": "^6.0.0",
"duplexify": "^4.0.0",
"extend": "^3.0.2",
"is": "^3.3.0",
"p-event": "^4.1.0",
"readable-stream": "^4.0.0",
"stream-events": "^1.0.5",
"uuid": "^8.0.0"
},
"dependencies": {
"readable-stream": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.2.0.tgz",
"integrity": "sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==",
"requires": {
"abort-controller": "^3.0.0",
"buffer": "^6.0.3",
"events": "^3.3.0",
"process": "^0.11.10"
}
},
"uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
}
}
},
"@google-cloud/common": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-4.0.3.tgz",
"integrity": "sha512-fUoMo5b8iAKbrYpneIRV3z95AlxVJPrjpevxs4SKoclngWZvTXBSGpNisF5+x5m+oNGve7jfB1e6vNBZBUs7Fw==",
"requires": {
"@google-cloud/projectify": "^3.0.0",
"@google-cloud/promisify": "^3.0.0",
"arrify": "^2.0.1",
"duplexify": "^4.1.1",
"ent": "^2.2.0",
"extend": "^3.0.2",
"google-auth-library": "^8.0.2",
"retry-request": "^5.0.0",
"teeny-request": "^8.0.0"
}
},
"@google-cloud/paginator": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-4.0.1.tgz",
"integrity": "sha512-6G1ui6bWhNyHjmbYwavdN7mpVPRBtyDg/bfqBTAlwr413On2TnFNfDxc9UhTJctkgoCDgQXEKiRPLPR9USlkbQ==",
"requires": {
"arrify": "^2.0.0",
"extend": "^3.0.2"
}
},
"@google-cloud/projectify": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-3.0.0.tgz",
"integrity": "sha512-HRkZsNmjScY6Li8/kb70wjGlDDyLkVk3KvoEo9uIoxSjYLJasGiCch9+PqRVDOCGUFvEIqyogl+BeqILL4OJHA=="
},
"@google-cloud/promisify": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-3.0.1.tgz",
"integrity": "sha512-z1CjRjtQyBOYL+5Qr9DdYIfrdLBe746jRTYfaYU6MeXkqp7UfYs/jX16lFFVzZ7PGEJvqZNqYUEtb1mvDww4pA=="
},
"@hapi/address": { "@hapi/address": {
"version": "2.1.4", "version": "2.1.4",
"resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@ -49503,6 +49859,11 @@
"lodash": "^4.17.15" "lodash": "^4.17.15"
} }
}, },
"@tootallnate/once": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
"integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A=="
},
"@uppy/utils": { "@uppy/utils": {
"version": "4.0.7", "version": "4.0.7",
"resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz", "resolved": "https://registry.npmjs.org/@uppy/utils/-/utils-4.0.7.tgz",
@ -49558,6 +49919,15 @@
} }
} }
}, },
"buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
"integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
"requires": {
"base64-js": "^1.3.1",
"ieee754": "^1.2.1"
}
},
"busboy": { "busboy": {
"version": "1.6.0", "version": "1.6.0",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@ -49888,6 +50258,17 @@
"integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==",
"dev": true "dev": true
}, },
"duplexify": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.2.tgz",
"integrity": "sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw==",
"requires": {
"end-of-stream": "^1.4.1",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1",
"stream-shift": "^1.0.0"
}
},
"emoji-regex": { "emoji-regex": {
"version": "9.2.2", "version": "9.2.2",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
@ -50040,6 +50421,26 @@
"universalify": "^0.1.0" "universalify": "^0.1.0"
} }
}, },
"gaxios": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz",
"integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==",
"requires": {
"extend": "^3.0.2",
"https-proxy-agent": "^5.0.0",
"is-stream": "^2.0.0",
"node-fetch": "^2.6.7"
}
},
"gcp-metadata": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.0.1.tgz",
"integrity": "sha512-jiRJ+Fk7e8FH68Z6TLaqwea307OktJpDjmYnU7/li6ziwvVvU2RlrCyQo5vkdeP94chm0kcSCOOszvmuaioq3g==",
"requires": {
"gaxios": "^5.0.0",
"json-bigint": "^1.0.0"
}
},
"glob-parent": { "glob-parent": {
"version": "6.0.2", "version": "6.0.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@ -50049,6 +50450,50 @@
"is-glob": "^4.0.3" "is-glob": "^4.0.3"
} }
}, },
"google-auth-library": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz",
"integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==",
"requires": {
"arrify": "^2.0.0",
"base64-js": "^1.3.0",
"ecdsa-sig-formatter": "^1.0.11",
"fast-text-encoding": "^1.0.0",
"gaxios": "^5.0.0",
"gcp-metadata": "^5.0.0",
"gtoken": "^6.1.0",
"jws": "^4.0.0",
"lru-cache": "^6.0.0"
},
"dependencies": {
"lru-cache": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"requires": {
"yallist": "^4.0.0"
}
}
}
},
"google-p12-pem": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz",
"integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==",
"requires": {
"node-forge": "^1.3.1"
}
},
"gtoken": {
"version": "6.1.2",
"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz",
"integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==",
"requires": {
"gaxios": "^5.0.1",
"google-p12-pem": "^4.0.0",
"jws": "^4.0.0"
}
},
"http-errors": { "http-errors": {
"version": "1.7.2", "version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
@ -50061,6 +50506,16 @@
"toidentifier": "1.0.0" "toidentifier": "1.0.0"
} }
}, },
"http-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
"integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
"requires": {
"@tootallnate/once": "2",
"agent-base": "6",
"debug": "4"
}
},
"icss-utils": { "icss-utils": {
"version": "5.1.0", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz",
@ -50068,6 +50523,11 @@
"dev": true, "dev": true,
"requires": {} "requires": {}
}, },
"ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
},
"inherits": { "inherits": {
"version": "2.0.3", "version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
@ -50707,6 +51167,15 @@
"path-parse": "^1.0.6" "path-parse": "^1.0.6"
} }
}, },
"retry-request": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/retry-request/-/retry-request-5.0.2.tgz",
"integrity": "sha512-wfI3pk7EE80lCIXprqh7ym48IHYdwmAAzESdbU8Q9l7pnRCk9LEhpbOTNKjz6FARLm/Bl5m+4F0ABxOkYUujSQ==",
"requires": {
"debug": "^4.1.1",
"extend": "^3.0.2"
}
},
"sandboxed-module": { "sandboxed-module": {
"version": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz", "version": "https://github.com/overleaf/node-sandboxed-module/archive/cafa2d60f17ce75cc023e6f296eb8de79d92d35d.tar.gz",
"integrity": "sha512-8h+3BGQj0Xou2mRYrRpFvDQ5Ne0/obZH9X27m/5zVMrvckNQazn1YPab7OE45VMzOI25iE9e0602QY2QmEnANQ==", "integrity": "sha512-8h+3BGQj0Xou2mRYrRpFvDQ5Ne0/obZH9X27m/5zVMrvckNQazn1YPab7OE45VMzOI25iE9e0602QY2QmEnANQ==",
@ -50873,6 +51342,25 @@
"stable": "^0.1.8" "stable": "^0.1.8"
} }
}, },
"teeny-request": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-8.0.2.tgz",
"integrity": "sha512-34pe0a4zASseXZCKdeTiIZqSKA8ETHb1EwItZr01PAR3CLPojeAKgSjzeNS4373gi59hNulyDrPKEbh2zO9sCg==",
"requires": {
"http-proxy-agent": "^5.0.0",
"https-proxy-agent": "^5.0.0",
"node-fetch": "^2.6.1",
"stream-events": "^1.0.5",
"uuid": "^9.0.0"
},
"dependencies": {
"uuid": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz",
"integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg=="
}
}
},
"toidentifier": { "toidentifier": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
@ -50908,8 +51396,7 @@
"yallist": { "yallist": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
"dev": true
} }
} }
}, },
@ -65239,9 +65726,9 @@
} }
}, },
"node-forge": { "node-forge": {
"version": "1.2.1", "version": "1.3.1",
"resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.2.1.tgz", "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz",
"integrity": "sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==" "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA=="
}, },
"node-gyp-build": { "node-gyp-build": {
"version": "3.9.0", "version": "3.9.0",

View file

@ -81,6 +81,7 @@
"@codemirror/view": "^6.7.1", "@codemirror/view": "^6.7.1",
"@contentful/rich-text-html-renderer": "^16.0.2", "@contentful/rich-text-html-renderer": "^16.0.2",
"@contentful/rich-text-types": "^16.0.2", "@contentful/rich-text-types": "^16.0.2",
"@google-cloud/bigquery": "^6.0.1",
"@lezer/common": "^1.0.2", "@lezer/common": "^1.0.2",
"@lezer/highlight": "^1.1.3", "@lezer/highlight": "^1.1.3",
"@lezer/lr": "^1.2.5", "@lezer/lr": "^1.2.5",

View file

@ -0,0 +1,30 @@
const GoogleBigQuery = require('@google-cloud/bigquery').BigQuery
let dataset = null
function getDataset() {
if (!dataset) {
console.log(
'Connecting to BigQuery dataset: ',
process.env.BQ_PROJECT_ID,
process.env.BQ_DATASET
)
dataset = new GoogleBigQuery({
projectId: process.env.BQ_PROJECT_ID,
keyFilename: process.env.GCS_KEY_FILE,
}).dataset(process.env.BQ_DATASET)
}
return dataset
}
async function query(query) {
const [job] = await getDataset().createQueryJob({ query })
const [rows] = await job.getQueryResults()
return rows
}
module.exports = {
query,
}

View file

@ -0,0 +1,241 @@
const GoogleBigQueryHelper = require('./helpers/GoogleBigQueryHelper')
const { Subscription } = require('../../app/src/models/Subscription')
const { waitForDb } = require('../../app/src/infrastructure/mongodb')
const AnalyticsManager = require('../../app/src/Features/Analytics/AnalyticsManager')
const {
DeletedSubscription,
} = require('../../app/src/models/DeletedSubscription')
const minimist = require('minimist')
const _ = require('lodash')
let FETCH_LIMIT, COMMIT, VERBOSE
async function main() {
await waitForDb()
console.log('## Syncing group subscription memberships...')
const subscriptionsCount = await Subscription.count({ groupPlan: true })
const deletedSubscriptionsCount = await DeletedSubscription.count({
'subscription.groupPlan': true,
})
console.log(
`## Going to synchronize ${subscriptionsCount} subscriptions and ${deletedSubscriptionsCount} deleted subscriptions`
)
await checkActiveSubscriptions()
await checkDeletedSubscriptions()
}
async function checkActiveSubscriptions() {
let totalSubscriptionsChecked = 0
let subscriptions
do {
subscriptions = await Subscription.find(
{ groupPlan: true },
{ recurlySubscription_id: 1, member_ids: 1 }
)
.sort('_id')
.skip(totalSubscriptionsChecked)
.limit(FETCH_LIMIT)
.lean()
if (subscriptions.length) {
const groupIds = subscriptions.map(sub => sub._id)
const bigQueryGroupMemberships = await fetchBigQueryMembershipStatuses(
groupIds
)
const membershipsByGroupId = _.groupBy(
bigQueryGroupMemberships,
'group_id'
)
for (const subscription of subscriptions) {
checkSubscriptionMemberships(
subscription,
membershipsByGroupId[subscription._id.toString()]
)
}
totalSubscriptionsChecked += subscriptions.length
}
} while (subscriptions.length > 0)
}
async function checkDeletedSubscriptions() {
let totalDeletedSubscriptionsChecked = 0
let deletedSubscriptions
do {
deletedSubscriptions = (
await DeletedSubscription.find(
{ 'subscription.groupPlan': true },
{ subscription: 1 }
)
.sort('deletedAt')
.skip(totalDeletedSubscriptionsChecked)
.limit(FETCH_LIMIT)
).map(sub => sub.toObject().subscription)
if (deletedSubscriptions.length) {
const groupIds = deletedSubscriptions.map(sub => sub._id.toString())
const bigQueryGroupMemberships = await fetchBigQueryMembershipStatuses(
groupIds
)
for (const deletedSubscription of deletedSubscriptions) {
checkDeletedSubscriptionMemberships(
deletedSubscription,
_.filter(bigQueryGroupMemberships, {
group_id: deletedSubscription._id.toString(),
})
)
}
totalDeletedSubscriptionsChecked += deletedSubscriptions.length
}
} while (deletedSubscriptions.length > 0)
}
function checkSubscriptionMemberships(subscription, membershipStatuses) {
if (VERBOSE) {
console.log(
'\n###########################################################################################',
'\n# Subscription (mongo): ',
'\n# _id: \t\t\t\t',
subscription._id.toString(),
'\n# member_ids: \t\t\t',
subscription.member_ids,
'\n# recurlySubscription_id: \t',
subscription.recurlySubscription_id
)
console.log('#\n# Membership statuses found in BigQuery: ')
console.table(membershipStatuses)
}
// create missing `joined` events when membership status is missing
for (const memberId of subscription.member_ids) {
if (
!_.find(membershipStatuses, {
user_id: memberId.toString(),
is_member: true,
})
) {
sendCorrectiveEvent(memberId, 'group-subscription-joined', subscription)
}
}
// create missing `left` events if user is not a member of the group anymore
for (const { user_id: userId, is_member: isMember } of membershipStatuses) {
if (
isMember &&
!subscription.member_ids.some(sub => sub.id.toString() === userId)
) {
sendCorrectiveEvent(userId, 'group-subscription-left', subscription)
}
}
}
function checkDeletedSubscriptionMemberships(subscription, membershipStatuses) {
if (VERBOSE) {
console.log(
'\n###########################################################################################',
'\n# Deleted subscription (mongo): ',
'\n# _id: \t\t\t\t',
subscription._id.toString(),
'\n# member_ids: \t\t\t',
subscription.member_ids,
'\n# recurlySubscription_id: \t',
subscription.recurlySubscription_id
)
console.log('#\n# Membership statuses found in BigQuery: ')
console.table(membershipStatuses)
}
const updatedUserIds = new Set()
// create missing `left` events if user was a member of the group in BQ and status is not up-to-date
for (const memberId of subscription.member_ids.map(id => id.toString())) {
if (
_.find(membershipStatuses, {
user_id: memberId,
is_member: true,
})
) {
sendCorrectiveEvent(memberId, 'group-subscription-left', subscription)
updatedUserIds.add(memberId)
}
}
// for cases where the user has been removed from the subscription before it was deleted and status is not up-to-date
for (const { user_id: userId, is_member: isMember } of membershipStatuses) {
if (isMember && !updatedUserIds.has(userId)) {
sendCorrectiveEvent(userId, 'group-subscription-left', subscription)
updatedUserIds.add(userId)
}
}
}
function sendCorrectiveEvent(userId, event, subscription) {
const segmentation = {
groupId: subscription._id.toString(),
subscriptionId: subscription.recurlySubscription_id,
source: 'sync',
}
if (COMMIT) {
console.log(
`Sending event '${event}' for user ${userId} with segmentation: ${JSON.stringify(
segmentation
)}`
)
AnalyticsManager.recordEventForUser(userId, event, segmentation)
} else {
console.log(
`Dry run - would send event '${event}' for user ${userId} with segmentation: ${JSON.stringify(
segmentation
)}`
)
}
}
async function fetchBigQueryMembershipStatuses(groupIds) {
const joinedGroupIds = groupIds.map(id => `"${id}"`).join(',')
const query = `\
WITH memberships AS (
SELECT
user_id, group_id, is_member, created_at,
ROW_NUMBER() OVER(PARTITION BY group_id, user_id ORDER BY created_at DESC) AS row_number
FROM analytics.user_group_memberships
WHERE group_id IN (${joinedGroupIds})
)
SELECT
group_id,
COALESCE(user_aliases.user_id, memberships.user_id) AS user_id,
is_member,
memberships.created_at
FROM memberships
LEFT JOIN analytics.user_aliases ON memberships.user_id = user_aliases.analytics_id
WHERE row_number = 1;
`
return GoogleBigQueryHelper.query(query)
}
const setup = () => {
const argv = minimist(process.argv.slice(2))
FETCH_LIMIT = argv.fetch ? argv.fetch : 100
COMMIT = argv.commit !== undefined
VERBOSE = argv.debug !== undefined
if (!COMMIT) {
console.warn('Doing dry run without --commit')
}
if (VERBOSE) {
console.log('Running in verbose mode')
}
}
setup()
main()
.then(() => {
console.error('Done.')
process.exit(0)
})
.catch(error => {
console.error({ error })
process.exit(1)
})