mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #6785 from overleaf/em-split-tests-analytics-enabled
Add "analytics enabled" setting to split tests GitOrigin-RevId: 9ddfda9e246cac7a13361b2d3df6884212583000
This commit is contained in:
parent
95eb69c268
commit
5ded04eaea
8 changed files with 212 additions and 71 deletions
|
@ -6,6 +6,7 @@ const crypto = require('crypto')
|
|||
const _ = require('lodash')
|
||||
const { callbackify } = require('util')
|
||||
const SplitTestCache = require('./SplitTestCache')
|
||||
const { SplitTest } = require('../../models/SplitTest')
|
||||
|
||||
const DEFAULT_VARIANT = 'default'
|
||||
const ALPHA_PHASE = 'alpha'
|
||||
|
@ -107,31 +108,37 @@ async function assignInLocalsContext(
|
|||
* Get a mapping of the active split test assignments for the given user
|
||||
*/
|
||||
async function getActiveAssignmentsForUser(userId) {
|
||||
const user = await UserGetter.promises.getUser(userId, { splitTests: 1 })
|
||||
if (user == null || user.splitTests == null) {
|
||||
const user = await _getUser(userId)
|
||||
if (user == null) {
|
||||
return {}
|
||||
}
|
||||
const activeAssignments = {}
|
||||
for (const [splitTestName, assignments] of Object.entries(user.splitTests)) {
|
||||
const splitTest = await SplitTestCache.get(splitTestName)
|
||||
if (splitTest == null) {
|
||||
continue
|
||||
}
|
||||
const currentVersion = splitTest.getCurrentVersion()
|
||||
if (!currentVersion || !currentVersion.active) {
|
||||
continue
|
||||
}
|
||||
|
||||
let assignment
|
||||
if (Array.isArray(assignments)) {
|
||||
assignment = _.maxBy(assignments, 'versionNumber')
|
||||
} else {
|
||||
// Older format is a single string rather than an array of objects
|
||||
assignment = { variantName: assignments }
|
||||
const splitTests = await SplitTest.find({
|
||||
$where: 'this.versions[this.versions.length - 1].active',
|
||||
}).exec()
|
||||
const assignments = {}
|
||||
for (const splitTest of splitTests) {
|
||||
const { activeForUser, selectedVariantName, phase, versionNumber } =
|
||||
await _getAssignmentMetadata(user.analyticsId, user, splitTest)
|
||||
if (activeForUser) {
|
||||
const assignment = {
|
||||
variantName: selectedVariantName,
|
||||
versionNumber,
|
||||
phase,
|
||||
}
|
||||
const userAssignments = user.splitTests?.[splitTest.name]
|
||||
if (Array.isArray(userAssignments)) {
|
||||
const userAssignment = userAssignments.find(
|
||||
x => x.versionNumber === versionNumber
|
||||
)
|
||||
if (userAssignment) {
|
||||
assignment.assignedAt = userAssignment.assignedAt
|
||||
}
|
||||
}
|
||||
assignments[splitTest.name] = assignment
|
||||
}
|
||||
activeAssignments[splitTestName] = assignment
|
||||
}
|
||||
return activeAssignments
|
||||
return assignments
|
||||
}
|
||||
|
||||
async function _getAssignment(
|
||||
|
@ -158,8 +165,9 @@ async function _getAssignment(
|
|||
return _makeAssignment(splitTest, cachedVariant, currentVersion)
|
||||
}
|
||||
}
|
||||
const user = userId && (await _getUser(userId))
|
||||
const { activeForUser, selectedVariantName, phase, versionNumber } =
|
||||
await _getAssignmentMetadata(analyticsId, userId, splitTest)
|
||||
await _getAssignmentMetadata(analyticsId, user, splitTest)
|
||||
if (activeForUser) {
|
||||
const assignmentConfig = {
|
||||
userId,
|
||||
|
@ -181,26 +189,19 @@ async function _getAssignment(
|
|||
return DEFAULT_ASSIGNMENT
|
||||
}
|
||||
|
||||
async function _getAssignmentMetadata(analyticsId, userId, splitTest) {
|
||||
async function _getAssignmentMetadata(analyticsId, user, splitTest) {
|
||||
const currentVersion = splitTest.getCurrentVersion()
|
||||
const phase = currentVersion.phase
|
||||
if ([ALPHA_PHASE, BETA_PHASE].includes(phase)) {
|
||||
if (userId) {
|
||||
const user = await _getUser(userId)
|
||||
if (
|
||||
(phase === ALPHA_PHASE && !(user && user.alphaProgram)) ||
|
||||
(phase === BETA_PHASE && !(user && user.betaProgram))
|
||||
) {
|
||||
return {
|
||||
activeForUser: false,
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
activeForUser: false,
|
||||
}
|
||||
if (
|
||||
!user ||
|
||||
(phase === ALPHA_PHASE && !user.alphaProgram) ||
|
||||
(phase === BETA_PHASE && !user.betaProgram)
|
||||
) {
|
||||
return {
|
||||
activeForUser: false,
|
||||
}
|
||||
}
|
||||
const userId = user?._id.toString()
|
||||
const percentile = _getPercentile(
|
||||
analyticsId || userId,
|
||||
splitTest.name,
|
||||
|
|
|
@ -58,6 +58,8 @@ async function createSplitTest(name, configuration, info = {}) {
|
|||
versionNumber: 1,
|
||||
phase: configuration.phase,
|
||||
active: configuration.active,
|
||||
analyticsEnabled:
|
||||
configuration.active && configuration.analyticsEnabled,
|
||||
variants: stripedVariants,
|
||||
},
|
||||
],
|
||||
|
@ -84,6 +86,7 @@ async function updateSplitTestConfig(name, configuration) {
|
|||
versionNumber: lastVersion.versionNumber + 1,
|
||||
phase: configuration.phase,
|
||||
active: configuration.active,
|
||||
analyticsEnabled: configuration.active && configuration.analyticsEnabled,
|
||||
variants: updatedVariants,
|
||||
})
|
||||
return _saveSplitTest(splitTest)
|
||||
|
|
|
@ -60,6 +60,11 @@ const VersionSchema = new Schema(
|
|||
default: true,
|
||||
required: true,
|
||||
},
|
||||
analyticsEnabled: {
|
||||
type: Boolean,
|
||||
default: true,
|
||||
required: true,
|
||||
},
|
||||
variants: [VariantSchema],
|
||||
createdAt: {
|
||||
type: Date,
|
||||
|
|
|
@ -118,6 +118,10 @@
|
|||
background: inherit;
|
||||
left: 20px;
|
||||
}
|
||||
&:disabled + label {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
}
|
||||
|
||||
label {
|
||||
|
|
|
@ -0,0 +1,51 @@
|
|||
exports.tags = ['saas']
|
||||
|
||||
exports.migrate = async client => {
|
||||
const { db } = client
|
||||
await db.splittests.updateMany(
|
||||
{},
|
||||
{ $set: { 'versions.$[version].analyticsEnabled': true } },
|
||||
{
|
||||
arrayFilters: [
|
||||
{
|
||||
'version.active': true,
|
||||
'version.analyticsEnabled': { $exists: false },
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
await db.splittests.updateMany(
|
||||
{},
|
||||
{ $set: { 'versions.$[version].analyticsEnabled': false } },
|
||||
{
|
||||
arrayFilters: [
|
||||
{
|
||||
'version.active': false,
|
||||
'version.analyticsEnabled': { $exists: false },
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
exports.rollback = async client => {
|
||||
const { db } = client
|
||||
await db.splittests.updateMany(
|
||||
{},
|
||||
{ $unset: { 'versions.$[version].analyticsEnabled': 1 } },
|
||||
{
|
||||
arrayFilters: [
|
||||
{ 'version.active': true, 'version.analyticsEnabled': true },
|
||||
],
|
||||
}
|
||||
)
|
||||
await db.splittests.updateMany(
|
||||
{},
|
||||
{ $unset: { 'versions.$[version].analyticsEnabled': 1 } },
|
||||
{
|
||||
arrayFilters: [
|
||||
{ 'version.active': false, 'version.analyticsEnabled': false },
|
||||
],
|
||||
}
|
||||
)
|
||||
}
|
1
services/web/test/acceptance/bootstrap.js
vendored
1
services/web/test/acceptance/bootstrap.js
vendored
|
@ -3,6 +3,7 @@ chai.should()
|
|||
chai.use(require('chai-as-promised'))
|
||||
chai.use(require('chaid'))
|
||||
chai.use(require('sinon-chai'))
|
||||
chai.use(require('chai-exclude'))
|
||||
|
||||
// Do not truncate assertion errors
|
||||
chai.config.truncateThreshold = 0
|
||||
|
|
|
@ -11,12 +11,15 @@ const MODULE_PATH = Path.join(
|
|||
|
||||
describe('SplitTestHandler', function () {
|
||||
beforeEach(function () {
|
||||
this.splitTest = {
|
||||
getCurrentVersion: sinon.stub().returns({ active: true }),
|
||||
}
|
||||
this.inactiveSplitTest = {
|
||||
getCurrentVersion: sinon.stub().returns({ active: false }),
|
||||
}
|
||||
this.splitTests = [
|
||||
makeSplitTest('active-test'),
|
||||
makeSplitTest('legacy-test'),
|
||||
makeSplitTest('no-analytics-test-1', { analyticsEnabled: false }),
|
||||
makeSplitTest('no-analytics-test-2', {
|
||||
analyticsEnabled: false,
|
||||
versionNumber: 2,
|
||||
}),
|
||||
]
|
||||
|
||||
this.UserGetter = {
|
||||
promises: {
|
||||
|
@ -24,19 +27,24 @@ describe('SplitTestHandler', function () {
|
|||
},
|
||||
}
|
||||
|
||||
this.SplitTest = {
|
||||
find: sinon.stub().returns({
|
||||
exec: sinon.stub().resolves(this.splitTests),
|
||||
}),
|
||||
}
|
||||
|
||||
this.SplitTestCache = {
|
||||
get: sinon.stub().resolves(null),
|
||||
}
|
||||
this.SplitTestCache.get.withArgs('legacy-test').resolves(this.splitTest)
|
||||
this.SplitTestCache.get.withArgs('other-test').resolves(this.splitTest)
|
||||
this.SplitTestCache.get
|
||||
.withArgs('inactive-test')
|
||||
.resolves(this.inactiveSplitTest)
|
||||
for (const splitTest of this.splitTests) {
|
||||
this.SplitTestCache.get.withArgs(splitTest.name).resolves(splitTest)
|
||||
}
|
||||
|
||||
this.SplitTestHandler = SandboxedModule.require(MODULE_PATH, {
|
||||
requires: {
|
||||
'../User/UserGetter': this.UserGetter,
|
||||
'./SplitTestCache': this.SplitTestCache,
|
||||
'../../models/SplitTest': { SplitTest: this.SplitTest },
|
||||
'../User/UserUpdater': {},
|
||||
'../Analytics/AnalyticsManager': {},
|
||||
'./LocalsHelper': {},
|
||||
|
@ -49,14 +57,23 @@ describe('SplitTestHandler', function () {
|
|||
this.user = {
|
||||
_id: ObjectId(),
|
||||
splitTests: {
|
||||
'legacy-test': 'legacy-variant',
|
||||
'other-test': [
|
||||
{ variantName: 'default', versionNumber: 1 },
|
||||
{ variantName: 'latest', versionNumber: 3 },
|
||||
{ variantName: 'experiment', versionNumber: 2 },
|
||||
'active-test': [
|
||||
{
|
||||
variantName: 'default',
|
||||
versionNumber: 1,
|
||||
assignedAt: 'active-test-assigned-at',
|
||||
},
|
||||
],
|
||||
'legacy-test': 'legacy-variant',
|
||||
'inactive-test': [{ variantName: 'trythis' }],
|
||||
'unknown-test': [{ variantName: 'trythis' }],
|
||||
'no-analytics-test-2': [
|
||||
{
|
||||
variantName: 'some-variant',
|
||||
versionNumber: 1,
|
||||
assignedAt: 'no-analytics-assigned-at',
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
this.UserGetter.promises.getUser
|
||||
|
@ -69,19 +86,36 @@ describe('SplitTestHandler', function () {
|
|||
})
|
||||
|
||||
it('handles the legacy assignment format', function () {
|
||||
expect(this.assignments).to.have.property('legacy-test')
|
||||
expect(this.assignments['legacy-test'].variantName).to.equal(
|
||||
'legacy-variant'
|
||||
)
|
||||
expect(this.assignments['legacy-test']).to.deep.equal({
|
||||
variantName: 'variant-1',
|
||||
phase: 'release',
|
||||
versionNumber: 1,
|
||||
})
|
||||
})
|
||||
|
||||
it('returns the last assignment for each active test', function () {
|
||||
expect(this.assignments).to.have.property('other-test')
|
||||
expect(this.assignments['other-test'].variantName).to.equal('latest')
|
||||
it('returns the current assignment for each active test', function () {
|
||||
expect(this.assignments['active-test']).to.deep.equal({
|
||||
variantName: 'variant-1',
|
||||
phase: 'release',
|
||||
versionNumber: 1,
|
||||
assignedAt: 'active-test-assigned-at',
|
||||
})
|
||||
})
|
||||
|
||||
it('does not return assignments for inactive tests', function () {
|
||||
expect(this.assignments).not.to.have.property('inactive-test')
|
||||
it('returns the current assignment for tests with analytics disabled', function () {
|
||||
expect(this.assignments['no-analytics-test-1']).to.deep.equal({
|
||||
variantName: 'variant-1',
|
||||
phase: 'release',
|
||||
versionNumber: 1,
|
||||
})
|
||||
})
|
||||
|
||||
it('returns the current assignment for tests with analytics disabled that had previous assignments', function () {
|
||||
expect(this.assignments['no-analytics-test-2']).to.deep.equal({
|
||||
variantName: 'variant-1',
|
||||
phase: 'release',
|
||||
versionNumber: 2,
|
||||
})
|
||||
})
|
||||
|
||||
it('does not return assignments for unknown tests', function () {
|
||||
|
@ -89,7 +123,7 @@ describe('SplitTestHandler', function () {
|
|||
})
|
||||
})
|
||||
|
||||
describe('with an inexistent user', function () {
|
||||
describe('with an non-existent user', function () {
|
||||
beforeEach(async function () {
|
||||
const unknownUserId = ObjectId()
|
||||
this.assignments =
|
||||
|
@ -115,8 +149,55 @@ describe('SplitTestHandler', function () {
|
|||
)
|
||||
})
|
||||
|
||||
it('returns empty assignments', function () {
|
||||
expect(this.assignments).to.deep.equal({})
|
||||
it('returns current assignments', function () {
|
||||
expect(this.assignments).to.deep.equal({
|
||||
'active-test': {
|
||||
phase: 'release',
|
||||
variantName: 'variant-1',
|
||||
versionNumber: 1,
|
||||
},
|
||||
'legacy-test': {
|
||||
phase: 'release',
|
||||
variantName: 'variant-1',
|
||||
versionNumber: 1,
|
||||
},
|
||||
'no-analytics-test-1': {
|
||||
phase: 'release',
|
||||
variantName: 'variant-1',
|
||||
versionNumber: 1,
|
||||
},
|
||||
'no-analytics-test-2': {
|
||||
phase: 'release',
|
||||
variantName: 'variant-1',
|
||||
versionNumber: 2,
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
function makeSplitTest(name, opts = {}) {
|
||||
const {
|
||||
active = true,
|
||||
analyticsEnabled = active,
|
||||
phase = 'release',
|
||||
versionNumber = 1,
|
||||
} = opts
|
||||
|
||||
return {
|
||||
name,
|
||||
getCurrentVersion: sinon.stub().returns({
|
||||
active,
|
||||
analyticsEnabled,
|
||||
phase,
|
||||
versionNumber,
|
||||
variants: [
|
||||
{
|
||||
name: 'variant-1',
|
||||
rolloutPercent: 100,
|
||||
rolloutStripes: [{ start: 0, end: 100 }],
|
||||
},
|
||||
],
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,10 +1,5 @@
|
|||
const sinon = require('sinon')
|
||||
const sinonChai = require('sinon-chai')
|
||||
const chai = require('chai')
|
||||
const chaiAsPromised = require('chai-as-promised')
|
||||
chai.use(sinonChai)
|
||||
chai.use(chaiAsPromised)
|
||||
const { expect } = chai
|
||||
const { expect } = require('chai')
|
||||
const recurly = require('recurly')
|
||||
const modulePath = '../../../../app/src/Features/Subscription/RecurlyClient'
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
|
Loading…
Reference in a new issue