github
/
overleaf
mirror of https://github.com/overleaf/overleaf.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[web] Create script to update forgotten `featuresUpdatedAt` after the migration to 20s compile timeout (#18113) 

* Copy `migration_compile_timeout_60s_to_20s.js` script

* Update `featuresUpdatedAt`

* Add a comment about `featuresUpdatedAt` in migration_compile_timeout_60s_to_20s.js

* Fix test on migration_compile_timeout_60s_to_20s.js

* Fix: Include users having `featuresUpdatedAt` undefined in the update

* Add test on `migration_compile_timeout_60s_to_20s_fixup_features_updated_at`

GitOrigin-RevId: 4b2baf955a6a9f39bf9ce00b7839af551064c6cb
This commit is contained in:
Antoine Clausse 2024-04-29 11:02:57 +02:00 committed by Copybot
parent 70c05dd5f7
commit 711d50a2f1
3 changed files with 333 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
services/web/scripts/migration_compile_timeout_60s_to_20s.js
View File

@ -34,7 +34,8 @@ const main = async ({ COMMIT, SKIP_COUNT }) => {
const nModified = await batchedUpdate(
'users',
{ 'features.compileTimeout': { $lte: 60, $ne: 20 } },
{ $set: { 'features.compileTimeout': 20 } }
// NOTE: Always update featuresUpdatedAt to ensure the user's features synced with BigQuery
{ $set: { 'features.compileTimeout': 20, featuresUpdatedAt: new Date() } }
)
console.log(`Updated ${nModified} records`)
}

65
services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js Normal file
View File

@ -0,0 +1,65 @@
#!/usr/bin/env node
const minimist = require('minimist')
const {
db,
READ_PREFERENCE_SECONDARY,
waitForDb,
} = require('../app/src/infrastructure/mongodb')
const { batchedUpdate } = require('./helpers/batchedUpdate')
// A few seconds after the previous migration script was run
const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z')
const query = {
'features.compileTimeout': 20,
$or: [
{ featuresUpdatedAt: { $exists: false } },
{ featuresUpdatedAt: { $lt: FEATURES_UPDATED_AT } },
],
}
async function logCount() {
const usersToUpdate = await db.users.countDocuments(query, {
readPreference: READ_PREFERENCE_SECONDARY,
})
console.log(
`Found ${usersToUpdate} users needing their featuresUpdatedAt updated`
)
}
const main = async ({ COMMIT, SKIP_COUNT }) => {
console.time('Script Duration')
await waitForDb()
if (!SKIP_COUNT) {
await logCount()
}
if (COMMIT) {
const nModified = await batchedUpdate('users', query, {
$set: { featuresUpdatedAt: FEATURES_UPDATED_AT },
})
console.log(`Updated ${nModified} records`)
}
console.timeEnd('Script Duration')
}
const setup = () => {
const argv = minimist(process.argv.slice(2))
const COMMIT = argv.commit !== undefined
const SKIP_COUNT = argv['skip-count'] !== undefined
if (!COMMIT) {
console.warn('Doing dry run. Add --commit to commit changes')
}
return { COMMIT, SKIP_COUNT }
}
main(setup())
.catch(err => {
console.error(err)
process.exit(1)
})
.then(() => process.exit(0))

363
services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js
View File

@ -17,118 +17,287 @@ async function runScript(args = []) {
}
}
describe('MigrateUserFeatureTimeoutTests', function () {
const usersInput = {
noFeatures: {},
noFeatureTimeout: { features: {} },
timeout10s: { features: { compileTimeout: 10, other: 'val1' }, bar: '1' },
timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' },
timeout30s: { features: { compileTimeout: 30, other: 'val3' }, bar: '3' },
timeout60s: { features: { compileTimeout: 60, other: 'val4' }, bar: '4' },
timeout120s: { features: { compileTimeout: 120, other: 'val5' }, bar: '5' },
timeout180s: { features: { compileTimeout: 180, other: 'val6' }, bar: '6' },
async function runFixupScript(args = []) {
try {
return await promisify(exec)(
[
'node',
'scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js',
...args,
].join(' ')
)
} catch (error) {
logger.error({ error }, 'script failed')
throw error
}
}
const usersKeys = Object.keys(usersInput)
const userIds = {}
beforeEach('insert users', async function () {
const usersInsertedValues = await db.users.insertMany(
usersKeys.map(key => ({
...usersInput[key],
email: `${key}@example.com`,
}))
)
usersKeys.forEach(
(key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
)
})
it('gives correct counts in dry mode', async function () {
const users = await db.users.find().toArray()
expect(users).to.have.lengthOf(usersKeys.length)
const result = await runScript([])
expect(result.stderr).to.contain(
'Doing dry run. Add --commit to commit changes'
)
expect(result.stdout).to.contain(
'Found 3 users with compileTimeout <= 60s && != 20s'
)
expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s')
expect(result.stdout).not.to.contain('Updated')
const usersAfter = await db.users.find().toArray()
expect(usersAfter).to.deep.equal(users)
})
it("updates users compileTimeout when '--commit' is set", async function () {
const users = await db.users.find().toArray()
expect(users).to.have.lengthOf(usersKeys.length)
const result = await runScript(['--commit'])
expect(result.stdout).to.contain(
'Found 3 users with compileTimeout <= 60s && != 20s'
)
expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s')
expect(result.stdout).to.contain('Updated 3 records')
const usersAfter = await db.users.find().toArray()
expect(usersAfter).to.deep.equal([
{ _id: userIds.noFeatures, email: 'noFeatures@example.com' },
{
_id: userIds.noFeatureTimeout,
email: 'noFeatureTimeout@example.com',
features: {},
},
{
_id: userIds.timeout10s,
email: 'timeout10s@example.com',
features: { compileTimeout: 20, other: 'val1' },
describe('MigrateUserFeatureTimeoutTests', function () {
describe('initial script', function () {
const usersInput = {
noFeatures: {},
noFeatureTimeout: { features: {} },
timeout10s: {
features: { compileTimeout: 10, other: 'val1' },
bar: '1',
featuresUpdatedAt: new Date('2020-01-01'),
},
{
_id: userIds.timeout20s,
email: 'timeout20s@example.com',
features: { compileTimeout: 20, other: 'val2' },
bar: '2',
},
{
_id: userIds.timeout30s,
email: 'timeout30s@example.com',
features: { compileTimeout: 20, other: 'val3' },
timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' },
timeout30s: {
features: { compileTimeout: 30, other: 'val3' },
bar: '3',
featuresUpdatedAt: new Date('2025-01-01'),
},
{
_id: userIds.timeout60s,
email: 'timeout60s@example.com',
features: { compileTimeout: 20, other: 'val4' },
timeout60s: {
features: { compileTimeout: 60, other: 'val4' },
bar: '4',
featuresUpdatedAt: new Date(),
},
{
_id: userIds.timeout120s,
email: 'timeout120s@example.com',
timeout120s: {
features: { compileTimeout: 120, other: 'val5' },
bar: '5',
},
{
_id: userIds.timeout180s,
email: 'timeout180s@example.com',
timeout180s: {
features: { compileTimeout: 180, other: 'val6' },
bar: '6',
featuresUpdatedAt: new Date('2020-01-01'),
},
])
}
const result2 = await runScript([])
const usersKeys = Object.keys(usersInput)
const userIds = {}
expect(result2.stdout).to.contain(
'Found 0 users with compileTimeout <= 60s && != 20s'
)
expect(result2.stdout).to.contain(
'Found 4 users with compileTimeout == 20s'
)
beforeEach('insert users', async function () {
const usersInsertedValues = await db.users.insertMany(
usersKeys.map(key => ({
...usersInput[key],
email: `${key}@example.com`,
}))
)
usersKeys.forEach(
(key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
)
})
afterEach('clear users', async function () {
await db.users.deleteMany({})
})
it('gives correct counts in dry mode', async function () {
const users = await db.users.find().toArray()
expect(users).to.have.lengthOf(usersKeys.length)
const result = await runScript([])
expect(result.stderr).to.contain(
'Doing dry run. Add --commit to commit changes'
)
expect(result.stdout).to.contain(
'Found 3 users with compileTimeout <= 60s && != 20s'
)
expect(result.stdout).to.contain(
'Found 1 users with compileTimeout == 20s'
)
expect(result.stdout).not.to.contain('Updated')
const usersAfter = await db.users.find().toArray()
expect(usersAfter).to.deep.equal(users)
})
it("updates users compileTimeout when '--commit' is set", async function () {
const users = await db.users.find().toArray()
expect(users).to.have.lengthOf(usersKeys.length)
const result = await runScript(['--commit'])
expect(result.stdout).to.contain(
'Found 3 users with compileTimeout <= 60s && != 20s'
)
expect(result.stdout).to.contain(
'Found 1 users with compileTimeout == 20s'
)
expect(result.stdout).to.contain('Updated 3 records')
const usersAfter = await db.users.find().toArray()
expect(
usersAfter.map(({ featuresUpdatedAt, ...rest }) => rest)
).to.deep.equal([
{ _id: userIds.noFeatures, email: 'noFeatures@example.com' },
{
_id: userIds.noFeatureTimeout,
email: 'noFeatureTimeout@example.com',
features: {},
},
{
_id: userIds.timeout10s,
email: 'timeout10s@example.com',
features: { compileTimeout: 20, other: 'val1' },
bar: '1',
},
{
_id: userIds.timeout20s,
email: 'timeout20s@example.com',
features: { compileTimeout: 20, other: 'val2' },
bar: '2',
},
{
_id: userIds.timeout30s,
email: 'timeout30s@example.com',
features: { compileTimeout: 20, other: 'val3' },
bar: '3',
},
{
_id: userIds.timeout60s,
email: 'timeout60s@example.com',
features: { compileTimeout: 20, other: 'val4' },
bar: '4',
},
{
_id: userIds.timeout120s,
email: 'timeout120s@example.com',
features: { compileTimeout: 120, other: 'val5' },
bar: '5',
},
{
_id: userIds.timeout180s,
email: 'timeout180s@example.com',
features: { compileTimeout: 180, other: 'val6' },
bar: '6',
},
])
expect(usersAfter[0].featuresUpdatedAt).to.be.undefined
expect(usersAfter[1].featuresUpdatedAt).to.be.undefined
expect(usersAfter[2].featuresUpdatedAt).to.be.instanceOf(Date)
expect(usersAfter[3].featuresUpdatedAt).to.be.undefined // was already 20s
expect(usersAfter[4].featuresUpdatedAt).to.be.instanceOf(Date)
expect(usersAfter[5].featuresUpdatedAt).to.be.instanceOf(Date)
expect(usersAfter[6].featuresUpdatedAt).to.be.undefined
const result2 = await runScript([])
expect(result2.stdout).to.contain(
'Found 0 users with compileTimeout <= 60s && != 20s'
)
expect(result2.stdout).to.contain(
'Found 4 users with compileTimeout == 20s'
)
})
})
describe('fixup script', function () {
const usersInput = {
timeout20s1: {
features: { compileTimeout: 20 },
},
timeout20s2: {
features: { compileTimeout: 20 },
featuresUpdatedAt: new Date('2023-01-01'),
},
timeout20s3: {
features: { compileTimeout: 20 },
featuresUpdatedAt: new Date('2025-01-01'),
},
timeout240s1: {
features: { compileTimeout: 240 },
},
timeout240s2: {
features: { compileTimeout: 240 },
featuresUpdatedAt: new Date('2023-01-01'),
},
timeout240s3: {
features: { compileTimeout: 240 },
featuresUpdatedAt: new Date('2025-01-01'),
},
}
const usersKeys = Object.keys(usersInput)
const userIds = {}
beforeEach('insert users', async function () {
const usersInsertedValues = await db.users.insertMany(
usersKeys.map(key => ({
...usersInput[key],
email: `${key}@example.com`,
}))
)
usersKeys.forEach(
(key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
)
})
afterEach('clear users', async function () {
await db.users.deleteMany({})
})
it('gives correct counts in dry mode', async function () {
const users = await db.users.find().toArray()
expect(users).to.have.lengthOf(usersKeys.length)
const result = await runFixupScript([])
expect(result.stderr).to.contain(
'Doing dry run. Add --commit to commit changes'
)
expect(result.stdout).to.contain(
'Found 2 users needing their featuresUpdatedAt updated'
)
expect(result.stdout).not.to.contain('Updated 2 records')
const usersAfter = await db.users.find().toArray()
expect(usersAfter).to.deep.equal(users)
})
it("updates users featuresUpdatedAt when '--commit' is set", async function () {
const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z')
const users = await db.users.find().toArray()
expect(users).to.have.lengthOf(usersKeys.length)
const result = await runFixupScript(['--commit'])
expect(result.stdout).to.contain(
'Found 2 users needing their featuresUpdatedAt updated'
)
expect(result.stdout).to.contain('Updated 2 records')
const usersAfter = await db.users.find().toArray()
expect(usersAfter).to.deep.equal([
{
_id: userIds.timeout20s1,
email: 'timeout20s1@example.com',
features: { compileTimeout: 20 },
featuresUpdatedAt: FEATURES_UPDATED_AT,
},
{
_id: userIds.timeout20s2,
email: 'timeout20s2@example.com',
features: { compileTimeout: 20 },
featuresUpdatedAt: FEATURES_UPDATED_AT,
},
{
_id: userIds.timeout20s3,
email: 'timeout20s3@example.com',
features: { compileTimeout: 20 },
featuresUpdatedAt: new Date('2025-01-01'),
},
{
_id: userIds.timeout240s1,
email: 'timeout240s1@example.com',
features: { compileTimeout: 240 },
},
{
_id: userIds.timeout240s2,
email: 'timeout240s2@example.com',
features: { compileTimeout: 240 },
featuresUpdatedAt: new Date('2023-01-01'),
},
{
_id: userIds.timeout240s3,
email: 'timeout240s3@example.com',
features: { compileTimeout: 240 },
featuresUpdatedAt: new Date('2025-01-01'),
},
])
const result2 = await runFixupScript([])
expect(result2.stdout).to.contain(
'Found 0 users needing their featuresUpdatedAt updated'
)
})
})
})