[web] Create script to update forgotten featuresUpdatedAt after the migration to 20s compile timeout (#18113)

* Copy `migration_compile_timeout_60s_to_20s.js` script

* Update `featuresUpdatedAt`

* Add a comment about `featuresUpdatedAt` in migration_compile_timeout_60s_to_20s.js

* Fix test on migration_compile_timeout_60s_to_20s.js

* Fix: Include users having `featuresUpdatedAt` undefined in the update

* Add test on `migration_compile_timeout_60s_to_20s_fixup_features_updated_at`

GitOrigin-RevId: 4b2baf955a6a9f39bf9ce00b7839af551064c6cb

services/web/scripts/migration_compile_timeout_60s_to_20s.js

@@ -34,7 +34,8 @@ const main = async ({ COMMIT, SKIP_COUNT }) => {
     const nModified = await batchedUpdate(
       'users',
       { 'features.compileTimeout': { $lte: 60, $ne: 20 } },
-      { $set: { 'features.compileTimeout': 20 } }
+      // NOTE: Always update featuresUpdatedAt to ensure the user's features synced with BigQuery
+      { $set: { 'features.compileTimeout': 20, featuresUpdatedAt: new Date() } }
     )
     console.log(`Updated ${nModified} records`)
   }

services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+
+const minimist = require('minimist')
+const {
+  db,
+  READ_PREFERENCE_SECONDARY,
+  waitForDb,
+} = require('../app/src/infrastructure/mongodb')
+const { batchedUpdate } = require('./helpers/batchedUpdate')
+
+// A few seconds after the previous migration script was run
+const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z')
+
+const query = {
+  'features.compileTimeout': 20,
+  $or: [
+    { featuresUpdatedAt: { $exists: false } },
+    { featuresUpdatedAt: { $lt: FEATURES_UPDATED_AT } },
+  ],
+}
+
+async function logCount() {
+  const usersToUpdate = await db.users.countDocuments(query, {
+    readPreference: READ_PREFERENCE_SECONDARY,
+  })
+  console.log(
+    `Found ${usersToUpdate} users needing their featuresUpdatedAt updated`
+  )
+}
+
+const main = async ({ COMMIT, SKIP_COUNT }) => {
+  console.time('Script Duration')
+
+  await waitForDb()
+
+  if (!SKIP_COUNT) {
+    await logCount()
+  }
+
+  if (COMMIT) {
+    const nModified = await batchedUpdate('users', query, {
+      $set: { featuresUpdatedAt: FEATURES_UPDATED_AT },
+    })
+    console.log(`Updated ${nModified} records`)
+  }
+
+  console.timeEnd('Script Duration')
+}
+
+const setup = () => {
+  const argv = minimist(process.argv.slice(2))
+  const COMMIT = argv.commit !== undefined
+  const SKIP_COUNT = argv['skip-count'] !== undefined
+  if (!COMMIT) {
+    console.warn('Doing dry run. Add --commit to commit changes')
+  }
+  return { COMMIT, SKIP_COUNT }
+}
+
+main(setup())
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(() => process.exit(0))

services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js

@@ -17,16 +17,51 @@ async function runScript(args = []) {
   }
 }
 
+async function runFixupScript(args = []) {
+  try {
+    return await promisify(exec)(
+      [
+        'node',
+        'scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js',
+        ...args,
+      ].join(' ')
+    )
+  } catch (error) {
+    logger.error({ error }, 'script failed')
+    throw error
+  }
+}
+
 describe('MigrateUserFeatureTimeoutTests', function () {
+  describe('initial script', function () {
     const usersInput = {
       noFeatures: {},
       noFeatureTimeout: { features: {} },
-    timeout10s: { features: { compileTimeout: 10, other: 'val1' }, bar: '1' },
+      timeout10s: {
+        features: { compileTimeout: 10, other: 'val1' },
+        bar: '1',
+        featuresUpdatedAt: new Date('2020-01-01'),
+      },
       timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' },
-    timeout30s: { features: { compileTimeout: 30, other: 'val3' }, bar: '3' },
+      timeout30s: {
-    timeout60s: { features: { compileTimeout: 60, other: 'val4' }, bar: '4' },
+        features: { compileTimeout: 30, other: 'val3' },
-    timeout120s: { features: { compileTimeout: 120, other: 'val5' }, bar: '5' },
+        bar: '3',
-    timeout180s: { features: { compileTimeout: 180, other: 'val6' }, bar: '6' },
+        featuresUpdatedAt: new Date('2025-01-01'),
+      },
+      timeout60s: {
+        features: { compileTimeout: 60, other: 'val4' },
+        bar: '4',
+        featuresUpdatedAt: new Date(),
+      },
+      timeout120s: {
+        features: { compileTimeout: 120, other: 'val5' },
+        bar: '5',
+      },
+      timeout180s: {
+        features: { compileTimeout: 180, other: 'val6' },
+        bar: '6',
+        featuresUpdatedAt: new Date('2020-01-01'),
+      },
     }
 
     const usersKeys = Object.keys(usersInput)
@@ -44,6 +79,10 @@ describe('MigrateUserFeatureTimeoutTests', function () {
       )
     })
 
+    afterEach('clear users', async function () {
+      await db.users.deleteMany({})
+    })
+
     it('gives correct counts in dry mode', async function () {
       const users = await db.users.find().toArray()
       expect(users).to.have.lengthOf(usersKeys.length)
@@ -56,7 +95,9 @@ describe('MigrateUserFeatureTimeoutTests', function () {
       expect(result.stdout).to.contain(
         'Found 3 users with compileTimeout <= 60s && != 20s'
       )
-    expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s')
+      expect(result.stdout).to.contain(
+        'Found 1 users with compileTimeout == 20s'
+      )
       expect(result.stdout).not.to.contain('Updated')
 
       const usersAfter = await db.users.find().toArray()
@@ -72,12 +113,16 @@ describe('MigrateUserFeatureTimeoutTests', function () {
       expect(result.stdout).to.contain(
         'Found 3 users with compileTimeout <= 60s && != 20s'
       )
-    expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s')
+      expect(result.stdout).to.contain(
+        'Found 1 users with compileTimeout == 20s'
+      )
       expect(result.stdout).to.contain('Updated 3 records')
 
       const usersAfter = await db.users.find().toArray()
 
-    expect(usersAfter).to.deep.equal([
+      expect(
+        usersAfter.map(({ featuresUpdatedAt, ...rest }) => rest)
+      ).to.deep.equal([
         { _id: userIds.noFeatures, email: 'noFeatures@example.com' },
         {
           _id: userIds.noFeatureTimeout,
@@ -122,6 +167,14 @@ describe('MigrateUserFeatureTimeoutTests', function () {
         },
       ])
 
+      expect(usersAfter[0].featuresUpdatedAt).to.be.undefined
+      expect(usersAfter[1].featuresUpdatedAt).to.be.undefined
+      expect(usersAfter[2].featuresUpdatedAt).to.be.instanceOf(Date)
+      expect(usersAfter[3].featuresUpdatedAt).to.be.undefined // was already 20s
+      expect(usersAfter[4].featuresUpdatedAt).to.be.instanceOf(Date)
+      expect(usersAfter[5].featuresUpdatedAt).to.be.instanceOf(Date)
+      expect(usersAfter[6].featuresUpdatedAt).to.be.undefined
+
       const result2 = await runScript([])
 
       expect(result2.stdout).to.contain(
@@ -131,4 +184,120 @@ describe('MigrateUserFeatureTimeoutTests', function () {
         'Found 4 users with compileTimeout == 20s'
       )
     })
+  })
+
+  describe('fixup script', function () {
+    const usersInput = {
+      timeout20s1: {
+        features: { compileTimeout: 20 },
+      },
+      timeout20s2: {
+        features: { compileTimeout: 20 },
+        featuresUpdatedAt: new Date('2023-01-01'),
+      },
+      timeout20s3: {
+        features: { compileTimeout: 20 },
+        featuresUpdatedAt: new Date('2025-01-01'),
+      },
+      timeout240s1: {
+        features: { compileTimeout: 240 },
+      },
+      timeout240s2: {
+        features: { compileTimeout: 240 },
+        featuresUpdatedAt: new Date('2023-01-01'),
+      },
+      timeout240s3: {
+        features: { compileTimeout: 240 },
+        featuresUpdatedAt: new Date('2025-01-01'),
+      },
+    }
+
+    const usersKeys = Object.keys(usersInput)
+    const userIds = {}
+
+    beforeEach('insert users', async function () {
+      const usersInsertedValues = await db.users.insertMany(
+        usersKeys.map(key => ({
+          ...usersInput[key],
+          email: `${key}@example.com`,
+        }))
+      )
+      usersKeys.forEach(
+        (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
+      )
+    })
+    afterEach('clear users', async function () {
+      await db.users.deleteMany({})
+    })
+
+    it('gives correct counts in dry mode', async function () {
+      const users = await db.users.find().toArray()
+      expect(users).to.have.lengthOf(usersKeys.length)
+      const result = await runFixupScript([])
+      expect(result.stderr).to.contain(
+        'Doing dry run. Add --commit to commit changes'
+      )
+      expect(result.stdout).to.contain(
+        'Found 2 users needing their featuresUpdatedAt updated'
+      )
+      expect(result.stdout).not.to.contain('Updated 2 records')
+      const usersAfter = await db.users.find().toArray()
+      expect(usersAfter).to.deep.equal(users)
+    })
+
+    it("updates users featuresUpdatedAt when '--commit' is set", async function () {
+      const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z')
+      const users = await db.users.find().toArray()
+      expect(users).to.have.lengthOf(usersKeys.length)
+      const result = await runFixupScript(['--commit'])
+      expect(result.stdout).to.contain(
+        'Found 2 users needing their featuresUpdatedAt updated'
+      )
+      expect(result.stdout).to.contain('Updated 2 records')
+      const usersAfter = await db.users.find().toArray()
+      expect(usersAfter).to.deep.equal([
+        {
+          _id: userIds.timeout20s1,
+          email: 'timeout20s1@example.com',
+          features: { compileTimeout: 20 },
+          featuresUpdatedAt: FEATURES_UPDATED_AT,
+        },
+        {
+          _id: userIds.timeout20s2,
+          email: 'timeout20s2@example.com',
+          features: { compileTimeout: 20 },
+          featuresUpdatedAt: FEATURES_UPDATED_AT,
+        },
+        {
+          _id: userIds.timeout20s3,
+          email: 'timeout20s3@example.com',
+          features: { compileTimeout: 20 },
+          featuresUpdatedAt: new Date('2025-01-01'),
+        },
+        {
+          _id: userIds.timeout240s1,
+          email: 'timeout240s1@example.com',
+          features: { compileTimeout: 240 },
+        },
+        {
+          _id: userIds.timeout240s2,
+          email: 'timeout240s2@example.com',
+          features: { compileTimeout: 240 },
+          featuresUpdatedAt: new Date('2023-01-01'),
+        },
+        {
+          _id: userIds.timeout240s3,
+          email: 'timeout240s3@example.com',
+          features: { compileTimeout: 240 },
+          featuresUpdatedAt: new Date('2025-01-01'),
+        },
+      ])
+
+      const result2 = await runFixupScript([])
+
+      expect(result2.stdout).to.contain(
+        'Found 0 users needing their featuresUpdatedAt updated'
+      )
+    })
+  })
 })