[web] Create script to update forgotten `featuresUpdatedAt` after the migration to 20s compile timeout (#18113)

* Copy `migration_compile_timeout_60s_to_20s.js` script

* Update `featuresUpdatedAt`

* Add a comment about `featuresUpdatedAt` in migration_compile_timeout_60s_to_20s.js

* Fix test on migration_compile_timeout_60s_to_20s.js

* Fix: Include users having `featuresUpdatedAt` undefined in the update

* Add test on `migration_compile_timeout_60s_to_20s_fixup_features_updated_at`

GitOrigin-RevId: 4b2baf955a6a9f39bf9ce00b7839af551064c6cb

services/web/scripts/migration_compile_timeout_60s_to_20s.js

@@ -34,7 +34,8 @@ const main = async ({ COMMIT, SKIP_COUNT }) => {
     const nModified = await batchedUpdate(
       'users',
       { 'features.compileTimeout': { $lte: 60, $ne: 20 } },
-      { $set: { 'features.compileTimeout': 20 } }
+      // NOTE: Always update featuresUpdatedAt to ensure the user's features synced with BigQuery
+      { $set: { 'features.compileTimeout': 20, featuresUpdatedAt: new Date() } }
     )
     console.log(`Updated ${nModified} records`)
   }

services/web/scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js

@@ -0,0 +1,65 @@
+#!/usr/bin/env node
+
+const minimist = require('minimist')
+const {
+  db,
+  READ_PREFERENCE_SECONDARY,
+  waitForDb,
+} = require('../app/src/infrastructure/mongodb')
+const { batchedUpdate } = require('./helpers/batchedUpdate')
+
+// A few seconds after the previous migration script was run
+const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z')
+
+const query = {
+  'features.compileTimeout': 20,
+  $or: [
+    { featuresUpdatedAt: { $exists: false } },
+    { featuresUpdatedAt: { $lt: FEATURES_UPDATED_AT } },
+  ],
+}
+
+async function logCount() {
+  const usersToUpdate = await db.users.countDocuments(query, {
+    readPreference: READ_PREFERENCE_SECONDARY,
+  })
+  console.log(
+    `Found ${usersToUpdate} users needing their featuresUpdatedAt updated`
+  )
+}
+
+const main = async ({ COMMIT, SKIP_COUNT }) => {
+  console.time('Script Duration')
+
+  await waitForDb()
+
+  if (!SKIP_COUNT) {
+    await logCount()
+  }
+
+  if (COMMIT) {
+    const nModified = await batchedUpdate('users', query, {
+      $set: { featuresUpdatedAt: FEATURES_UPDATED_AT },
+    })
+    console.log(`Updated ${nModified} records`)
+  }
+
+  console.timeEnd('Script Duration')
+}
+
+const setup = () => {
+  const argv = minimist(process.argv.slice(2))
+  const COMMIT = argv.commit !== undefined
+  const SKIP_COUNT = argv['skip-count'] !== undefined
+  if (!COMMIT) {
+    console.warn('Doing dry run. Add --commit to commit changes')
+  }
+  return { COMMIT, SKIP_COUNT }
+}
+
+main(setup())
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })
+  .then(() => process.exit(0))

services/web/test/acceptance/src/MigrateUserFeatureTimeoutTests.js

@@ -17,118 +17,287 @@ async function runScript(args = []) {
   }
 }
 
-describe('MigrateUserFeatureTimeoutTests', function () {
-  const usersInput = {
-    noFeatures: {},
-    noFeatureTimeout: { features: {} },
-    timeout10s: { features: { compileTimeout: 10, other: 'val1' }, bar: '1' },
-    timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' },
-    timeout30s: { features: { compileTimeout: 30, other: 'val3' }, bar: '3' },
-    timeout60s: { features: { compileTimeout: 60, other: 'val4' }, bar: '4' },
-    timeout120s: { features: { compileTimeout: 120, other: 'val5' }, bar: '5' },
-    timeout180s: { features: { compileTimeout: 180, other: 'val6' }, bar: '6' },
+async function runFixupScript(args = []) {
+  try {
+    return await promisify(exec)(
+      [
+        'node',
+        'scripts/migration_compile_timeout_60s_to_20s_fixup_features_updated_at.js',
+        ...args,
+      ].join(' ')
+    )
+  } catch (error) {
+    logger.error({ error }, 'script failed')
+    throw error
   }
+}
 
-  const usersKeys = Object.keys(usersInput)
-  const userIds = {}
+describe('MigrateUserFeatureTimeoutTests', function () {
+  describe('initial script', function () {
+    const usersInput = {
+      noFeatures: {},
+      noFeatureTimeout: { features: {} },
+      timeout10s: {
+        features: { compileTimeout: 10, other: 'val1' },
+        bar: '1',
+        featuresUpdatedAt: new Date('2020-01-01'),
+      },
+      timeout20s: { features: { compileTimeout: 20, other: 'val2' }, bar: '2' },
+      timeout30s: {
+        features: { compileTimeout: 30, other: 'val3' },
+        bar: '3',
+        featuresUpdatedAt: new Date('2025-01-01'),
+      },
+      timeout60s: {
+        features: { compileTimeout: 60, other: 'val4' },
+        bar: '4',
+        featuresUpdatedAt: new Date(),
+      },
+      timeout120s: {
+        features: { compileTimeout: 120, other: 'val5' },
+        bar: '5',
+      },
+      timeout180s: {
+        features: { compileTimeout: 180, other: 'val6' },
+        bar: '6',
+        featuresUpdatedAt: new Date('2020-01-01'),
+      },
+    }
 
-  beforeEach('insert users', async function () {
-    const usersInsertedValues = await db.users.insertMany(
-      usersKeys.map(key => ({
-        ...usersInput[key],
-        email: `${key}@example.com`,
-      }))
-    )
-    usersKeys.forEach(
-      (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
-    )
-  })
+    const usersKeys = Object.keys(usersInput)
+    const userIds = {}
 
-  it('gives correct counts in dry mode', async function () {
-    const users = await db.users.find().toArray()
-    expect(users).to.have.lengthOf(usersKeys.length)
+    beforeEach('insert users', async function () {
+      const usersInsertedValues = await db.users.insertMany(
+        usersKeys.map(key => ({
+          ...usersInput[key],
+          email: `${key}@example.com`,
+        }))
+      )
+      usersKeys.forEach(
+        (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
+      )
+    })
 
-    const result = await runScript([])
+    afterEach('clear users', async function () {
+      await db.users.deleteMany({})
+    })
 
-    expect(result.stderr).to.contain(
-      'Doing dry run. Add --commit to commit changes'
-    )
-    expect(result.stdout).to.contain(
-      'Found 3 users with compileTimeout <= 60s && != 20s'
-    )
-    expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s')
-    expect(result.stdout).not.to.contain('Updated')
+    it('gives correct counts in dry mode', async function () {
+      const users = await db.users.find().toArray()
+      expect(users).to.have.lengthOf(usersKeys.length)
 
-    const usersAfter = await db.users.find().toArray()
+      const result = await runScript([])
 
-    expect(usersAfter).to.deep.equal(users)
-  })
+      expect(result.stderr).to.contain(
+        'Doing dry run. Add --commit to commit changes'
+      )
+      expect(result.stdout).to.contain(
+        'Found 3 users with compileTimeout <= 60s && != 20s'
+      )
+      expect(result.stdout).to.contain(
+        'Found 1 users with compileTimeout == 20s'
+      )
+      expect(result.stdout).not.to.contain('Updated')
 
-  it("updates users compileTimeout when '--commit' is set", async function () {
-    const users = await db.users.find().toArray()
-    expect(users).to.have.lengthOf(usersKeys.length)
-    const result = await runScript(['--commit'])
+      const usersAfter = await db.users.find().toArray()
 
-    expect(result.stdout).to.contain(
-      'Found 3 users with compileTimeout <= 60s && != 20s'
-    )
-    expect(result.stdout).to.contain('Found 1 users with compileTimeout == 20s')
-    expect(result.stdout).to.contain('Updated 3 records')
+      expect(usersAfter).to.deep.equal(users)
+    })
 
-    const usersAfter = await db.users.find().toArray()
+    it("updates users compileTimeout when '--commit' is set", async function () {
+      const users = await db.users.find().toArray()
+      expect(users).to.have.lengthOf(usersKeys.length)
+      const result = await runScript(['--commit'])
 
-    expect(usersAfter).to.deep.equal([
-      { _id: userIds.noFeatures, email: 'noFeatures@example.com' },
-      {
-        _id: userIds.noFeatureTimeout,
-        email: 'noFeatureTimeout@example.com',
-        features: {},
-      },
-      {
-        _id: userIds.timeout10s,
-        email: 'timeout10s@example.com',
-        features: { compileTimeout: 20, other: 'val1' },
-        bar: '1',
+      expect(result.stdout).to.contain(
+        'Found 3 users with compileTimeout <= 60s && != 20s'
+      )
+      expect(result.stdout).to.contain(
+        'Found 1 users with compileTimeout == 20s'
+      )
+      expect(result.stdout).to.contain('Updated 3 records')
+
+      const usersAfter = await db.users.find().toArray()
+
+      expect(
+        usersAfter.map(({ featuresUpdatedAt, ...rest }) => rest)
+      ).to.deep.equal([
+        { _id: userIds.noFeatures, email: 'noFeatures@example.com' },
+        {
+          _id: userIds.noFeatureTimeout,
+          email: 'noFeatureTimeout@example.com',
+          features: {},
+        },
+        {
+          _id: userIds.timeout10s,
+          email: 'timeout10s@example.com',
+          features: { compileTimeout: 20, other: 'val1' },
+          bar: '1',
+        },
+        {
+          _id: userIds.timeout20s,
+          email: 'timeout20s@example.com',
+          features: { compileTimeout: 20, other: 'val2' },
+          bar: '2',
+        },
+        {
+          _id: userIds.timeout30s,
+          email: 'timeout30s@example.com',
+          features: { compileTimeout: 20, other: 'val3' },
+          bar: '3',
+        },
+        {
+          _id: userIds.timeout60s,
+          email: 'timeout60s@example.com',
+          features: { compileTimeout: 20, other: 'val4' },
+          bar: '4',
+        },
+        {
+          _id: userIds.timeout120s,
+          email: 'timeout120s@example.com',
+          features: { compileTimeout: 120, other: 'val5' },
+          bar: '5',
+        },
+        {
+          _id: userIds.timeout180s,
+          email: 'timeout180s@example.com',
+          features: { compileTimeout: 180, other: 'val6' },
+          bar: '6',
+        },
+      ])
+
+      expect(usersAfter[0].featuresUpdatedAt).to.be.undefined
+      expect(usersAfter[1].featuresUpdatedAt).to.be.undefined
+      expect(usersAfter[2].featuresUpdatedAt).to.be.instanceOf(Date)
+      expect(usersAfter[3].featuresUpdatedAt).to.be.undefined // was already 20s
+      expect(usersAfter[4].featuresUpdatedAt).to.be.instanceOf(Date)
+      expect(usersAfter[5].featuresUpdatedAt).to.be.instanceOf(Date)
+      expect(usersAfter[6].featuresUpdatedAt).to.be.undefined
+
+      const result2 = await runScript([])
+
+      expect(result2.stdout).to.contain(
+        'Found 0 users with compileTimeout <= 60s && != 20s'
+      )
+      expect(result2.stdout).to.contain(
+        'Found 4 users with compileTimeout == 20s'
+      )
+    })
+  })
+
+  describe('fixup script', function () {
+    const usersInput = {
+      timeout20s1: {
+        features: { compileTimeout: 20 },
       },
-      {
-        _id: userIds.timeout20s,
-        email: 'timeout20s@example.com',
-        features: { compileTimeout: 20, other: 'val2' },
-        bar: '2',
+      timeout20s2: {
+        features: { compileTimeout: 20 },
+        featuresUpdatedAt: new Date('2023-01-01'),
       },
-      {
-        _id: userIds.timeout30s,
-        email: 'timeout30s@example.com',
-        features: { compileTimeout: 20, other: 'val3' },
-        bar: '3',
+      timeout20s3: {
+        features: { compileTimeout: 20 },
+        featuresUpdatedAt: new Date('2025-01-01'),
       },
-      {
-        _id: userIds.timeout60s,
-        email: 'timeout60s@example.com',
-        features: { compileTimeout: 20, other: 'val4' },
-        bar: '4',
+      timeout240s1: {
+        features: { compileTimeout: 240 },
       },
-      {
-        _id: userIds.timeout120s,
-        email: 'timeout120s@example.com',
-        features: { compileTimeout: 120, other: 'val5' },
-        bar: '5',
+      timeout240s2: {
+        features: { compileTimeout: 240 },
+        featuresUpdatedAt: new Date('2023-01-01'),
       },
-      {
-        _id: userIds.timeout180s,
-        email: 'timeout180s@example.com',
-        features: { compileTimeout: 180, other: 'val6' },
-        bar: '6',
+      timeout240s3: {
+        features: { compileTimeout: 240 },
+        featuresUpdatedAt: new Date('2025-01-01'),
       },
-    ])
+    }
 
-    const result2 = await runScript([])
+    const usersKeys = Object.keys(usersInput)
+    const userIds = {}
 
-    expect(result2.stdout).to.contain(
-      'Found 0 users with compileTimeout <= 60s && != 20s'
-    )
-    expect(result2.stdout).to.contain(
-      'Found 4 users with compileTimeout == 20s'
-    )
+    beforeEach('insert users', async function () {
+      const usersInsertedValues = await db.users.insertMany(
+        usersKeys.map(key => ({
+          ...usersInput[key],
+          email: `${key}@example.com`,
+        }))
+      )
+      usersKeys.forEach(
+        (key, index) => (userIds[key] = usersInsertedValues.insertedIds[index])
+      )
+    })
+    afterEach('clear users', async function () {
+      await db.users.deleteMany({})
+    })
+
+    it('gives correct counts in dry mode', async function () {
+      const users = await db.users.find().toArray()
+      expect(users).to.have.lengthOf(usersKeys.length)
+      const result = await runFixupScript([])
+      expect(result.stderr).to.contain(
+        'Doing dry run. Add --commit to commit changes'
+      )
+      expect(result.stdout).to.contain(
+        'Found 2 users needing their featuresUpdatedAt updated'
+      )
+      expect(result.stdout).not.to.contain('Updated 2 records')
+      const usersAfter = await db.users.find().toArray()
+      expect(usersAfter).to.deep.equal(users)
+    })
+
+    it("updates users featuresUpdatedAt when '--commit' is set", async function () {
+      const FEATURES_UPDATED_AT = new Date('2024-04-16T12:41:00Z')
+      const users = await db.users.find().toArray()
+      expect(users).to.have.lengthOf(usersKeys.length)
+      const result = await runFixupScript(['--commit'])
+      expect(result.stdout).to.contain(
+        'Found 2 users needing their featuresUpdatedAt updated'
+      )
+      expect(result.stdout).to.contain('Updated 2 records')
+      const usersAfter = await db.users.find().toArray()
+      expect(usersAfter).to.deep.equal([
+        {
+          _id: userIds.timeout20s1,
+          email: 'timeout20s1@example.com',
+          features: { compileTimeout: 20 },
+          featuresUpdatedAt: FEATURES_UPDATED_AT,
+        },
+        {
+          _id: userIds.timeout20s2,
+          email: 'timeout20s2@example.com',
+          features: { compileTimeout: 20 },
+          featuresUpdatedAt: FEATURES_UPDATED_AT,
+        },
+        {
+          _id: userIds.timeout20s3,
+          email: 'timeout20s3@example.com',
+          features: { compileTimeout: 20 },
+          featuresUpdatedAt: new Date('2025-01-01'),
+        },
+        {
+          _id: userIds.timeout240s1,
+          email: 'timeout240s1@example.com',
+          features: { compileTimeout: 240 },
+        },
+        {
+          _id: userIds.timeout240s2,
+          email: 'timeout240s2@example.com',
+          features: { compileTimeout: 240 },
+          featuresUpdatedAt: new Date('2023-01-01'),
+        },
+        {
+          _id: userIds.timeout240s3,
+          email: 'timeout240s3@example.com',
+          features: { compileTimeout: 240 },
+          featuresUpdatedAt: new Date('2025-01-01'),
+        },
+      ])
+
+      const result2 = await runFixupScript([])
+
+      expect(result2.stdout).to.contain(
+        'Found 0 users needing their featuresUpdatedAt updated'
+      )
+    })
   })
 })