Merge pull request #12277 from overleaf/jpa-cleanup-batched-update-interface

[web] simplify interface for custom update function in batchedUpdate GitOrigin-RevId: a00a24a012db400d4161de0bcefa2681206ab296
2024-11-21 20:47:08 -05:00 · 2023-03-22 11:21:47 +00:00 · 2023-03-22 11:21:47 +00:00 · 5d9923ad1b
commit 5d9923ad1b
parent a140e3dc8c
16 changed files with 27 additions and 25 deletions
--- a/services/web/scripts/back_fill_deleted_files.js
+++ b/services/web/scripts/back_fill_deleted_files.js
@ -24,14 +24,14 @@ async function main(options) {
    'projects',
    // array is not empty ~ array has one item
    { 'deletedFiles.0': { $exists: true } },
-    async (x, projects) => {
-      await processBatch(x, projects, options)
+    async projects => {
+      await processBatch(projects, options)
    },
    { _id: 1, deletedFiles: 1 }
  )
 }

-async function processBatch(_, projects, options) {
+async function processBatch(projects, options) {
  await promiseMapWithLimit(
    options.writeConcurrency,
    projects,
--- a/services/web/scripts/back_fill_doc_name_for_deleted_docs.js
+++ b/services/web/scripts/back_fill_doc_name_for_deleted_docs.js
@ -23,14 +23,14 @@ async function main(options) {
    'projects',
    // array is not empty ~ array has one item
    { 'deletedDocs.0': { $exists: true } },
-    async (_collection, projects) => {
-      await processBatch(_collection, projects, options)
+    async projects => {
+      await processBatch(projects, options)
    },
    { _id: 1, deletedDocs: 1 }
  )
 }

-async function processBatch(_, projects, options) {
+async function processBatch(projects, options) {
  await promiseMapWithLimit(
    options.writeConcurrency,
    projects,
--- a/services/web/scripts/back_fill_doc_rev.js
+++ b/services/web/scripts/back_fill_doc_rev.js
@ -1,3 +1,4 @@
+const { db } = require('../app/src/infrastructure/mongodb')
 const { batchedUpdate } = require('./helpers/batchedUpdate')

 const DRY_RUN = !process.argv.includes('--dry-run=false')
@ -15,9 +16,9 @@ async function main(DRY_RUN) {
  await batchedUpdate(
    'docs',
    { rev: { $exists: false } },
-    async (docsCollection, docs) => {
+    async docs => {
      if (!DRY_RUN) {
-        await docsCollection.updateMany(
+        await db.docs.updateMany(
          {
            _id: { $in: docs.map(doc => doc._id) },
            rev: { $exists: false },
--- a/services/web/scripts/convert_archived_state.js
+++ b/services/web/scripts/convert_archived_state.js
@ -2,6 +2,7 @@ const _ = require('lodash')

 const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10

+const { db } = require('../app/src/infrastructure/mongodb')
 const { batchedUpdate } = require('./helpers/batchedUpdate')
 const { promiseMapWithLimit } = require('../app/src/util/promises')

@ -25,13 +26,13 @@ async function main(STAGE) {
      await batchedUpdate(
        'projects',
        { [FIELD]: true },
-        async function performUpdate(collection, nextBatch) {
+        async function performUpdate(nextBatch) {
          await promiseMapWithLimit(
            WRITE_CONCURRENCY,
            nextBatch,
            async project => {
              try {
-                await upgradeFieldToArray({ collection, project, FIELD })
+                await upgradeFieldToArray({ project, FIELD })
              } catch (err) {
                console.error(project._id, err)
                throw err
@ -67,8 +68,8 @@ if (require.main === module) {
    })
 }

-async function upgradeFieldToArray({ collection, project, FIELD }) {
-  return collection.updateOne(
+async function upgradeFieldToArray({ project, FIELD }) {
+  return db.projects.updateOne(
    { _id: project._id },
    {
      $set: { [FIELD]: getAllUserIds(project) },
--- a/services/web/scripts/delete_orphaned_chat_threads.js
+++ b/services/web/scripts/delete_orphaned_chat_threads.js
@ -32,7 +32,7 @@ const RESULT = {
  continueFrom: null,
 }

-async function processBatch(_, rooms) {
+async function processBatch(rooms) {
  if (rooms.length && rooms[0]._id) {
    RESULT.continueFrom = rooms[0]._id
  }
--- a/services/web/scripts/fix_group_invite_emails_to_lowercase.js
+++ b/services/web/scripts/fix_group_invite_emails_to_lowercase.js
@ -13,7 +13,7 @@ function anyInviteEmailHasUppercaseChars(subscription) {
  })
 }

-async function processBatch(_, subscriptions) {
+async function processBatch(subscriptions) {
  for (const subscription of subscriptions) {
    if (anyInviteEmailHasUppercaseChars(subscription)) {
      console.log('fixing emails in group invites for', subscription._id)
--- a/services/web/scripts/helpers/batchedUpdate.js
+++ b/services/web/scripts/helpers/batchedUpdate.js
@ -161,7 +161,7 @@ async function batchedUpdate(
      }

      if (typeof update === 'function') {
-        await update(collection, nextBatch)
+        await update(nextBatch)
      } else {
        await performUpdate(collection, nextBatch, update)
      }
--- a/services/web/scripts/history/count_project_history_categories.js
+++ b/services/web/scripts/history/count_project_history_categories.js
@ -29,7 +29,7 @@ const COUNT = {
  TotalProjects: 0,
 }

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
  console.log(COUNT)
 }
--- a/services/web/scripts/history/reset_incorrect_doc_revision.js
+++ b/services/web/scripts/history/reset_incorrect_doc_revision.js
@ -49,7 +49,7 @@ async function main() {
  console.log('Final')
 }

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
 }

--- a/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js
+++ b/services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js
@ -68,7 +68,7 @@ const RESULT = {

 let INTERRUPT = false

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  if (projects.length && projects[0]._id) {
    RESULT.continueFrom = projects[0]._id
  }
--- a/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js
+++ b/services/web/scripts/history/upgrade_none_without_conversion_if_no_sl_history.js
@ -42,7 +42,7 @@ const RESULT = {

 let INTERRUPT = false

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  if (projects.length && projects[0]._id) {
    RESULT.continueFrom = projects[0]._id
  }
--- a/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js
+++ b/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js
@ -44,7 +44,7 @@ const RESULT = {

 let INTERRUPT = false

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  if (projects.length && projects[0]._id) {
    RESULT.continueFrom = projects[0]._id
  }
--- a/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js
+++ b/services/web/scripts/history/upgrade_v1_without_conversion_if_created_after_fph_enabled.js
@ -42,7 +42,7 @@ if (!process.env.BATCH_LAST_ID) {
  process.env.BATCH_LAST_ID = ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
 }

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
  console.log(RESULT)
 }
--- a/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js
+++ b/services/web/scripts/history/upgrade_v1_without_conversion_if_no_sl_history.js
@ -31,7 +31,7 @@ const RESULT = {
  projectsUpgraded: 0,
 }

-async function processBatch(_, projects) {
+async function processBatch(projects) {
  await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
  console.log(RESULT)
 }
--- a/services/web/scripts/migrate_audit_logs.js
+++ b/services/web/scripts/migrate_audit_logs.js
@ -49,7 +49,7 @@ async function main(options) {
      await batchedUpdate(
        'users',
        { auditLog: { $exists: true } },
-        async (_, users) => {
+        async users => {
          await processUsersBatch(users, options)
        },
        { _id: 1, auditLog: 1 }
@ -62,7 +62,7 @@ async function main(options) {
    await batchedUpdate(
      'projects',
      { auditLog: { $exists: true } },
-      async (x, projects) => {
+      async projects => {
        await processProjectsBatch(projects, options)
      },
      { _id: 1, auditLog: 1 }
--- a/services/web/scripts/regenerate_duplicate_referral_ids.js
+++ b/services/web/scripts/regenerate_duplicate_referral_ids.js
@ -59,7 +59,7 @@ async function rewriteDuplicates(duplicateReferralIds) {
  )
 }

-async function processBatch(_, users) {
+async function processBatch(users) {
  const uniqueReferalIdsInBatch = Array.from(
    new Set(users.map(user => user.referal_id))
  )