Merge pull request #18985 from overleaf/em-migrate-bulk

Bulk history ranges migration

GitOrigin-RevId: 0d1846b412cfcddead63a7bc15bd06a82fbb47f3

services/web/app/src/Features/History/HistoryRangesSupportMigration.js

@@ -1,7 +1,103 @@
 // @ts-check
 
 const { callbackify } = require('util')
+const { ObjectId } = require('mongodb')
+const logger = require('@overleaf/logger')
 const HistoryManager = require('../History/HistoryManager')
+const { db } = require('../../infrastructure/mongodb')
+
+/**
+ * Migrate projects based on a query.
+ *
+ * @param {object} opts
+ * @param {string[]} [opts.projectIds]
+ * @param {string[]} [opts.ownerIds]
+ * @param {string} [opts.minId]
+ * @param {string} [opts.maxId]
+ * @param {number} [opts.maxCount]
+ * @param {"forwards" | "backwards"} [opts.direction]
+ * @param {boolean} [opts.force]
+ * @param {boolean} [opts.stopOnError]
+ */
+async function migrateProjects(opts = {}) {
+  const {
+    ownerIds,
+    projectIds,
+    minId,
+    maxId,
+    maxCount = Infinity,
+    direction = 'forwards',
+    force = false,
+    stopOnError = false,
+  } = opts
+
+  const clauses = []
+  if (projectIds != null) {
+    clauses.push({ _id: { $in: projectIds.map(id => new ObjectId(id)) } })
+  }
+  if (ownerIds != null) {
+    clauses.push({ owner_ref: { $in: ownerIds.map(id => new ObjectId(id)) } })
+  }
+  if (minId) {
+    clauses.push({ _id: { $gte: new ObjectId(minId) } })
+  }
+  if (maxId) {
+    clauses.push({ _id: { $lte: new ObjectId(maxId) } })
+  }
+
+  const filter = {}
+  if (clauses.length > 0) {
+    filter.$and = clauses
+  }
+
+  const projects = db.projects
+    .find(filter, {
+      projection: { _id: 1, overleaf: 1 },
+    })
+    .sort({ _id: -1 })
+
+  let projectsProcessed = 0
+  for await (const project of projects) {
+    if (projectsProcessed >= maxCount) {
+      break
+    }
+    const projectId = project._id.toString()
+
+    if (!force) {
+      // Skip projects that are already migrated
+      if (
+        (direction === 'forwards' &&
+          project.overleaf.history.rangesSupportEnabled) ||
+        (direction === 'backwards' &&
+          !project.overleaf.history.rangesSupportEnabled)
+      ) {
+        continue
+      }
+    }
+
+    const startTimeMs = Date.now()
+    try {
+      await migrateProject(projectId, direction)
+    } catch (err) {
+      logger.error(
+        { projectId, direction, projectsProcessed },
+        'Failed to migrate history ranges support'
+      )
+      projectsProcessed += 1
+      if (stopOnError) {
+        break
+      } else {
+        continue
+      }
+    }
+    const elapsedMs = Date.now() - startTimeMs
+    projectsProcessed += 1
+    logger.info(
+      { projectId, direction, projectsProcessed, elapsedMs },
+      'Migrated history ranges support'
+    )
+  }
+}
 
 /**
  * Migrate a single project
@@ -17,6 +113,7 @@ async function migrateProject(projectId, direction = 'forwards') {
 }
 
 module.exports = {
+  migrateProjects: callbackify(migrateProjects),
   migrateProject: callbackify(migrateProject),
-  promises: { migrateProject },
+  promises: { migrateProjects, migrateProject },
 }

services/web/scripts/history/migrate_ranges_support.js

@@ -4,30 +4,101 @@ const minimist = require('minimist')
 
 async function main() {
   await waitForDb()
-  const { projectId, direction } = parseArgs()
-  await HistoryRangesSupportMigration.promises.migrateProject(
-    projectId,
-    direction
-  )
+  const {
+    projectIds,
+    ownerIds,
+    minId,
+    maxId,
+    maxCount,
+    direction,
+    force,
+    stopOnError,
+  } = parseArgs()
+  await HistoryRangesSupportMigration.promises.migrateProjects({
+    projectIds,
+    ownerIds,
+    minId,
+    maxId,
+    maxCount,
+    direction,
+    force,
+    stopOnError,
+  })
 }
 
 function usage() {
-  console.log('Usage: migrate_ranges_support.js PROJECT_ID [--backwards]')
+  console.error(`Usage: migrate_ranges_support.js [OPTIONS]
+
+Options:
+
+    --help           Print this help
+    --owner-id       Migrate all projects owned by this owner
+    --project-id     Migrate this project
+    --min-id         Migrate projects from this id
+    --max-id         Migrate projects to this id
+    --max-count      Migrate at most this number of projects
+    --all            Migrate all projects
+    --backwards      Disable history ranges support for selected project ids
+    --force          Migrate projects even if they were already migrated
+    --stop-on-error  Stop after first migration error
+`)
 }
 
 function parseArgs() {
   const args = minimist(process.argv.slice(2), {
-    boolean: ['backwards'],
+    boolean: ['backwards', 'help', 'all', 'force'],
+    string: ['owner-id', 'project-id', 'min-id', 'max-id'],
   })
 
-  if (args._.length !== 1) {
+  if (args.help) {
+    usage()
+    process.exit(0)
+  }
+
+  const direction = args.backwards ? 'backwards' : 'forwards'
+  const ownerIds = arrayOpt(args['owner-id'])
+  const projectIds = arrayOpt(args['project-id'])
+  const minId = args['min-id']
+  const maxId = args['max-id']
+  const maxCount = args['max-count']
+  const force = args.force
+  const stopOnError = args['stop-on-error']
+  const all = args.all
+
+  if (
+    !all &&
+    ownerIds == null &&
+    projectIds == null &&
+    minId == null &&
+    maxId == null &&
+    maxCount == null
+  ) {
+    console.error(
+      'Please specify at least one filter, or --all to process all projects\n'
+    )
     usage()
     process.exit(1)
   }
 
   return {
-    direction: args.backwards ? 'backwards' : 'forwards',
-    projectId: args._[0],
+    ownerIds,
+    projectIds,
+    minId,
+    maxId,
+    maxCount,
+    direction,
+    force,
+    stopOnError,
+  }
+}
+
+function arrayOpt(value) {
+  if (typeof value === 'string') {
+    return [value]
+  } else if (Array.isArray(value)) {
+    return value
+  } else {
+    return undefined
   }
 }