Merge pull request #6180 from overleaf/tm-history-upgrade-individual-projects

Add script for converting individual projects to FPH and history upgrade helper

GitOrigin-RevId: 4cf075e08045869fe87724fb28949fe91b780e93

services/web/scripts/history/HistoryUpgradeHelper.js

@@ -0,0 +1,299 @@
+const { ReadPreference, ObjectId } = require('mongodb')
+const { db } = require('../../app/src/infrastructure/mongodb')
+
+const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
+const HistoryManager = require('../../app/src/Features/History/HistoryManager')
+const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
+
+// Timestamp of when 'Enable history for SL in background' release
+const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
+const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
+  ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
+)
+const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
+  OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
+
+async function determineProjectHistoryType(project) {
+  if (project.overleaf && project.overleaf.history) {
+    if (project.overleaf.history.upgradeFailed) {
+      return 'UpgradeFailed'
+    }
+    if (project.overleaf.history.conversionFailed) {
+      return 'ConversionFailed'
+    }
+  }
+  if (
+    project.overleaf &&
+    project.overleaf.history &&
+    project.overleaf.history.id
+  ) {
+    if (project.overleaf.history.display) {
+      // v2: full project history, do nothing
+      return 'V2'
+    } else {
+      if (projectCreatedAfterFullProjectHistoryEnabled(project)) {
+        // IF project initialised after full project history enabled for all projects
+        //    THEN project history should contain all information we need, without intervention
+        return 'V1WithoutConversion'
+      } else {
+        // ELSE SL history may predate full project history
+        //    THEN delete full project history and convert their SL history to full project history
+        // --
+        // TODO: how to verify this, can get rough start date of SL history, but not full project history
+        const preserveHistory = await shouldPreserveHistory(project)
+        const anyDocHistory = await anyDocHistoryExists(project)
+        const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
+        if (preserveHistory) {
+          if (anyDocHistory || anyDocHistoryIndex) {
+            // if SL history exists that we need to preserve, then we must convert
+            return 'V1WithConversion'
+          } else {
+            // otherwise just upgrade without conversion
+            return 'V1WithoutConversion'
+          }
+        } else {
+          // if preserveHistory false, then max 7 days of SL history
+          // but v1 already record to both histories, so safe to upgrade
+          return 'V1WithoutConversion'
+        }
+      }
+    }
+  } else {
+    const preserveHistory = await shouldPreserveHistory(project)
+    const anyDocHistory = await anyDocHistoryExists(project)
+    const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
+    if (anyDocHistory || anyDocHistoryIndex) {
+      // IF there is SL history ->
+      if (preserveHistory) {
+        // that needs to be preserved:
+        //    THEN initialise full project history and convert SL history to full project history
+        return 'NoneWithConversion'
+      } else {
+        return 'NoneWithTemporaryHistory'
+      }
+    } else {
+      // ELSE there is not any SL history ->
+      //    THEN initialise full project history and sync with current content
+      return 'NoneWithoutConversion'
+    }
+  }
+}
+
+async function upgradeProject(project) {
+  const historyType = await determineProjectHistoryType(project)
+  if (historyType === 'V2') {
+    return { historyType, upgraded: true }
+  }
+  const upgradeFn = getUpgradeFunctionForType(historyType)
+  if (!upgradeFn) {
+    return { error: 'unsupported history type' }
+  }
+  const result = await upgradeFn(project)
+  result.historyType = historyType
+  return result
+}
+
+// Do upgrades/conversion:
+
+function getUpgradeFunctionForType(historyType) {
+  return UpgradeFunctionMapping[historyType]
+}
+
+const UpgradeFunctionMapping = {
+  NoneWithoutConversion: doUpgradeForNoneWithoutConversion,
+  UpgradeFailed: doUpgradeForNoneWithoutConversion,
+  ConversionFailed: doUpgradeForNoneWithConversion,
+  V1WithoutConversion: doUpgradeForV1WithoutConversion,
+  V1WithConversion: doUpgradeForV1WithConversion,
+  NoneWithConversion: doUpgradeForNoneWithConversion,
+  NoneWithTemporaryHistory: doUpgradeForNoneWithConversion,
+}
+
+async function doUpgradeForV1WithoutConversion(project) {
+  await db.projects.updateOne(
+    { _id: project._id },
+    {
+      $set: {
+        'overleaf.history.display': true,
+        'overleaf.history.upgradedAt': new Date(),
+        'overleaf.history.upgradeReason': `v1-without-sl-history`,
+      },
+    }
+  )
+  return { upgraded: true }
+}
+
+async function doUpgradeForV1WithConversion(project) {
+  const result = {}
+  const projectId = project._id
+  // migrateProjectHistory expects project id as a string
+  const projectIdString = project._id.toString()
+  try {
+    // We treat these essentially as None projects, the V1 history is irrelevant,
+    // so we will delete it, and do a conversion as if we're a None project
+    await ProjectHistoryController.deleteProjectHistory(projectIdString)
+    await ProjectHistoryController.migrateProjectHistory(projectIdString)
+  } catch (err) {
+    // if migrateProjectHistory fails, it cleans up by deleting
+    // the history and unsetting the history id
+    // therefore a failed project will still look like a 'None with conversion' project
+    result.error = err
+    await db.projects.updateOne(
+      { _id: projectId },
+      {
+        $set: {
+          'overleaf.history.conversionFailed': true,
+        },
+      }
+    )
+    return result
+  }
+  await db.projects.updateOne(
+    { _id: projectId },
+    {
+      $set: {
+        'overleaf.history.upgradeReason': `v1-with-conversion`,
+      },
+      $unset: {
+        'overleaf.history.upgradeFailed': true,
+        'overleaf.history.conversionFailed': true,
+      },
+    }
+  )
+  result.upgraded = true
+  return result
+}
+
+async function doUpgradeForNoneWithoutConversion(project) {
+  const result = {}
+  const projectId = project._id
+  try {
+    // Logic originally from ProjectHistoryHandler.ensureHistoryExistsForProject
+    // However sends a force resync project to project history instead
+    // of a resync request to doc-updater
+    const historyId = await ProjectHistoryHandler.promises.getHistoryId(
+      projectId
+    )
+    if (!historyId) {
+      const history = await HistoryManager.promises.initializeProject()
+      if (history && history.overleaf_id) {
+        await ProjectHistoryHandler.promises.setHistoryId(
+          projectId,
+          history.overleaf_id
+        )
+      }
+    }
+    await HistoryManager.promises.resyncProject(projectId, {
+      force: true,
+      origin: { kind: 'history-migration' },
+    })
+    await HistoryManager.promises.flushProject(projectId)
+  } catch (err) {
+    result.error = err
+    await db.projects.updateOne(
+      { _id: project._id },
+      {
+        $set: {
+          'overleaf.history.upgradeFailed': true,
+        },
+      }
+    )
+    return result
+  }
+  await db.projects.updateOne(
+    { _id: project._id },
+    {
+      $set: {
+        'overleaf.history.display': true,
+        'overleaf.history.upgradedAt': new Date(),
+        'overleaf.history.upgradeReason': `none-without-conversion`,
+      },
+    }
+  )
+  result.upgraded = true
+  return result
+}
+
+async function doUpgradeForNoneWithConversion(project) {
+  const result = {}
+  const projectId = project._id
+  // migrateProjectHistory expects project id as a string
+  const projectIdString = project._id.toString()
+  try {
+    await ProjectHistoryController.migrateProjectHistory(projectIdString)
+  } catch (err) {
+    // if migrateProjectHistory fails, it cleans up by deleting
+    // the history and unsetting the history id
+    // therefore a failed project will still look like a 'None with conversion' project
+    result.error = err
+    await db.projects.updateOne(
+      { _id: projectId },
+      {
+        $set: {
+          'overleaf.history.conversionFailed': true,
+        },
+      }
+    )
+    return result
+  }
+  await db.projects.updateOne(
+    { _id: projectId },
+    {
+      $set: {
+        'overleaf.history.upgradeReason': `none-with-conversion`,
+      },
+      $unset: {
+        'overleaf.history.upgradeFailed': true,
+        'overleaf.history.conversionFailed': true,
+      },
+    }
+  )
+  result.upgraded = true
+  return result
+}
+
+// Util
+
+function projectCreatedAfterFullProjectHistoryEnabled(project) {
+  return (
+    project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
+  )
+}
+
+async function shouldPreserveHistory(project) {
+  return await db.projectHistoryMetaData.findOne(
+    {
+      $and: [
+        { project_id: { $eq: project._id } },
+        { preserveHistory: { $eq: true } },
+      ],
+    },
+    { readPreference: ReadPreference.SECONDARY }
+  )
+}
+
+async function anyDocHistoryExists(project) {
+  return await db.docHistory.findOne(
+    { project_id: { $eq: project._id } },
+    {
+      projection: { _id: 1 },
+      readPreference: ReadPreference.SECONDARY,
+    }
+  )
+}
+
+async function anyDocHistoryIndexExists(project) {
+  return await db.docHistoryIndex.findOne(
+    { project_id: { $eq: project._id } },
+    {
+      projection: { _id: 1 },
+      readPreference: ReadPreference.SECONDARY,
+    }
+  )
+}
+
+module.exports = {
+  determineProjectHistoryType,
+  getUpgradeFunctionForType,
+  upgradeProject,
+}

services/web/scripts/history/count_project_history_categories.js

@@ -9,230 +9,36 @@ process.env.BATCH_SIZE = BATCH_SIZE
 process.env.MONGO_SOCKET_TIMEOUT =
   parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000
 
-const { ReadPreference, ObjectId } = require('mongodb')
-const { db } = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
+const { determineProjectHistoryType } = require('./HistoryUpgradeHelper')
 
 const COUNT = {
-  v2: 0,
+  V2: 0,
-  v1WithoutConversion: 0,
+  V1WithoutConversion: 0,
-  v1WithConversion: 0,
+  V1WithConversion: 0,
   NoneWithoutConversion: 0,
   NoneWithConversion: 0,
   NoneWithTemporaryHistory: 0,
-  HistoryUpgradeFailed: 0,
+  UpgradeFailed: 0,
-  HistoryConversionFailed: 0,
+  ConversionFailed: 0,
 }
 
-// Timestamp of when 'Enable history for SL in background' release
-const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
-const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
-  ID_WHEN_FULL_PROJECT_HISTORY_ENABLED
-)
-const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
-  OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
-
 async function processBatch(_, projects) {
   await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject)
   console.log(COUNT)
 }
 
 async function processProject(project) {
-  if (project.overleaf && project.overleaf.history) {
+  const historyType = await determineProjectHistoryType(project)
-    if (project.overleaf.history.upgradeFailed) {
-      // a failed history upgrade might look like a v1 project, but history may be broken
-      COUNT.HistoryUpgradeFailed += 1
-      if (VERBOSE_LOGGING) {
-        console.log(
-          `project ${
-            project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-          } has a history upgrade failure recorded`
-        )
-      }
-      return
-    } else if (project.overleaf.history.conversionFailed) {
-      COUNT.HistoryConversionFailed += 1
-      if (VERBOSE_LOGGING) {
-        console.log(
-          `project ${
-            project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-          } has a history conversion failure recorded`
-        )
-      }
-      return
-    }
-  }
-  if (
-    project.overleaf &&
-    project.overleaf.history &&
-    project.overleaf.history.id
-  ) {
-    if (project.overleaf.history.display) {
-      // v2: full project history, do nothing, (query shoudln't include any, but we should stlll check?)
-      COUNT.v2 += 1
-      if (VERBOSE_LOGGING) {
-        console.log(
-          `project ${
-            project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-          } is already v2`
-        )
-      }
-    } else {
-      if (projectCreatedAfterFullProjectHistoryEnabled(project)) {
-        // IF project initialised after full project history enabled for all projects
-        //    THEN project history should contain all information we need, without intervention
-        await doUpgradeForV1WithoutConversion(project) // CASE #1
-      } else {
-        // ELSE SL history may predate full project history
-        //    THEN delete full project history and convert their SL history to full project history
-        // --
-        // TODO: how to verify this, can get rough start date of SL history, but not full project history
-        const preserveHistory = await shouldPreserveHistory(project)
-        const anyDocHistory = await anyDocHistoryExists(project)
-        const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-        if (preserveHistory) {
-          if (anyDocHistory || anyDocHistoryIndex) {
-            // if SL history exists that we need to preserve, then we must convert
-            await doUpgradeForV1WithConversion(project) // CASE #4
-          } else {
-            // otherwise just upgrade without conversion
-            await doUpgradeForV1WithoutConversion(project) // CASE #1
-          }
-        } else {
-          // if preserveHistory false, then max 7 days of SL history
-          // but v1 already record to both histories, so safe to upgrade
-          await doUpgradeForV1WithoutConversion(project) // CASE #1
-        }
-      }
-    }
-  } else {
-    const preserveHistory = await shouldPreserveHistory(project)
-    const anyDocHistory = await anyDocHistoryExists(project)
-    const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-    if (anyDocHistory || anyDocHistoryIndex) {
-      // IF there is SL history ->
-      if (preserveHistory) {
-        // that needs to be preserved:
-        //    THEN initialise full project history and convert SL history to full project history
-        await doUpgradeForNoneWithConversion(project) // CASE #3
-      } else {
-        await doUpgradeForNoneWithTemporaryHistory(project) // Either #3 or #2
-      }
-    } else {
-      // ELSE there is not any SL history ->
-      //    THEN initialise full project history and sync with current content
-      await doUpgradeForNoneWithoutConversion(project) // CASE #2
-    }
-  }
-}
-
-// Helpers:
-
-async function shouldPreserveHistory(project) {
-  return await db.projectHistoryMetaData.findOne(
-    {
-      $and: [
-        { project_id: { $eq: project._id } },
-        { preserveHistory: { $eq: true } },
-      ],
-    },
-    { readPreference: ReadPreference.SECONDARY }
-  )
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
-    }
-  )
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
-    }
-  )
-}
-
-function projectCreatedAfterFullProjectHistoryEnabled(project) {
-  return (
-    project._id.getTimestamp() >= DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED
-  )
-}
-
-// Do upgrades/conversion:
-
-async function doUpgradeForV1WithoutConversion(project) {
-  // Simply:
-  // project.overleaf.history.display = true
-  // TODO: Sanity check(?)
-  COUNT.v1WithoutConversion += 1
   if (VERBOSE_LOGGING) {
     console.log(
       `project ${
         project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      } is v1 and does not require conversion`
+      } is type ${historyType}`
-    )
-  }
-}
-
-async function doUpgradeForV1WithConversion(project) {
-  // Delete full project history (or create new)
-  // Use conversion script to convert SL history to full project history
-  COUNT.v1WithConversion += 1
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${
-        project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      } is v1 and requires conversion`
-    )
-  }
-}
-
-async function doUpgradeForNoneWithoutConversion(project) {
-  // Initialise full project history with current content
-  COUNT.NoneWithoutConversion += 1
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${
-        project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      } is None and and does not require conversion`
-    )
-  }
-}
-
-async function doUpgradeForNoneWithConversion(project) {
-  // Initialise full project history
-  // Use conversion script to convert SL history to full project history
-  COUNT.NoneWithConversion += 1
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${
-        project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      } is None and and requires conversion`
-    )
-  }
-}
-
-async function doUpgradeForNoneWithTemporaryHistory(project) {
-  // If project doesn't have preserveHistory set
-  // but it has SL history:
-  // The history is temporary, we could convert, or do a 7 day staged rollout
-  COUNT.NoneWithTemporaryHistory += 1
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${
-        project[VERBOSE_PROJECT_NAMES ? 'name' : '_id']
-      } is None and and has temporary history (MAYBE requires conversion)`
     )
   }
+  COUNT[historyType] += 1
 }
 
 async function main() {

services/web/scripts/history/upgrade_project.js

@@ -0,0 +1,38 @@
+const { ReadPreference, ObjectId } = require('mongodb')
+const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
+const { upgradeProject } = require('./HistoryUpgradeHelper')
+
+async function processProject(project) {
+  const result = await upgradeProject(project)
+  console.log(result)
+}
+
+async function main() {
+  await waitForDb()
+  const args = process.argv.slice(2)
+  const projectId = args[0]
+  const query = { _id: ObjectId(projectId) }
+  const projection = {
+    _id: 1,
+    overleaf: 1,
+  }
+  const options = {
+    projection,
+    readPreference: ReadPreference.SECONDARY,
+  }
+  const project = await db.projects.findOne(query, options)
+  if (project) {
+    await processProject(project)
+  } else {
+    console.error(`project ${projectId} not found`)
+  }
+}
+
+main()
+  .then(() => {
+    process.exit(0)
+  })
+  .catch(error => {
+    console.error({ error })
+    process.exit(1)
+  })