Merge pull request #11280 from overleaf/bg-issue11277

Move history migration logic to web module

GitOrigin-RevId: 6f1ba33519277b9ba13ecb2a13ae2c43ee06f675

services/web/config/settings.defaults.js

@@ -789,7 +789,12 @@ module.exports = {
     editorLeftMenuSync: [],
   },
 
-  moduleImportSequence: ['launchpad', 'server-ce-scripts', 'user-activate'],
+  moduleImportSequence: [
+    'launchpad',
+    'server-ce-scripts',
+    'user-activate',
+    'history-migration',
+  ],
 
   csp: {
     enabled: process.env.CSP_ENABLED === 'true',

services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js

@@ -1,12 +1,12 @@
 const { ReadPreference, ObjectId } = require('mongodb')
-const { db } = require('../../app/src/infrastructure/mongodb')
+const { db } = require('../../../../app/src/infrastructure/mongodb')
 const Settings = require('@overleaf/settings')
 
-const ProjectHistoryHandler = require('../../app/src/Features/Project/ProjectHistoryHandler')
+const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler')
-const HistoryManager = require('../../app/src/Features/History/HistoryManager')
+const HistoryManager = require('../../../../app/src/Features/History/HistoryManager')
-const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
+const ProjectHistoryController = require('../../../admin-panel/app/src/ProjectHistoryController')
-const ProjectEntityHandler = require('../../app/src/Features/Project/ProjectEntityHandler')
+const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler')
-const ProjectEntityUpdateHandler = require('../../app/src/Features/Project/ProjectEntityUpdateHandler')
+const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler')
 
 // Timestamp of when 'Enable history for SL in background' release
 const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = '5a8d8a370000000000000000'
@@ -16,6 +16,21 @@ const OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = new ObjectId(
 const DATETIME_WHEN_FULL_PROJECT_HISTORY_ENABLED =
   OBJECT_ID_WHEN_FULL_PROJECT_HISTORY_ENABLED.getTimestamp()
 
+async function countProjects(query = {}) {
+  const count = await db.projects.count(query)
+  return count
+}
+
+async function countDocHistory(query = {}) {
+  const count = await db.docHistory.count(query)
+  return count
+}
+
+async function findProjects(query = {}, projection = {}) {
+  const projects = await db.projects.find(query).project(projection).toArray()
+  return projects
+}
+
 async function determineProjectHistoryType(project) {
   if (project.overleaf && project.overleaf.history) {
     if (project.overleaf.history.upgradeFailed) {
@@ -212,18 +227,28 @@ async function doUpgradeForNoneWithoutConversion(project) {
   return result
 }
 
-async function doUpgradeForNoneWithConversion(project) {
+async function doUpgradeForNoneWithConversion(project, options = {}) {
   const result = {}
   const projectId = project._id
   // migrateProjectHistory expects project id as a string
   const projectIdString = project._id.toString()
   try {
-    await ProjectHistoryController.migrateProjectHistory(projectIdString)
+    if (options.convertLargeDocsToFile) {
+      result.convertedDocCount = await convertLargeDocsToFile(
+        projectId,
+        options.userId
+      )
+    }
+    await ProjectHistoryController.migrateProjectHistory(
+      projectIdString,
+      options.migrationOptions
+    )
   } catch (err) {
     // if migrateProjectHistory fails, it cleans up by deleting
     // the history and unsetting the history id
     // therefore a failed project will still look like a 'None with conversion' project
     result.error = err
+    // We set a failed flag so future runs of the script don't automatically retry
     await db.projects.updateOne(
       { _id: projectId },
       {
@@ -238,7 +263,8 @@ async function doUpgradeForNoneWithConversion(project) {
     { _id: projectId },
     {
       $set: {
-        'overleaf.history.upgradeReason': `none-with-conversion`,
+        'overleaf.history.upgradeReason':
+          `none-with-conversion` + options.reason ? `/${options.reason}` : ``,
       },
       $unset: {
         'overleaf.history.upgradeFailed': true,
@@ -328,6 +354,9 @@ function docIsTooLarge(estimatedSize, lines, maxDocLength) {
 }
 
 module.exports = {
+  countProjects,
+  countDocHistory,
+  findProjects,
   determineProjectHistoryType,
   getUpgradeFunctionForType,
   upgradeProject,

services/web/modules/history-migration/index.js

@@ -0,0 +1 @@
+module.exports = {}

services/web/scripts/history/count_project_history_categories.js

@@ -11,7 +11,10 @@ process.env.MONGO_SOCKET_TIMEOUT =
 
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
-const { determineProjectHistoryType } = require('./HistoryUpgradeHelper')
+const {
+  determineProjectHistoryType,
+  countProjects,
+} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
 
 const COUNT = {
   V2: 0,
@@ -22,6 +25,8 @@ const COUNT = {
   NoneWithTemporaryHistory: 0,
   UpgradeFailed: 0,
   ConversionFailed: 0,
+  MigratedProjects: 0,
+  TotalProjects: 0,
 }
 
 async function processBatch(_, projects) {
@@ -60,6 +65,10 @@ async function main() {
     projection,
     options
   )
+  COUNT.MigratedProjects = await countProjects({
+    'overleaf.history.display': true,
+  })
+  COUNT.TotalProjects = await countProjects()
   console.log('Final')
   console.log(COUNT)
 }

services/web/scripts/history/upgrade_none_with_conversion_if_sl_history.js

@@ -29,12 +29,15 @@ const USER_ID = process.env.USER_ID
 const CONVERT_LARGE_DOCS_TO_FILE =
   process.env.CONVERT_LARGE_DOCS_TO_FILE === 'true'
 
-const { ObjectId, ReadPreference } = require('mongodb')
+const { ObjectId } = require('mongodb')
 const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
 const { promiseMapWithLimit } = require('../../app/src/util/promises')
 const { batchedUpdate } = require('../helpers/batchedUpdate')
-const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController')
+const {
-const HistoryUpgradeHelper = require('./HistoryUpgradeHelper')
+  anyDocHistoryExists,
+  anyDocHistoryIndexExists,
+  doUpgradeForNoneWithConversion,
+} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
 
 console.log({
   DRY_RUN,
@@ -111,99 +114,49 @@ async function processProject(project) {
       }
     }
   }
-  const anyDocHistory = await anyDocHistoryExists(project)
-  if (anyDocHistory) {
-    return await doUpgradeForNoneWithConversion(project)
-  }
-  const anyDocHistoryIndex = await anyDocHistoryIndexExists(project)
-  if (anyDocHistoryIndex) {
-    return await doUpgradeForNoneWithConversion(project)
-  }
-}
-
-async function doUpgradeForNoneWithConversion(project) {
   if (RESULT.failed >= MAX_FAILURES) {
     return
   }
   if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) {
     return
   }
-  RESULT.attempted += 1
+  const anyDocHistoryOrIndex =
-  const projectId = project._id
+    (await anyDocHistoryExists(project)) ||
-  // migrateProjectHistory expects project id as a string
+    (await anyDocHistoryIndexExists(project))
-  const projectIdString = project._id.toString()
+  if (anyDocHistoryOrIndex) {
-  if (!DRY_RUN) {
+    RESULT.attempted += 1
-    try {
+    if (DRY_RUN) {
-      if (CONVERT_LARGE_DOCS_TO_FILE) {
+      return
-        const convertedDocCount =
+    }
-          await HistoryUpgradeHelper.convertLargeDocsToFile(projectId, USER_ID)
+    const result = await doUpgradeForNoneWithConversion(project, {
-        console.log(
+      migrationOptions: {
-          `converted ${convertedDocCount} large docs to binary files for project ${projectId}`
-        )
-      }
-      await ProjectHistoryController.migrateProjectHistory(projectIdString, {
         archiveOnFailure: ARCHIVE_ON_FAILURE,
         fixInvalidCharacters: FIX_INVALID_CHARACTERS,
         forceNewHistoryOnFailure: FORCE_NEW_HISTORY_ON_FAILURE,
         importZipFilePath: IMPORT_ZIP_FILE_PATH,
         cutoffDate: CUTOFF_DATE,
-      })
+      },
-    } catch (err) {
+      convertLargeDocsToFile: CONVERT_LARGE_DOCS_TO_FILE,
-      // if migrateProjectHistory fails, it cleans up by deleting
+      userId: USER_ID,
-      // the history and unsetting the history id
+      reason: `${SCRIPT_VERSION}`,
-      // therefore a failed project will still look like a 'None with conversion' project
+    })
-      RESULT.failed += 1
+    if (result.convertedDocCount) {
-      console.error(`project ${projectId} FAILED with error: `, err)
+      console.log(
-      // We set a failed flag so future runs of the script don't automatically retry
+        `project ${project._id} converted ${result.convertedDocCount} docs to filestore`
-      await db.projects.updateOne(
-        { _id: projectId },
-        {
-          $set: {
-            'overleaf.history.conversionFailed': true,
-          },
-        }
       )
-      return
     }
-    await db.projects.updateOne(
+    if (result.error) {
-      { _id: projectId },
+      console.error(`project ${project._id} FAILED with error: `, result.error)
-      {
+      RESULT.failed += 1
-        $set: {
+    } else if (result.upgraded) {
-          'overleaf.history.upgradeReason': `none-with-conversion/${SCRIPT_VERSION}`,
+      if (VERBOSE_LOGGING) {
-        },
+        console.log(
-        $unset: {
+          `project ${project._id} converted and upgraded to full project history`
-          'overleaf.history.upgradeFailed': true,
+        )
-          'overleaf.history.conversionFailed': true,
-        },
       }
-    )
+      RESULT.projectsUpgraded += 1
-  }
-  if (VERBOSE_LOGGING) {
-    console.log(
-      `project ${projectId} converted and upgraded to full project history`
-    )
-  }
-  RESULT.projectsUpgraded += 1
-}
-
-async function anyDocHistoryExists(project) {
-  return await db.docHistory.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
     }
-  )
+  }
-}
-
-async function anyDocHistoryIndexExists(project) {
-  return await db.docHistoryIndex.findOne(
-    { project_id: { $eq: project._id } },
-    {
-      projection: { _id: 1 },
-      readPreference: ReadPreference.SECONDARY,
-    }
-  )
 }
 
 async function main() {

services/web/scripts/history/upgrade_project.js

@@ -1,6 +1,8 @@
 const { ReadPreference, ObjectId } = require('mongodb')
 const { db, waitForDb } = require('../../app/src/infrastructure/mongodb')
-const { upgradeProject } = require('./HistoryUpgradeHelper')
+const {
+  upgradeProject,
+} = require('../../modules/history-migration/app/src/HistoryUpgradeHelper')
 
 async function processProject(project) {
   const result = await upgradeProject(project)