[CE/SP] Hotfix 5.0.3 (#18075)

* [CE/SP] Hotfix 5.0.3 * [CE/SP] Apply hotfix changes to init script There's some divergences in the current state of 910_initiate_doc_version_recovery in server-ce/init_scripts and 5.0.2 hotfix. With this change we ensure the same content in server-ce/init_scripts and the current hotfix * Updated patch for recover_doc_versions.js GitOrigin-RevId: 175af6b3ac584575764cbd03a5105c6933618c28
2024-11-21 20:47:08 -05:00 · 2024-04-24 12:58:40 +02:00 · 2024-04-24 12:58:40 +02:00 · f3ccafee94
commit f3ccafee94
parent a697f9e7b0
4 changed files with 367 additions and 2 deletions
--- a/server-ce/hotfix/5.0.3/910_initiate_doc_version_recovery
+++ b/server-ce/hotfix/5.0.3/910_initiate_doc_version_recovery
@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source /etc/container_environment.sh
+source /etc/overleaf/env.sh
+
+LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log
+RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs-5.0.3.log
+
+echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
+cd /overleaf/services/history-v1
+LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
+
+function resyncAllProjectsInBackground() {
+  waitForService docstore 3016
+  waitForService document-updater 3003
+  waitForService filestore 3009
+  waitForService history-v1 3100
+  waitForService project-history 3054
+  waitForService web-api 4000
+
+  # Resync files that had their versions updated
+  while read -r project_id; do
+    echo "Resyncing project $project_id..."
+    curl -X POST --silent "http://127.0.0.1:3054/project/$project_id/resync?force=true"
+  done < "$RESYNCS_NEEDED_FILE"
+
+  # Resync files that have broken histories
+  /overleaf/bin/force-history-resyncs
+
+  echo "Finished resyncing history for all projects. Adding .done suffix to log file"
+  mv "$RESYNCS_NEEDED_FILE" "$RESYNCS_NEEDED_FILE.done"
+}
+
+function waitForService() {
+  local name=$1
+  local port=$2
+  while ! curl --fail --silent "http://127.0.0.1:$port/status"; do
+    echo "Waiting for $name service to start up"
+    sleep 10
+  done
+}
+
+if [ -f "$RESYNCS_NEEDED_FILE" ]; then
+  echo "Finished recovery of doc versions. Resyncing history for all projects in the background."
+  resyncAllProjectsInBackground &
+else
+  echo "No recovery of doc versions needed."
+fi
+
--- a/server-ce/hotfix/5.0.3/Dockerfile
+++ b/server-ce/hotfix/5.0.3/Dockerfile
@ -0,0 +1,7 @@
+FROM sharelatex/sharelatex:5.0.2-RC6
+
+# Patch: https://github.com/overleaf/internal/pull/18065
+RUN npm install @overleaf/redis-wrapper @overleaf/settings -w services/history-v1
+ADD 910_initiate_doc_version_recovery /etc/my_init.d/910_initiate_doc_version_recovery
+COPY pr_18065.patch .
+RUN patch -p0 < pr_18065.patch && rm pr_18065.patch
--- a/server-ce/hotfix/5.0.3/pr_18065.patch
+++ b/server-ce/hotfix/5.0.3/pr_18065.patch
@ -0,0 +1,307 @@
+--- services/history-v1/storage/scripts/recover_doc_versions.js
+++ services/history-v1/storage/scripts/recover_doc_versions.js
+@@ -2,6 +2,10 @@ const fsPromises = require('fs/promises')
+ const { ObjectId } = require('mongodb')
+ const BPromise = require('bluebird')
+ const logger = require('@overleaf/logger')
+const Settings = require('@overleaf/settings')
+const rclient = require('@overleaf/redis-wrapper').createClient(
+  Settings.redis.documentupdater
+)
+ const mongodb = require('../lib/mongodb')
+ const { chunkStore } = require('..')
+ const Events = require('events')
+@@ -28,8 +32,14 @@ const db = {
+ const BAD_MIGRATION_NAME =
+   '20231219081700_move_doc_versions_from_docops_to_docs'
+
+const RECOVERY_FILES_502 = [
+  '/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log',
+  '/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log.done',
+]
+
+ let loggingChain = Promise.resolve()
+ const projectIdsThatNeedResyncing = []
+const unflushedDocIds = new Set()
+
+ async function flushLogQueue() {
+   const logPath = OPTIONS['resyncs-needed-file']
+@@ -55,23 +65,67 @@ async function recordProjectNeedsResync(projectId) {
+ }
+
+ async function main() {
+  const recovery502Ran = await did502RecoveryRun()
+  await getUnflushedDocIds()
+   const badMigration = await db.migrations.findOne({ name: BAD_MIGRATION_NAME })
+-  if (OPTIONS.force || badMigration != null) {
+
+  if (unflushedDocIds.size > 0 && !recovery502Ran && badMigration != null) {
+    // Tell customers that they need to flush
+    console.log(`
+--------------------------------------------------------------------
+Detected unflushed changes while recovering doc versions.
+Please go back to version 5.0.1 and follow the recovery procedure
+for flushing document updates:
+
+https://github.com/overleaf/overleaf/wiki/Doc-version-recovery
+--------------------------------------------------------------------`)
+    process.exit(1)
+  }
+
+  if (OPTIONS.force || recovery502Ran || badMigration != null) {
+     console.warn('Need to recover doc versions. This will take a while.')
+     await runRecovery()
+    await db.migrations.deleteOne({ name: BAD_MIGRATION_NAME })
+    await delete502RecoveryFiles()
+   }
+-  await db.migrations.deleteOne({ name: BAD_MIGRATION_NAME })
+
+   console.log('Done.')
+ }
+
+async function did502RecoveryRun() {
+  for (const file of RECOVERY_FILES_502) {
+    try {
+      await fsPromises.stat(file)
+      return true
+    } catch (err) {
+      // file doesn't exist. continue
+    }
+  }
+  return false
+}
+
+async function delete502RecoveryFiles() {
+  for (const file of RECOVERY_FILES_502) {
+    try {
+      await fsPromises.rename(file, file.replace('.log', '-5.0.2.log'))
+    } catch (err) {
+      // file doesn't exist. continue
+    }
+  }
+}
+
+ async function runRecovery() {
+   let batch = []
+   const summary = {
+-    updated: 0,
+     ignored: 0,
+     skipped: 0,
+-    deletedUpdated: 0,
+    deletedUpdatedMongo: 0,
+    deletedUpdatedRedis: 0,
+    deletedUpdatedBoth: 0,
+     deletedIgnored: 0,
+    updatedMongo: 0,
+    updatedRedis: 0,
+    updatedBoth: 0,
+   }
+   const processBatchAndLogProgress = async () => {
+     try {
+@@ -79,9 +133,21 @@ async function runRecovery() {
+         concurrency: OPTIONS.concurrency,
+       })
+     } finally {
+-      console.log(`${summary.updated} projects updated`)
+      console.log(`${summary.updatedRedis} projects updated in Redis`)
+      console.log(`${summary.updatedMongo} projects updated in Mongo`)
+      console.log(
+        `${summary.updatedBoth} projects updated in both Mongo and Redis`
+      )
+       console.log(`${summary.ignored} projects had good versions`)
+-      console.log(`${summary.deletedUpdated} deleted projects updated`)
+      console.log(
+        `${summary.deletedUpdatedMongo} deleted projects updated in Mongo`
+      )
+      console.log(
+        `${summary.deletedUpdatedRedis} deleted projects updated in Redis`
+      )
+      console.log(
+        `${summary.deletedUpdatedBoth} deleted projects updated in both Mongo and Redis`
+      )
+       console.log(
+         `${summary.deletedIgnored} deleted projects had good versions`
+       )
+@@ -91,7 +157,7 @@ async function runRecovery() {
+   }
+
+   await printDBStats()
+-  await touchResyncsNeededFile()
+  await initResyncsNeededFile()
+   for await (const project of getProjects()) {
+     batch.push(project)
+     if (batch.length >= BATCH_SIZE) {
+@@ -115,17 +181,38 @@ async function runRecovery() {
+   await backfillMissingVersions()
+ }
+
+async function getUnflushedDocIds() {
+  const batchSize = 1000
+  let cursor = '0'
+  do {
+    const [newCursor, keys] = await rclient.scan(
+      cursor,
+      'MATCH',
+      Settings.redis.documentupdater.key_schema.docVersion({ doc_id: '*' }),
+      'COUNT',
+      batchSize
+    )
+    for (const key of keys) {
+      unflushedDocIds.add(key.slice('DocVersion:'.length))
+    }
+    cursor = newCursor
+  } while (cursor !== '0')
+}
+
+ async function printDBStats() {
+   const projects = await db.projects.estimatedDocumentCount()
+  const deletedProjects = await db.deletedProjects.countDocuments()
+   const docs = await db.docs.estimatedDocumentCount()
+   console.log(
+-    `Need to check ${projects} projects with a total of ${docs} docs.`
+    `Need to check ${projects} projects and up-to ${deletedProjects} deleted projects with a total of ${docs} docs.`
+   )
+ }
+
+-async function touchResyncsNeededFile() {
+-  if (OPTIONS['resyncs-needed-file']) {
+-    await fsPromises.appendFile(OPTIONS['resyncs-needed-file'], '')
+async function initResyncsNeededFile() {
+  const logPath = OPTIONS['resyncs-needed-file']
+  if (logPath) {
+    await fsPromises.writeFile(logPath, '')
+    await fsPromises.rm(`${logPath}.done`, { force: true })
+   }
+ }
+
+@@ -135,34 +222,47 @@ function getProjects() {
+
+ function getDeletedProjects() {
+   return db.deletedProjects.find(
+-    { project: { $ne: null } },
+    { 'project.overleaf.history.id': { $exists: true } },
+     { projection: { 'project._id': 1, 'project.overleaf': 1 } }
+   )
+ }
+
+ async function processProject(project, summary) {
+   const projectId = project._id.toString()
+-  let updated = false
+  let updatedMongo = false
+  let updatedRedis = false
+   try {
+     const historyDocVersions = await getHistoryDocVersions(project)
+
+     for (const { docId, version } of historyDocVersions) {
+-      const update = await fixMongoDocVersion(docId, version)
+      const update = await fixDocVersion(docId, version)
+       if (update != null) {
+-        updated = true
+        if (update.in === 'mongo') {
+          updatedMongo = true
+        } else if (update.in === 'redis') {
+          updatedRedis = true
+        }
+       }
+     }
+
+     if (project.isDeleted) {
+-      if (updated) {
+-        summary.deletedUpdated += 1
+      if (updatedMongo && updatedRedis) {
+        summary.deletedUpdatedBoth += 1
+      } else if (updatedMongo) {
+        summary.deletedUpdatedMongo += 1
+      } else if (updatedRedis) {
+        summary.deletedUpdatedRedis += 1
+       } else {
+         summary.deletedIgnored += 1
+       }
+     } else {
+       await recordProjectNeedsResync(projectId)
+-      if (updated) {
+-        summary.updated += 1
+      if (updatedMongo && updatedRedis) {
+        summary.updatedBoth += 1
+      } else if (updatedMongo) {
+        summary.updatedMongo += 1
+      } else if (updatedRedis) {
+        summary.updatedRedis += 1
+       } else {
+         summary.ignored += 1
+       }
+@@ -197,25 +297,61 @@ async function getHistoryDocVersions(project) {
+   }))
+ }
+
+-async function fixMongoDocVersion(docId, historyVersion) {
+-  const docBeforeUpdate = await db.docs.findOneAndUpdate(
+-    {
+-      _id: new ObjectId(docId),
+-      $or: [
+-        { version: { $lte: historyVersion } },
+-        { version: { $exists: false } },
+-      ],
+-    },
+-    { $set: { version: historyVersion + 1 } }
+-  )
+-  if (docBeforeUpdate != null) {
+async function fixDocVersion(docId, historyVersion) {
+  const redisVersion = await getRedisDocVersion(docId)
+  if (redisVersion != null && historyVersion >= redisVersion) {
+    await setRedisDocVersion(docId, historyVersion + 1)
+     return {
+-      previousVersion: docBeforeUpdate.version,
+      in: 'redis',
+      previousVersion: redisVersion,
+       newVersion: historyVersion + 1,
+     }
+   } else {
+    const docBeforeUpdate = await db.docs.findOneAndUpdate(
+      {
+        _id: new ObjectId(docId),
+        $or: [
+          { version: { $lte: historyVersion } },
+          { version: { $exists: false } },
+        ],
+      },
+      { $set: { version: historyVersion + 1 } },
+      { projection: { _id: 1, version: 1 } }
+    )
+
+    if (docBeforeUpdate != null) {
+      return {
+        in: 'mongo',
+        previousVersion: docBeforeUpdate.version,
+        newVersion: historyVersion + 1,
+      }
+    } else {
+      return null
+    }
+  }
+}
+
+async function getRedisDocVersion(docId) {
+  if (!unflushedDocIds.has(docId)) {
+     return null
+   }
+  const result = await rclient.get(
+    Settings.redis.documentupdater.key_schema.docVersion({ doc_id: docId })
+  )
+  if (result == null) {
+    return null
+  }
+  return parseInt(result, 10)
+}
+
+async function setRedisDocVersion(docId, version) {
+  const multi = rclient.multi()
+  multi.set(
+    Settings.redis.documentupdater.key_schema.docVersion({ doc_id: docId }),
+    version
+  )
+  multi.set(`UnflushedTime:{${docId}}`, Date.now(), 'NX')
+  await multi.exec()
+ }
+
+ /**
--- a/server-ce/init_scripts/910_initiate_doc_version_recovery
+++ b/server-ce/init_scripts/910_initiate_doc_version_recovery
@ -6,11 +6,11 @@ source /etc/container_environment.sh
 source /etc/overleaf/env.sh

 LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log
-export RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log
+RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs-5.0.3.log

 echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
 cd /overleaf/services/history-v1
-LOG_LEVEL=info node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
+LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"

 function resyncAllProjectsInBackground() {
  waitForService docstore 3016