[CE/SP] Hotfix 5.0.3 (#18075)

* [CE/SP] Hotfix 5.0.3

* [CE/SP] Apply hotfix changes to init script

There's some divergences in the current state of
910_initiate_doc_version_recovery in server-ce/init_scripts
and 5.0.2 hotfix. With this change we ensure the same content
in server-ce/init_scripts and the current hotfix

* Updated patch for recover_doc_versions.js

GitOrigin-RevId: 175af6b3ac584575764cbd03a5105c6933618c28
This commit is contained in:
Miguel Serrano 2024-04-24 12:58:40 +02:00 committed by Copybot
parent a697f9e7b0
commit f3ccafee94
4 changed files with 367 additions and 2 deletions

View file

@ -0,0 +1,51 @@
#!/bin/bash
set -euo pipefail
source /etc/container_environment.sh
source /etc/overleaf/env.sh
LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log
RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs-5.0.3.log
echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
cd /overleaf/services/history-v1
LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
function resyncAllProjectsInBackground() {
waitForService docstore 3016
waitForService document-updater 3003
waitForService filestore 3009
waitForService history-v1 3100
waitForService project-history 3054
waitForService web-api 4000
# Resync files that had their versions updated
while read -r project_id; do
echo "Resyncing project $project_id..."
curl -X POST --silent "http://127.0.0.1:3054/project/$project_id/resync?force=true"
done < "$RESYNCS_NEEDED_FILE"
# Resync files that have broken histories
/overleaf/bin/force-history-resyncs
echo "Finished resyncing history for all projects. Adding .done suffix to log file"
mv "$RESYNCS_NEEDED_FILE" "$RESYNCS_NEEDED_FILE.done"
}
function waitForService() {
local name=$1
local port=$2
while ! curl --fail --silent "http://127.0.0.1:$port/status"; do
echo "Waiting for $name service to start up"
sleep 10
done
}
if [ -f "$RESYNCS_NEEDED_FILE" ]; then
echo "Finished recovery of doc versions. Resyncing history for all projects in the background."
resyncAllProjectsInBackground &
else
echo "No recovery of doc versions needed."
fi

View file

@ -0,0 +1,7 @@
FROM sharelatex/sharelatex:5.0.2-RC6
# Patch: https://github.com/overleaf/internal/pull/18065
RUN npm install @overleaf/redis-wrapper @overleaf/settings -w services/history-v1
ADD 910_initiate_doc_version_recovery /etc/my_init.d/910_initiate_doc_version_recovery
COPY pr_18065.patch .
RUN patch -p0 < pr_18065.patch && rm pr_18065.patch

View file

@ -0,0 +1,307 @@
--- services/history-v1/storage/scripts/recover_doc_versions.js
+++ services/history-v1/storage/scripts/recover_doc_versions.js
@@ -2,6 +2,10 @@ const fsPromises = require('fs/promises')
const { ObjectId } = require('mongodb')
const BPromise = require('bluebird')
const logger = require('@overleaf/logger')
+const Settings = require('@overleaf/settings')
+const rclient = require('@overleaf/redis-wrapper').createClient(
+ Settings.redis.documentupdater
+)
const mongodb = require('../lib/mongodb')
const { chunkStore } = require('..')
const Events = require('events')
@@ -28,8 +32,14 @@ const db = {
const BAD_MIGRATION_NAME =
'20231219081700_move_doc_versions_from_docops_to_docs'
+const RECOVERY_FILES_502 = [
+ '/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log',
+ '/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log.done',
+]
+
let loggingChain = Promise.resolve()
const projectIdsThatNeedResyncing = []
+const unflushedDocIds = new Set()
async function flushLogQueue() {
const logPath = OPTIONS['resyncs-needed-file']
@@ -55,23 +65,67 @@ async function recordProjectNeedsResync(projectId) {
}
async function main() {
+ const recovery502Ran = await did502RecoveryRun()
+ await getUnflushedDocIds()
const badMigration = await db.migrations.findOne({ name: BAD_MIGRATION_NAME })
- if (OPTIONS.force || badMigration != null) {
+
+ if (unflushedDocIds.size > 0 && !recovery502Ran && badMigration != null) {
+ // Tell customers that they need to flush
+ console.log(`
+--------------------------------------------------------------------
+Detected unflushed changes while recovering doc versions.
+Please go back to version 5.0.1 and follow the recovery procedure
+for flushing document updates:
+
+https://github.com/overleaf/overleaf/wiki/Doc-version-recovery
+--------------------------------------------------------------------`)
+ process.exit(1)
+ }
+
+ if (OPTIONS.force || recovery502Ran || badMigration != null) {
console.warn('Need to recover doc versions. This will take a while.')
await runRecovery()
+ await db.migrations.deleteOne({ name: BAD_MIGRATION_NAME })
+ await delete502RecoveryFiles()
}
- await db.migrations.deleteOne({ name: BAD_MIGRATION_NAME })
+
console.log('Done.')
}
+async function did502RecoveryRun() {
+ for (const file of RECOVERY_FILES_502) {
+ try {
+ await fsPromises.stat(file)
+ return true
+ } catch (err) {
+ // file doesn't exist. continue
+ }
+ }
+ return false
+}
+
+async function delete502RecoveryFiles() {
+ for (const file of RECOVERY_FILES_502) {
+ try {
+ await fsPromises.rename(file, file.replace('.log', '-5.0.2.log'))
+ } catch (err) {
+ // file doesn't exist. continue
+ }
+ }
+}
+
async function runRecovery() {
let batch = []
const summary = {
- updated: 0,
ignored: 0,
skipped: 0,
- deletedUpdated: 0,
+ deletedUpdatedMongo: 0,
+ deletedUpdatedRedis: 0,
+ deletedUpdatedBoth: 0,
deletedIgnored: 0,
+ updatedMongo: 0,
+ updatedRedis: 0,
+ updatedBoth: 0,
}
const processBatchAndLogProgress = async () => {
try {
@@ -79,9 +133,21 @@ async function runRecovery() {
concurrency: OPTIONS.concurrency,
})
} finally {
- console.log(`${summary.updated} projects updated`)
+ console.log(`${summary.updatedRedis} projects updated in Redis`)
+ console.log(`${summary.updatedMongo} projects updated in Mongo`)
+ console.log(
+ `${summary.updatedBoth} projects updated in both Mongo and Redis`
+ )
console.log(`${summary.ignored} projects had good versions`)
- console.log(`${summary.deletedUpdated} deleted projects updated`)
+ console.log(
+ `${summary.deletedUpdatedMongo} deleted projects updated in Mongo`
+ )
+ console.log(
+ `${summary.deletedUpdatedRedis} deleted projects updated in Redis`
+ )
+ console.log(
+ `${summary.deletedUpdatedBoth} deleted projects updated in both Mongo and Redis`
+ )
console.log(
`${summary.deletedIgnored} deleted projects had good versions`
)
@@ -91,7 +157,7 @@ async function runRecovery() {
}
await printDBStats()
- await touchResyncsNeededFile()
+ await initResyncsNeededFile()
for await (const project of getProjects()) {
batch.push(project)
if (batch.length >= BATCH_SIZE) {
@@ -115,17 +181,38 @@ async function runRecovery() {
await backfillMissingVersions()
}
+async function getUnflushedDocIds() {
+ const batchSize = 1000
+ let cursor = '0'
+ do {
+ const [newCursor, keys] = await rclient.scan(
+ cursor,
+ 'MATCH',
+ Settings.redis.documentupdater.key_schema.docVersion({ doc_id: '*' }),
+ 'COUNT',
+ batchSize
+ )
+ for (const key of keys) {
+ unflushedDocIds.add(key.slice('DocVersion:'.length))
+ }
+ cursor = newCursor
+ } while (cursor !== '0')
+}
+
async function printDBStats() {
const projects = await db.projects.estimatedDocumentCount()
+ const deletedProjects = await db.deletedProjects.countDocuments()
const docs = await db.docs.estimatedDocumentCount()
console.log(
- `Need to check ${projects} projects with a total of ${docs} docs.`
+ `Need to check ${projects} projects and up-to ${deletedProjects} deleted projects with a total of ${docs} docs.`
)
}
-async function touchResyncsNeededFile() {
- if (OPTIONS['resyncs-needed-file']) {
- await fsPromises.appendFile(OPTIONS['resyncs-needed-file'], '')
+async function initResyncsNeededFile() {
+ const logPath = OPTIONS['resyncs-needed-file']
+ if (logPath) {
+ await fsPromises.writeFile(logPath, '')
+ await fsPromises.rm(`${logPath}.done`, { force: true })
}
}
@@ -135,34 +222,47 @@ function getProjects() {
function getDeletedProjects() {
return db.deletedProjects.find(
- { project: { $ne: null } },
+ { 'project.overleaf.history.id': { $exists: true } },
{ projection: { 'project._id': 1, 'project.overleaf': 1 } }
)
}
async function processProject(project, summary) {
const projectId = project._id.toString()
- let updated = false
+ let updatedMongo = false
+ let updatedRedis = false
try {
const historyDocVersions = await getHistoryDocVersions(project)
for (const { docId, version } of historyDocVersions) {
- const update = await fixMongoDocVersion(docId, version)
+ const update = await fixDocVersion(docId, version)
if (update != null) {
- updated = true
+ if (update.in === 'mongo') {
+ updatedMongo = true
+ } else if (update.in === 'redis') {
+ updatedRedis = true
+ }
}
}
if (project.isDeleted) {
- if (updated) {
- summary.deletedUpdated += 1
+ if (updatedMongo && updatedRedis) {
+ summary.deletedUpdatedBoth += 1
+ } else if (updatedMongo) {
+ summary.deletedUpdatedMongo += 1
+ } else if (updatedRedis) {
+ summary.deletedUpdatedRedis += 1
} else {
summary.deletedIgnored += 1
}
} else {
await recordProjectNeedsResync(projectId)
- if (updated) {
- summary.updated += 1
+ if (updatedMongo && updatedRedis) {
+ summary.updatedBoth += 1
+ } else if (updatedMongo) {
+ summary.updatedMongo += 1
+ } else if (updatedRedis) {
+ summary.updatedRedis += 1
} else {
summary.ignored += 1
}
@@ -197,25 +297,61 @@ async function getHistoryDocVersions(project) {
}))
}
-async function fixMongoDocVersion(docId, historyVersion) {
- const docBeforeUpdate = await db.docs.findOneAndUpdate(
- {
- _id: new ObjectId(docId),
- $or: [
- { version: { $lte: historyVersion } },
- { version: { $exists: false } },
- ],
- },
- { $set: { version: historyVersion + 1 } }
- )
- if (docBeforeUpdate != null) {
+async function fixDocVersion(docId, historyVersion) {
+ const redisVersion = await getRedisDocVersion(docId)
+ if (redisVersion != null && historyVersion >= redisVersion) {
+ await setRedisDocVersion(docId, historyVersion + 1)
return {
- previousVersion: docBeforeUpdate.version,
+ in: 'redis',
+ previousVersion: redisVersion,
newVersion: historyVersion + 1,
}
} else {
+ const docBeforeUpdate = await db.docs.findOneAndUpdate(
+ {
+ _id: new ObjectId(docId),
+ $or: [
+ { version: { $lte: historyVersion } },
+ { version: { $exists: false } },
+ ],
+ },
+ { $set: { version: historyVersion + 1 } },
+ { projection: { _id: 1, version: 1 } }
+ )
+
+ if (docBeforeUpdate != null) {
+ return {
+ in: 'mongo',
+ previousVersion: docBeforeUpdate.version,
+ newVersion: historyVersion + 1,
+ }
+ } else {
+ return null
+ }
+ }
+}
+
+async function getRedisDocVersion(docId) {
+ if (!unflushedDocIds.has(docId)) {
return null
}
+ const result = await rclient.get(
+ Settings.redis.documentupdater.key_schema.docVersion({ doc_id: docId })
+ )
+ if (result == null) {
+ return null
+ }
+ return parseInt(result, 10)
+}
+
+async function setRedisDocVersion(docId, version) {
+ const multi = rclient.multi()
+ multi.set(
+ Settings.redis.documentupdater.key_schema.docVersion({ doc_id: docId }),
+ version
+ )
+ multi.set(`UnflushedTime:{${docId}}`, Date.now(), 'NX')
+ await multi.exec()
}
/**

View file

@ -6,11 +6,11 @@ source /etc/container_environment.sh
source /etc/overleaf/env.sh
LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log
export RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log
RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs-5.0.3.log
echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
cd /overleaf/services/history-v1
LOG_LEVEL=info node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
LOG_LEVEL=info DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE="$RESYNCS_NEEDED_FILE" node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
function resyncAllProjectsInBackground() {
waitForService docstore 3016