github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #17798 from overleaf/msm-ce-sp-hotfix-5-0-2

Browse source 
[CE/SP] Hotfix 5.0.2 / 4.2.4

GitOrigin-RevId: 2f9ce416b95a0124edb1a9cf35c2dfa94f6f4a19
This commit is contained in:
Jakob Ackermann 2024-04-22 10:19:13 +01:00 committed by Copybot
parent 2989be8e45
commit 5b6e229c21
17 changed files with 754 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
server-ce/Dockerfile-base
View file

@ -10,7 +10,7 @@ ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var
# Update to ensure dependencies are updated # Update to ensure dependencies are updated
# ------------------------------------------ # ------------------------------------------
ENV REBUILT_AFTER="2024-14-02" ENV REBUILT_AFTER="2024-04-08"
# Install dependencies # Install dependencies
# -------------------- # --------------------

23
server-ce/hotfix/4.2.4/Dockerfile Normal file
View file

@ -0,0 +1,23 @@
FROM sharelatex/sharelatex:4.2.3
# Upgrade Node.js to version 18.20.2
RUN apt-get update \
&& apt-get install -y nodejs=18.20.2-1nodesource1 \
&& rm -rf /var/lib/apt/lists/*
# Patch: force services to use ipv4 in server-ce container
ADD env.sh /etc/sharelatex/env.sh
COPY pr_17601-1.patch /etc/sharelatex/
RUN cd /etc/sharelatex && patch -p0 < pr_17601-1.patch && rm pr_17601-1.patch
COPY pr_17601-2.patch /overleaf/cron/
RUN cd /overleaf/cron && patch -p0 < pr_17601-2.patch && rm pr_17601-2.patch
COPY pr_17601-3.patch /etc/service/
RUN cd /etc/service && patch -p0 < pr_17601-3.patch && rm pr_17601-3.patch
# Add history utility scripts
ADD bin/* /overleaf/bin/
# Patch: https://github.com/overleaf/internal/pull/17885
COPY pr_17885.patch .
RUN patch -p0 -d /etc/my_init.pre_shutdown.d < pr_17885.patch \
&& rm pr_17885.patch

7
server-ce/hotfix/4.2.4/bin/flush-history-queues Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -euo pipefail
source /etc/container_environment.sh
cd /overleaf/services/project-history
node scripts/flush_all.js 100000

7
server-ce/hotfix/4.2.4/bin/force-history-resyncs Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -euo pipefail
source /etc/container_environment.sh
cd /overleaf/services/project-history
node scripts/force_resync.js 1000 force

14
server-ce/hotfix/4.2.4/env.sh Normal file
View file

@ -0,0 +1,14 @@
export CHAT_HOST=127.0.0.1
export CLSI_HOST=127.0.0.1
export CONTACTS_HOST=127.0.0.1
export DOCSTORE_HOST=127.0.0.1
export DOCUMENT_UPDATER_HOST=127.0.0.1
export DOCUPDATER_HOST=127.0.0.1
export FILESTORE_HOST=127.0.0.1
export HISTORY_V1_HOST=127.0.0.1
export NOTIFICATIONS_HOST=127.0.0.1
export PROJECT_HISTORY_HOST=127.0.0.1
export REALTIME_HOST=127.0.0.1
export SPELLING_HOST=127.0.0.1
export WEB_HOST=127.0.0.1
export WEB_API_HOST=127.0.0.1

31
server-ce/hotfix/4.2.4/pr_17601-1.patch Normal file
View file

@ -0,0 +1,31 @@
--- settings.js
+++ settings.js
@@ -256,16 +256,16 @@ const settings = {
apis: {
web: {
- url: 'http://localhost:3000',
+ url: 'http://127.0.0.1:3000',
user: httpAuthUser,
pass: httpAuthPass,
},
project_history: {
sendProjectStructureOps: true,
- url: 'http://localhost:3054',
+ url: 'http://127.0.0.1:3054',
},
v1_history: {
- url: process.env.V1_HISTORY_URL || 'http://localhost:3100/api',
+ url: process.env.V1_HISTORY_URL || 'http://127.0.0.1:3100/api',
user: 'staging',
pass: process.env.STAGING_PASSWORD,
requestTimeout: parseInt(
@@ -409,7 +409,7 @@ if (
if (parse(process.env.OVERLEAF_IS_SERVER_PRO) === true) {
settings.bypassPercentageRollouts = true
- settings.apis.references = { url: 'http://localhost:3040' }
+ settings.apis.references = { url: 'http://127.0.0.1:3040' }
}
// Compiler

63
server-ce/hotfix/4.2.4/pr_17601-2.patch Normal file
View file

@ -0,0 +1,63 @@
--- deactivate-projects.sh
+++ deactivate-projects.sh
@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then
exit 0
fi
-WEB_URL='http://localhost:3000'
+WEB_URL='http://127.0.0.1:3000'
USER=$(cat /etc/container_environment/WEB_API_USER)
PASS=$(cat /etc/container_environment/WEB_API_PASSWORD)
--- delete-projects.sh
+++ delete-projects.sh
@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then
exit 0
fi
-WEB_URL='http://localhost:3000'
+WEB_URL='http://127.0.0.1:3000'
USER=$(cat /etc/container_environment/WEB_API_USER)
PASS=$(cat /etc/container_environment/WEB_API_PASSWORD)
--- delete-users.sh
+++ delete-users.sh
@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then
exit 0
fi
-WEB_URL='http://localhost:3000'
+WEB_URL='http://127.0.0.1:3000'
USER=$(cat /etc/container_environment/WEB_API_USER)
PASS=$(cat /etc/container_environment/WEB_API_PASSWORD)
--- project-history-periodic-flush.sh
+++ project-history-periodic-flush.sh
@@ -7,6 +7,6 @@ echo "Flush project-history queue"
echo "--------------------------"
date
-PROJECT_HISTORY_URL='http://localhost:3054'
+PROJECT_HISTORY_URL='http://127.0.0.1:3054'
curl -X POST "${PROJECT_HISTORY_URL}/flush/old?timeout=3600000&limit=5000&background=1"
--- project-history-retry-hard.sh
+++ project-history-retry-hard.sh
@@ -7,6 +7,6 @@ echo "Retry project-history errors (hard)"
echo "-----------------------------------"
date
-PROJECT_HISTORY_URL='http://localhost:3054'
+PROJECT_HISTORY_URL='http://127.0.0.1:3054'
curl -X POST "${PROJECT_HISTORY_URL}/retry/failures?failureType=hard&timeout=3600000&limit=10000"
--- project-history-retry-soft.sh
+++ project-history-retry-soft.sh
@@ -6,6 +6,6 @@ echo "-----------------------------------"
echo "Retry project-history errors (soft)"
echo "-----------------------------------"
-PROJECT_HISTORY_URL='http://localhost:3054'
+PROJECT_HISTORY_URL='http://127.0.0.1:3054'
curl -X POST "${PROJECT_HISTORY_URL}/retry/failures?failureType=soft&timeout=3600000&limit=10000"

118
server-ce/hotfix/4.2.4/pr_17601-3.patch Normal file
View file

@ -0,0 +1,118 @@
--- chat-sharelatex/run
+++ chat-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30100"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/chat/app.js >> /var/log/sharelatex/chat.log 2>&1
--- clsi-sharelatex/run
+++ clsi-sharelatex/run
@@ -15,4 +15,7 @@ if [ -e '/var/run/docker.sock' ]; then
usermod -aG dockeronhost www-data
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/clsi/app.js >> /var/log/sharelatex/clsi.log 2>&1
--- contacts-sharelatex/run
+++ contacts-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30360"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/contacts/app.js >> /var/log/sharelatex/contacts.log 2>&1
--- docstore-sharelatex/run
+++ docstore-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30160"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/docstore/app.js >> /var/log/sharelatex/docstore.log 2>&1
--- document-updater-sharelatex/run
+++ document-updater-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30030"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/document-updater/app.js >> /var/log/sharelatex/document-updater.log 2>&1
--- filestore-sharelatex/run
+++ filestore-sharelatex/run
@@ -1,2 +1,6 @@
#!/bin/bash
+
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node /overleaf/services/filestore/app.js >> /var/log/sharelatex/filestore.log 2>&1
--- notifications-sharelatex/run
+++ notifications-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30420"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/notifications/app.js >> /var/log/sharelatex/notifications.log 2>&1
--- project-history-sharelatex/run
+++ project-history-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30540"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/project-history/app.js >> /var/log/sharelatex/project-history.log 2>&1
--- real-time-sharelatex/run
+++ real-time-sharelatex/run
@@ -1,2 +1,6 @@
#!/bin/bash
+
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node /overleaf/services/real-time/app.js >> /var/log/sharelatex/real-time.log 2>&1
--- spelling-sharelatex/run
+++ spelling-sharelatex/run
@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30050"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
+
exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/spelling/app.js >> /var/log/sharelatex/spelling.log 2>&1
--- web-api-sharelatex/run
+++ web-api-sharelatex/run
@@ -6,6 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:30000"
fi
+source /etc/sharelatex/env.sh
export LISTEN_ADDRESS=0.0.0.0
export ENABLED_SERVICES="api"
export METRICS_APP_NAME="web-api"
--- web-sharelatex/run
+++ web-sharelatex/run
@@ -6,6 +6,8 @@ if [ "$DEBUG_NODE" == "true" ]; then
NODE_PARAMS="--inspect=0.0.0.0:40000"
fi
+source /etc/sharelatex/env.sh
+export LISTEN_ADDRESS=127.0.0.1
export ENABLED_SERVICES="web"
export WEB_PORT="4000"

33
server-ce/hotfix/4.2.4/pr_17885.patch Normal file
View file

@ -0,0 +1,33 @@
--- 00_close_site
+++ 00_close_site
@@ -1,5 +1,8 @@
#!/bin/sh
+. /etc/container_environment.sh
+. /etc/sharelatex/env.sh
+
SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown"
mv "${SITE_MAINTENANCE_FILE}" "${SITE_MAINTENANCE_FILE_BAK}"
--- 01_flush_document_updater
+++ 01_flush_document_updater
@@ -1,5 +1,8 @@
#!/bin/sh
+. /etc/container_environment.sh
+. /etc/sharelatex/env.sh
+
cd /overleaf/services/document-updater && node scripts/flush_all.js >> /var/log/sharelatex/document-updater.log 2>&1
EXIT_CODE="$?"
--- 02_flush_project_history
+++ 02_flush_project_history
@@ -1,5 +1,8 @@
#!/bin/sh
+. /etc/container_environment.sh
+. /etc/sharelatex/env.sh
+
cd /overleaf/services/project-history && node scripts/flush_all.js >> /var/log/sharelatex/project-history.log 2>&1
EXIT_CODE="$?"

50
server-ce/hotfix/5.0.2/910_initiate_doc_version_recovery Executable file
View file

@ -0,0 +1,50 @@
#!/bin/bash
set -euo pipefail
source /etc/container_environment.sh
source /etc/overleaf/env.sh
LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log
export DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log
echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE"
cd /overleaf/services/history-v1
LOG_LEVEL=info node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE"
function resyncAllProjectsInBackground() {
waitForService docstore 3016
waitForService document-updater 3003
waitForService filestore 3009
waitForService history-v1 3100
waitForService project-history 3054
waitForService web-api 4000
# Resync files that had their versions updated
while read -r project_id; do
echo "Resyncing project $project_id..."
curl -X POST --silent "http://127.0.0.1:3054/project/$project_id/resync?force=true"
done < "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE"
# Resync files that have broken histories
/overleaf/bin/force-history-resyncs
echo "Finished resyncing history for all projects. Adding .done suffix to log file"
mv "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE" "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE.done"
}
function waitForService() {
local name=$1
local port=$2
while ! curl --fail --silent "http://127.0.0.1:$port/status"; do
echo "Waiting for $name service to start up"
sleep 10
done
}
if [ -f "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE" ]; then
echo "Finished recovery of doc versions. Resyncing history for all projects in the background."
resyncAllProjectsInBackground &
else
echo "No recovery of doc versions needed."
fi

35
server-ce/hotfix/5.0.2/Dockerfile Normal file
View file

@ -0,0 +1,35 @@
FROM sharelatex/sharelatex:5.0.1
# Upgrade Node.js to version 18.20.2
RUN apt-get update \
&& apt-get install -y nodejs=18.20.2-1nodesource1 \
&& rm -rf /var/lib/apt/lists/*
# Patch: https://github.com/overleaf/internal/pull/17843
COPY pr_17843.patch .
RUN patch -p0 < pr_17843.patch \
&& rm pr_17843.patch
# Add history utility scripts
ADD bin/* /overleaf/bin/
# Patch: https://github.com/overleaf/internal/pull/17885
COPY pr_17885.patch .
RUN patch -p0 -d /etc/my_init.pre_shutdown.d < pr_17885.patch \
&& rm pr_17885.patch
# Recompile frontend for Grammarly patch in 5.0.1
RUN node genScript compile | bash
# Patch: https://github.com/overleaf/internal/pull/17960
COPY pr_17960.patch .
RUN patch -p0 < pr_17960.patch \
&& rm pr_17960.patch
# Fix bad ordering of migrations
RUN mv /overleaf/services/web/migrations/20231219081700_move_doc_versions_from_docops_to_docs.js \
/overleaf/services/web/migrations/20231105000000_move_doc_versions_from_docops_to_docs.js
# Add doc versions recovery scripts
ADD 910_initiate_doc_version_recovery /etc/my_init.d/910_initiate_doc_version_recovery
ADD recover_doc_versions.js /overleaf/services/history-v1/storage/scripts/recover_doc_versions.js

8
server-ce/hotfix/5.0.2/bin/flush-history-queues Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
set -euo pipefail
source /etc/container_environment.sh
source /etc/overleaf/env.sh
cd /overleaf/services/project-history
node scripts/flush_all.js 100000

8
server-ce/hotfix/5.0.2/bin/force-history-resyncs Executable file
View file

@ -0,0 +1,8 @@
#!/bin/bash
set -euo pipefail
source /etc/container_environment.sh
source /etc/overleaf/env.sh
cd /overleaf/services/project-history
node scripts/force_resync.js 1000 force

48
server-ce/hotfix/5.0.2/pr_17843.patch Normal file
View file

@ -0,0 +1,48 @@
--- services/project-history/app/js/ErrorRecorder.js
+++ services/project-history/app/js/ErrorRecorder.js
@@ -210,6 +210,14 @@ export function getFailures(callback) {
'Error: bad response from filestore: 404': 'filestore-404',
'Error: bad response from filestore: 500': 'filestore-500',
'NotFoundError: got a 404 from web api': 'web-api-404',
+ 'OError: history store a non-success status code: 413':
+ 'history-store-413',
+ 'OError: history store a non-success status code: 422':
+ 'history-store-422',
+ 'OError: history store a non-success status code: 500':
+ 'history-store-500',
+ 'OError: history store a non-success status code: 503':
+ 'history-store-503',
'Error: history store a non-success status code: 413':
'history-store-413',
'Error: history store a non-success status code: 422':
--- services/project-history/app/js/RetryManager.js
+++ services/project-history/app/js/RetryManager.js
@@ -20,6 +20,7 @@ const TEMPORARY_FAILURES = [
const HARD_FAILURES = [
'Error: history store a non-success status code: 422',
+ 'OError: history store a non-success status code: 422',
'OpsOutOfOrderError: project structure version out of order',
'OpsOutOfOrderError: project structure version out of order on incoming updates',
'OpsOutOfOrderError: doc version out of order',
--- services/project-history/scripts/clear_deleted_history.js
+++ services/project-history/scripts/clear_deleted_history.js
@@ -143,7 +143,7 @@ function checkAndClear(project, callback) {
// find all the broken projects from the failure records
async function main() {
const results = await db.projectHistoryFailures
- .find({ error: 'Error: history store a non-success status code: 422' })
+ .find({ error: /history store a non-success status code: 422/ })
.toArray()
console.log('number of queues without history store 442 =', results.length)
--- services/project-history/scripts/force_resync.js
+++ services/project-history/scripts/force_resync.js
@@ -198,6 +198,7 @@ function checkAndClear(project, callback) {
// find all the broken projects from the failure records
const errorsToResync = [
'Error: history store a non-success status code: 422',
+ 'OError: history store a non-success status code: 422',
'OpsOutOfOrderError: project structure version out of order',
]

33
server-ce/hotfix/5.0.2/pr_17885.patch Normal file
View file

@ -0,0 +1,33 @@
--- 00_close_site
+++ 00_close_site
@@ -1,5 +1,8 @@
#!/bin/sh
+. /etc/container_environment.sh
+. /etc/overleaf/env.sh
+
SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown"
mv "${SITE_MAINTENANCE_FILE}" "${SITE_MAINTENANCE_FILE_BAK}"
--- 01_flush_document_updater
+++ 01_flush_document_updater
@@ -1,5 +1,8 @@
#!/bin/sh
+. /etc/container_environment.sh
+. /etc/overleaf/env.sh
+
cd /overleaf/services/document-updater && node scripts/flush_all.js >> /var/log/overleaf/document-updater.log 2>&1
EXIT_CODE="$?"
--- 02_flush_project_history
+++ 02_flush_project_history
@@ -1,5 +1,8 @@
#!/bin/sh
+. /etc/container_environment.sh
+. /etc/overleaf/env.sh
+
cd /overleaf/services/project-history && node scripts/flush_all.js >> /var/log/overleaf/project-history.log 2>&1
EXIT_CODE="$?"

32
server-ce/hotfix/5.0.2/pr_17960.patch Normal file
View file

@ -0,0 +1,32 @@
diff --git a/services/project-history/scripts/force_resync.js b/services/project-history/scripts/force_resync.js
index 5e77b35826..13e7d3cd5c 100755
--- services/project-history/scripts/force_resync.js
+++ services/project-history/scripts/force_resync.js
@@ -77,7 +77,7 @@ function checkAndClear(project, callback) {
function startResync(cb) {
if (force) {
console.log('2. starting resync for', projectId)
- SyncManager.startResync(projectId, err => {
+ SyncManager.startHardResync(projectId, err => {
if (err) {
console.log('ERR', JSON.stringify(err.message))
return cb(err)
@@ -195,17 +195,8 @@ function checkAndClear(project, callback) {
)
}
-// find all the broken projects from the failure records
-const errorsToResync = [
- 'Error: history store a non-success status code: 422',
- 'OError: history store a non-success status code: 422',
- 'OpsOutOfOrderError: project structure version out of order',
-]
-
async function main() {
- const results = await db.projectHistoryFailures
- .find({ error: { $in: errorsToResync } })
- .toArray()
+ const results = await db.projectHistoryFailures.find().toArray()
console.log('number of queues without history store 442 =', results.length)
// now check if the project is truly deleted in mongo

243
server-ce/hotfix/5.0.2/recover_doc_versions.js Normal file
View file

@ -0,0 +1,243 @@
const fsPromises = require('fs/promises')
const { ObjectId } = require('mongodb')
const BPromise = require('bluebird')
const logger = require('@overleaf/logger')
const mongodb = require('../lib/mongodb')
const { chunkStore } = require('..')
const Events = require('events')
// Silence warning.
Events.setMaxListeners(20)
const BATCH_SIZE = 1000
const OPTIONS = {
concurrency: parseInt(process.env.DOC_VERSION_RECOVERY_CONCURRENCY, 10) || 20,
force: process.env.DOC_VERSION_RECOVERY_FORCE === 'true',
'skip-history-failures':
process.env.DOC_VERSION_RECOVERY_SKIP_HISTORY_FAILURES === 'true',
'resyncs-needed-file': process.env.DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE,
}
const db = {
deletedProjects: mongodb.db.collection('deletedProjects'),
docs: mongodb.db.collection('docs'),
migrations: mongodb.db.collection('migrations'),
projects: mongodb.db.collection('projects'),
}
const BAD_MIGRATION_NAME =
'20231219081700_move_doc_versions_from_docops_to_docs'
let loggingChain = Promise.resolve()
const projectIdsThatNeedResyncing = []
async function flushLogQueue() {
const logPath = OPTIONS['resyncs-needed-file']
loggingChain = loggingChain.then(async () => {
const batch = projectIdsThatNeedResyncing.splice(0)
if (batch.length === 0) return
try {
await fsPromises.appendFile(logPath, batch.join('\n') + '\n')
} catch (err) {
projectIdsThatNeedResyncing.push(...batch)
logger.err({ err, logPath, batch }, 'Failed to write to log file')
}
})
await loggingChain
}
async function recordProjectNeedsResync(projectId) {
if (OPTIONS['resyncs-needed-file']) {
projectIdsThatNeedResyncing.push(projectId)
await flushLogQueue()
} else {
console.log(`Project ${projectId} needs a hard resync.`)
}
}
async function main() {
const badMigration = await db.migrations.findOne({ name: BAD_MIGRATION_NAME })
if (OPTIONS.force || badMigration != null) {
console.warn('Need to recover doc versions. This will take a while.')
await runRecovery()
}
await db.migrations.deleteOne({ name: BAD_MIGRATION_NAME })
console.log('Done.')
}
async function runRecovery() {
let batch = []
const summary = {
updated: 0,
ignored: 0,
skipped: 0,
deletedUpdated: 0,
deletedIgnored: 0,
}
const processBatchAndLogProgress = async () => {
try {
await BPromise.map(batch, project => processProject(project, summary), {
concurrency: OPTIONS.concurrency,
})
} finally {
console.log(`${summary.updated} projects updated`)
console.log(`${summary.ignored} projects had good versions`)
console.log(`${summary.deletedUpdated} deleted projects updated`)
console.log(
`${summary.deletedIgnored} deleted projects had good versions`
)
console.log(`${summary.skipped} projects skipped`)
}
batch = []
}
await printDBStats()
await touchResyncsNeededFile()
for await (const project of getProjects()) {
batch.push(project)
if (batch.length >= BATCH_SIZE) {
await processBatchAndLogProgress()
}
}
for await (const deletedProject of getDeletedProjects()) {
const project = deletedProject.project
project.isDeleted = true
batch.push(project)
if (batch.length >= BATCH_SIZE) {
await processBatchAndLogProgress()
}
}
if (batch.length > 0) {
await processBatchAndLogProgress()
}
await backfillMissingVersions()
}
async function printDBStats() {
const projects = await db.projects.estimatedDocumentCount()
const docs = await db.docs.estimatedDocumentCount()
console.log(
`Need to check ${projects} projects with a total of ${docs} docs.`
)
}
async function touchResyncsNeededFile() {
if (OPTIONS['resyncs-needed-file']) {
await fsPromises.appendFile(OPTIONS['resyncs-needed-file'], '')
}
}
function getProjects() {
return db.projects.find({}, { projection: { _id: 1, overleaf: 1 } })
}
function getDeletedProjects() {
return db.deletedProjects.find(
{ project: { $ne: null } },
{ projection: { 'project._id': 1, 'project.overleaf': 1 } }
)
}
async function processProject(project, summary) {
const projectId = project._id.toString()
let updated = false
try {
const historyDocVersions = await getHistoryDocVersions(project)
for (const { docId, version } of historyDocVersions) {
const update = await fixMongoDocVersion(docId, version)
if (update != null) {
updated = true
}
}
if (project.isDeleted) {
if (updated) {
summary.deletedUpdated += 1
} else {
summary.deletedIgnored += 1
}
} else {
await recordProjectNeedsResync(projectId)
if (updated) {
summary.updated += 1
} else {
summary.ignored += 1
}
}
} catch (err) {
logger.error({ err, projectId }, 'Failed to process project')
if (OPTIONS['skip-history-failures']) {
summary.skipped += 1
} else {
throw err
}
}
}
async function getHistoryDocVersions(project) {
const historyId = project.overleaf.history.id
const chunk = await chunkStore.loadLatest(historyId)
if (chunk == null) {
return []
}
const snapshot = chunk.getSnapshot()
const changes = chunk.getChanges()
snapshot.applyAll(changes)
const v2DocVersions = snapshot.getV2DocVersions()
if (v2DocVersions == null) {
return []
}
return Object.entries(v2DocVersions.data).map(([docId, versionInfo]) => ({
docId,
version: versionInfo.v,
}))
}
async function fixMongoDocVersion(docId, historyVersion) {
const docBeforeUpdate = await db.docs.findOneAndUpdate(
{
_id: new ObjectId(docId),
$or: [
{ version: { $lte: historyVersion } },
{ version: { $exists: false } },
],
},
{ $set: { version: historyVersion + 1 } }
)
if (docBeforeUpdate != null) {
return {
previousVersion: docBeforeUpdate.version,
newVersion: historyVersion + 1,
}
} else {
return null
}
}
/**
* Set all remaining versions to 0
*/
async function backfillMissingVersions() {
console.log('Defaulting version to 0 for remaining docs.')
await db.docs.updateMany(
{ version: { $exists: false } },
{ $set: { version: 0 } }
)
}
main()
.finally(async () => {
console.log('Flushing log queue.')
await flushLogQueue()
})
.then(() => {
process.exit(0)
})
.catch(err => {
console.error(err)
process.exit(1)
})