diff --git a/server-ce/Dockerfile-base b/server-ce/Dockerfile-base index ad50a66637..c048a9fcab 100644 --- a/server-ce/Dockerfile-base +++ b/server-ce/Dockerfile-base @@ -10,7 +10,7 @@ ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var # Update to ensure dependencies are updated # ------------------------------------------ -ENV REBUILT_AFTER="2024-14-02" +ENV REBUILT_AFTER="2024-04-08" # Install dependencies # -------------------- diff --git a/server-ce/hotfix/4.2.4/Dockerfile b/server-ce/hotfix/4.2.4/Dockerfile new file mode 100644 index 0000000000..30d3be8bf4 --- /dev/null +++ b/server-ce/hotfix/4.2.4/Dockerfile @@ -0,0 +1,23 @@ +FROM sharelatex/sharelatex:4.2.3 + +# Upgrade Node.js to version 18.20.2 +RUN apt-get update \ +&& apt-get install -y nodejs=18.20.2-1nodesource1 \ +&& rm -rf /var/lib/apt/lists/* + +# Patch: force services to use ipv4 in server-ce container +ADD env.sh /etc/sharelatex/env.sh +COPY pr_17601-1.patch /etc/sharelatex/ +RUN cd /etc/sharelatex && patch -p0 < pr_17601-1.patch && rm pr_17601-1.patch +COPY pr_17601-2.patch /overleaf/cron/ +RUN cd /overleaf/cron && patch -p0 < pr_17601-2.patch && rm pr_17601-2.patch +COPY pr_17601-3.patch /etc/service/ +RUN cd /etc/service && patch -p0 < pr_17601-3.patch && rm pr_17601-3.patch + +# Add history utility scripts +ADD bin/* /overleaf/bin/ + +# Patch: https://github.com/overleaf/internal/pull/17885 +COPY pr_17885.patch . +RUN patch -p0 -d /etc/my_init.pre_shutdown.d < pr_17885.patch \ +&& rm pr_17885.patch diff --git a/server-ce/hotfix/4.2.4/bin/flush-history-queues b/server-ce/hotfix/4.2.4/bin/flush-history-queues new file mode 100755 index 0000000000..fcfe33bafe --- /dev/null +++ b/server-ce/hotfix/4.2.4/bin/flush-history-queues @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/container_environment.sh +cd /overleaf/services/project-history +node scripts/flush_all.js 100000 diff --git a/server-ce/hotfix/4.2.4/bin/force-history-resyncs b/server-ce/hotfix/4.2.4/bin/force-history-resyncs new file mode 100755 index 0000000000..c43b1ced8e --- /dev/null +++ b/server-ce/hotfix/4.2.4/bin/force-history-resyncs @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/container_environment.sh +cd /overleaf/services/project-history +node scripts/force_resync.js 1000 force diff --git a/server-ce/hotfix/4.2.4/env.sh b/server-ce/hotfix/4.2.4/env.sh new file mode 100644 index 0000000000..2dee36a151 --- /dev/null +++ b/server-ce/hotfix/4.2.4/env.sh @@ -0,0 +1,14 @@ +export CHAT_HOST=127.0.0.1 +export CLSI_HOST=127.0.0.1 +export CONTACTS_HOST=127.0.0.1 +export DOCSTORE_HOST=127.0.0.1 +export DOCUMENT_UPDATER_HOST=127.0.0.1 +export DOCUPDATER_HOST=127.0.0.1 +export FILESTORE_HOST=127.0.0.1 +export HISTORY_V1_HOST=127.0.0.1 +export NOTIFICATIONS_HOST=127.0.0.1 +export PROJECT_HISTORY_HOST=127.0.0.1 +export REALTIME_HOST=127.0.0.1 +export SPELLING_HOST=127.0.0.1 +export WEB_HOST=127.0.0.1 +export WEB_API_HOST=127.0.0.1 diff --git a/server-ce/hotfix/4.2.4/pr_17601-1.patch b/server-ce/hotfix/4.2.4/pr_17601-1.patch new file mode 100644 index 0000000000..3f8c136df8 --- /dev/null +++ b/server-ce/hotfix/4.2.4/pr_17601-1.patch @@ -0,0 +1,31 @@ +--- settings.js ++++ settings.js +@@ -256,16 +256,16 @@ const settings = { + + apis: { + web: { +- url: 'http://localhost:3000', ++ url: 'http://127.0.0.1:3000', + user: httpAuthUser, + pass: httpAuthPass, + }, + project_history: { + sendProjectStructureOps: true, +- url: 'http://localhost:3054', ++ url: 'http://127.0.0.1:3054', + }, + v1_history: { +- url: process.env.V1_HISTORY_URL || 'http://localhost:3100/api', ++ url: process.env.V1_HISTORY_URL || 'http://127.0.0.1:3100/api', + user: 'staging', + pass: process.env.STAGING_PASSWORD, + requestTimeout: parseInt( +@@ -409,7 +409,7 @@ if ( + + if (parse(process.env.OVERLEAF_IS_SERVER_PRO) === true) { + settings.bypassPercentageRollouts = true +- settings.apis.references = { url: 'http://localhost:3040' } ++ settings.apis.references = { url: 'http://127.0.0.1:3040' } + } + + // Compiler diff --git a/server-ce/hotfix/4.2.4/pr_17601-2.patch b/server-ce/hotfix/4.2.4/pr_17601-2.patch new file mode 100644 index 0000000000..2322ad9afc --- /dev/null +++ b/server-ce/hotfix/4.2.4/pr_17601-2.patch @@ -0,0 +1,63 @@ +--- deactivate-projects.sh ++++ deactivate-projects.sh +@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then + exit 0 + fi + +-WEB_URL='http://localhost:3000' ++WEB_URL='http://127.0.0.1:3000' + + USER=$(cat /etc/container_environment/WEB_API_USER) + PASS=$(cat /etc/container_environment/WEB_API_PASSWORD) +--- delete-projects.sh ++++ delete-projects.sh +@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then + exit 0 + fi + +-WEB_URL='http://localhost:3000' ++WEB_URL='http://127.0.0.1:3000' + + USER=$(cat /etc/container_environment/WEB_API_USER) + PASS=$(cat /etc/container_environment/WEB_API_PASSWORD) +--- delete-users.sh ++++ delete-users.sh +@@ -14,7 +14,7 @@ if [[ "${ENABLE_CRON_RESOURCE_DELETION:-null}" != "true" ]]; then + exit 0 + fi + +-WEB_URL='http://localhost:3000' ++WEB_URL='http://127.0.0.1:3000' + + USER=$(cat /etc/container_environment/WEB_API_USER) + PASS=$(cat /etc/container_environment/WEB_API_PASSWORD) +--- project-history-periodic-flush.sh ++++ project-history-periodic-flush.sh +@@ -7,6 +7,6 @@ echo "Flush project-history queue" + echo "--------------------------" + date + +-PROJECT_HISTORY_URL='http://localhost:3054' ++PROJECT_HISTORY_URL='http://127.0.0.1:3054' + + curl -X POST "${PROJECT_HISTORY_URL}/flush/old?timeout=3600000&limit=5000&background=1" +--- project-history-retry-hard.sh ++++ project-history-retry-hard.sh +@@ -7,6 +7,6 @@ echo "Retry project-history errors (hard)" + echo "-----------------------------------" + date + +-PROJECT_HISTORY_URL='http://localhost:3054' ++PROJECT_HISTORY_URL='http://127.0.0.1:3054' + + curl -X POST "${PROJECT_HISTORY_URL}/retry/failures?failureType=hard&timeout=3600000&limit=10000" +--- project-history-retry-soft.sh ++++ project-history-retry-soft.sh +@@ -6,6 +6,6 @@ echo "-----------------------------------" + echo "Retry project-history errors (soft)" + echo "-----------------------------------" + +-PROJECT_HISTORY_URL='http://localhost:3054' ++PROJECT_HISTORY_URL='http://127.0.0.1:3054' + + curl -X POST "${PROJECT_HISTORY_URL}/retry/failures?failureType=soft&timeout=3600000&limit=10000" diff --git a/server-ce/hotfix/4.2.4/pr_17601-3.patch b/server-ce/hotfix/4.2.4/pr_17601-3.patch new file mode 100644 index 0000000000..de64af589a --- /dev/null +++ b/server-ce/hotfix/4.2.4/pr_17601-3.patch @@ -0,0 +1,118 @@ +--- chat-sharelatex/run ++++ chat-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30100" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/chat/app.js >> /var/log/sharelatex/chat.log 2>&1 +--- clsi-sharelatex/run ++++ clsi-sharelatex/run +@@ -15,4 +15,7 @@ if [ -e '/var/run/docker.sock' ]; then + usermod -aG dockeronhost www-data + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/clsi/app.js >> /var/log/sharelatex/clsi.log 2>&1 +--- contacts-sharelatex/run ++++ contacts-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30360" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/contacts/app.js >> /var/log/sharelatex/contacts.log 2>&1 +--- docstore-sharelatex/run ++++ docstore-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30160" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/docstore/app.js >> /var/log/sharelatex/docstore.log 2>&1 +--- document-updater-sharelatex/run ++++ document-updater-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30030" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/document-updater/app.js >> /var/log/sharelatex/document-updater.log 2>&1 +--- filestore-sharelatex/run ++++ filestore-sharelatex/run +@@ -1,2 +1,6 @@ + #!/bin/bash ++ ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node /overleaf/services/filestore/app.js >> /var/log/sharelatex/filestore.log 2>&1 +--- notifications-sharelatex/run ++++ notifications-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30420" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/notifications/app.js >> /var/log/sharelatex/notifications.log 2>&1 +--- project-history-sharelatex/run ++++ project-history-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30540" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/project-history/app.js >> /var/log/sharelatex/project-history.log 2>&1 +--- real-time-sharelatex/run ++++ real-time-sharelatex/run +@@ -1,2 +1,6 @@ + #!/bin/bash ++ ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node /overleaf/services/real-time/app.js >> /var/log/sharelatex/real-time.log 2>&1 +--- spelling-sharelatex/run ++++ spelling-sharelatex/run +@@ -6,4 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30050" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 ++ + exec /sbin/setuser www-data /usr/bin/node $NODE_PARAMS /overleaf/services/spelling/app.js >> /var/log/sharelatex/spelling.log 2>&1 +--- web-api-sharelatex/run ++++ web-api-sharelatex/run +@@ -6,6 +6,7 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:30000" + fi + ++source /etc/sharelatex/env.sh + export LISTEN_ADDRESS=0.0.0.0 + export ENABLED_SERVICES="api" + export METRICS_APP_NAME="web-api" +--- web-sharelatex/run ++++ web-sharelatex/run +@@ -6,6 +6,8 @@ if [ "$DEBUG_NODE" == "true" ]; then + NODE_PARAMS="--inspect=0.0.0.0:40000" + fi + ++source /etc/sharelatex/env.sh ++export LISTEN_ADDRESS=127.0.0.1 + export ENABLED_SERVICES="web" + export WEB_PORT="4000" diff --git a/server-ce/hotfix/4.2.4/pr_17885.patch b/server-ce/hotfix/4.2.4/pr_17885.patch new file mode 100644 index 0000000000..d9816ebac4 --- /dev/null +++ b/server-ce/hotfix/4.2.4/pr_17885.patch @@ -0,0 +1,33 @@ +--- 00_close_site ++++ 00_close_site +@@ -1,5 +1,8 @@ + #!/bin/sh + ++. /etc/container_environment.sh ++. /etc/sharelatex/env.sh ++ + SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown" + + mv "${SITE_MAINTENANCE_FILE}" "${SITE_MAINTENANCE_FILE_BAK}" +--- 01_flush_document_updater ++++ 01_flush_document_updater +@@ -1,5 +1,8 @@ + #!/bin/sh + ++. /etc/container_environment.sh ++. /etc/sharelatex/env.sh ++ + cd /overleaf/services/document-updater && node scripts/flush_all.js >> /var/log/sharelatex/document-updater.log 2>&1 + + EXIT_CODE="$?" +--- 02_flush_project_history ++++ 02_flush_project_history +@@ -1,5 +1,8 @@ + #!/bin/sh + ++. /etc/container_environment.sh ++. /etc/sharelatex/env.sh ++ + cd /overleaf/services/project-history && node scripts/flush_all.js >> /var/log/sharelatex/project-history.log 2>&1 + + EXIT_CODE="$?" diff --git a/server-ce/hotfix/5.0.2/910_initiate_doc_version_recovery b/server-ce/hotfix/5.0.2/910_initiate_doc_version_recovery new file mode 100755 index 0000000000..cda3d671c9 --- /dev/null +++ b/server-ce/hotfix/5.0.2/910_initiate_doc_version_recovery @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/container_environment.sh +source /etc/overleaf/env.sh + +LOG_FILE=/var/lib/overleaf/data/history/doc-version-recovery.log +export DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE=/var/lib/overleaf/data/history/doc-version-recovery-resyncs.log + +echo "Checking for doc version recovery. This can take a while if needed. Logs are in $LOG_FILE" +cd /overleaf/services/history-v1 +LOG_LEVEL=info node storage/scripts/recover_doc_versions.js 2>&1 | tee -a "$LOG_FILE" + +function resyncAllProjectsInBackground() { + waitForService docstore 3016 + waitForService document-updater 3003 + waitForService filestore 3009 + waitForService history-v1 3100 + waitForService project-history 3054 + waitForService web-api 4000 + + # Resync files that had their versions updated + while read -r project_id; do + echo "Resyncing project $project_id..." + curl -X POST --silent "http://127.0.0.1:3054/project/$project_id/resync?force=true" + done < "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE" + + # Resync files that have broken histories + /overleaf/bin/force-history-resyncs + + echo "Finished resyncing history for all projects. Adding .done suffix to log file" + mv "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE" "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE.done" +} + +function waitForService() { + local name=$1 + local port=$2 + while ! curl --fail --silent "http://127.0.0.1:$port/status"; do + echo "Waiting for $name service to start up" + sleep 10 + done +} + +if [ -f "$DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE" ]; then + echo "Finished recovery of doc versions. Resyncing history for all projects in the background." + resyncAllProjectsInBackground & +else + echo "No recovery of doc versions needed." +fi diff --git a/server-ce/hotfix/5.0.2/Dockerfile b/server-ce/hotfix/5.0.2/Dockerfile new file mode 100644 index 0000000000..e5bad9ff50 --- /dev/null +++ b/server-ce/hotfix/5.0.2/Dockerfile @@ -0,0 +1,35 @@ +FROM sharelatex/sharelatex:5.0.1 + +# Upgrade Node.js to version 18.20.2 +RUN apt-get update \ +&& apt-get install -y nodejs=18.20.2-1nodesource1 \ +&& rm -rf /var/lib/apt/lists/* + +# Patch: https://github.com/overleaf/internal/pull/17843 +COPY pr_17843.patch . +RUN patch -p0 < pr_17843.patch \ +&& rm pr_17843.patch + +# Add history utility scripts +ADD bin/* /overleaf/bin/ + +# Patch: https://github.com/overleaf/internal/pull/17885 +COPY pr_17885.patch . +RUN patch -p0 -d /etc/my_init.pre_shutdown.d < pr_17885.patch \ +&& rm pr_17885.patch + +# Recompile frontend for Grammarly patch in 5.0.1 +RUN node genScript compile | bash + +# Patch: https://github.com/overleaf/internal/pull/17960 +COPY pr_17960.patch . +RUN patch -p0 < pr_17960.patch \ +&& rm pr_17960.patch + +# Fix bad ordering of migrations +RUN mv /overleaf/services/web/migrations/20231219081700_move_doc_versions_from_docops_to_docs.js \ + /overleaf/services/web/migrations/20231105000000_move_doc_versions_from_docops_to_docs.js + +# Add doc versions recovery scripts +ADD 910_initiate_doc_version_recovery /etc/my_init.d/910_initiate_doc_version_recovery +ADD recover_doc_versions.js /overleaf/services/history-v1/storage/scripts/recover_doc_versions.js diff --git a/server-ce/hotfix/5.0.2/bin/flush-history-queues b/server-ce/hotfix/5.0.2/bin/flush-history-queues new file mode 100755 index 0000000000..b54bc5558c --- /dev/null +++ b/server-ce/hotfix/5.0.2/bin/flush-history-queues @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/container_environment.sh +source /etc/overleaf/env.sh +cd /overleaf/services/project-history +node scripts/flush_all.js 100000 diff --git a/server-ce/hotfix/5.0.2/bin/force-history-resyncs b/server-ce/hotfix/5.0.2/bin/force-history-resyncs new file mode 100755 index 0000000000..389c98a4ad --- /dev/null +++ b/server-ce/hotfix/5.0.2/bin/force-history-resyncs @@ -0,0 +1,8 @@ +#!/bin/bash + +set -euo pipefail + +source /etc/container_environment.sh +source /etc/overleaf/env.sh +cd /overleaf/services/project-history +node scripts/force_resync.js 1000 force diff --git a/server-ce/hotfix/5.0.2/pr_17843.patch b/server-ce/hotfix/5.0.2/pr_17843.patch new file mode 100644 index 0000000000..6865b9b490 --- /dev/null +++ b/server-ce/hotfix/5.0.2/pr_17843.patch @@ -0,0 +1,48 @@ +--- services/project-history/app/js/ErrorRecorder.js ++++ services/project-history/app/js/ErrorRecorder.js +@@ -210,6 +210,14 @@ export function getFailures(callback) { + 'Error: bad response from filestore: 404': 'filestore-404', + 'Error: bad response from filestore: 500': 'filestore-500', + 'NotFoundError: got a 404 from web api': 'web-api-404', ++ 'OError: history store a non-success status code: 413': ++ 'history-store-413', ++ 'OError: history store a non-success status code: 422': ++ 'history-store-422', ++ 'OError: history store a non-success status code: 500': ++ 'history-store-500', ++ 'OError: history store a non-success status code: 503': ++ 'history-store-503', + 'Error: history store a non-success status code: 413': + 'history-store-413', + 'Error: history store a non-success status code: 422': +--- services/project-history/app/js/RetryManager.js ++++ services/project-history/app/js/RetryManager.js +@@ -20,6 +20,7 @@ const TEMPORARY_FAILURES = [ + + const HARD_FAILURES = [ + 'Error: history store a non-success status code: 422', ++ 'OError: history store a non-success status code: 422', + 'OpsOutOfOrderError: project structure version out of order', + 'OpsOutOfOrderError: project structure version out of order on incoming updates', + 'OpsOutOfOrderError: doc version out of order', +--- services/project-history/scripts/clear_deleted_history.js ++++ services/project-history/scripts/clear_deleted_history.js +@@ -143,7 +143,7 @@ function checkAndClear(project, callback) { + // find all the broken projects from the failure records + async function main() { + const results = await db.projectHistoryFailures +- .find({ error: 'Error: history store a non-success status code: 422' }) ++ .find({ error: /history store a non-success status code: 422/ }) + .toArray() + + console.log('number of queues without history store 442 =', results.length) +--- services/project-history/scripts/force_resync.js ++++ services/project-history/scripts/force_resync.js +@@ -198,6 +198,7 @@ function checkAndClear(project, callback) { + // find all the broken projects from the failure records + const errorsToResync = [ + 'Error: history store a non-success status code: 422', ++ 'OError: history store a non-success status code: 422', + 'OpsOutOfOrderError: project structure version out of order', + ] + diff --git a/server-ce/hotfix/5.0.2/pr_17885.patch b/server-ce/hotfix/5.0.2/pr_17885.patch new file mode 100644 index 0000000000..0e7d3266bd --- /dev/null +++ b/server-ce/hotfix/5.0.2/pr_17885.patch @@ -0,0 +1,33 @@ +--- 00_close_site ++++ 00_close_site +@@ -1,5 +1,8 @@ + #!/bin/sh + ++. /etc/container_environment.sh ++. /etc/overleaf/env.sh ++ + SITE_MAINTENANCE_FILE_BAK="$SITE_MAINTENANCE_FILE.bak.shutdown" + + mv "${SITE_MAINTENANCE_FILE}" "${SITE_MAINTENANCE_FILE_BAK}" +--- 01_flush_document_updater ++++ 01_flush_document_updater +@@ -1,5 +1,8 @@ + #!/bin/sh + ++. /etc/container_environment.sh ++. /etc/overleaf/env.sh ++ + cd /overleaf/services/document-updater && node scripts/flush_all.js >> /var/log/overleaf/document-updater.log 2>&1 + + EXIT_CODE="$?" +--- 02_flush_project_history ++++ 02_flush_project_history +@@ -1,5 +1,8 @@ + #!/bin/sh + ++. /etc/container_environment.sh ++. /etc/overleaf/env.sh ++ + cd /overleaf/services/project-history && node scripts/flush_all.js >> /var/log/overleaf/project-history.log 2>&1 + + EXIT_CODE="$?" diff --git a/server-ce/hotfix/5.0.2/pr_17960.patch b/server-ce/hotfix/5.0.2/pr_17960.patch new file mode 100644 index 0000000000..9a04321291 --- /dev/null +++ b/server-ce/hotfix/5.0.2/pr_17960.patch @@ -0,0 +1,32 @@ +diff --git a/services/project-history/scripts/force_resync.js b/services/project-history/scripts/force_resync.js +index 5e77b35826..13e7d3cd5c 100755 +--- services/project-history/scripts/force_resync.js ++++ services/project-history/scripts/force_resync.js +@@ -77,7 +77,7 @@ function checkAndClear(project, callback) { + function startResync(cb) { + if (force) { + console.log('2. starting resync for', projectId) +- SyncManager.startResync(projectId, err => { ++ SyncManager.startHardResync(projectId, err => { + if (err) { + console.log('ERR', JSON.stringify(err.message)) + return cb(err) +@@ -195,17 +195,8 @@ function checkAndClear(project, callback) { + ) + } + +-// find all the broken projects from the failure records +-const errorsToResync = [ +- 'Error: history store a non-success status code: 422', +- 'OError: history store a non-success status code: 422', +- 'OpsOutOfOrderError: project structure version out of order', +-] +- + async function main() { +- const results = await db.projectHistoryFailures +- .find({ error: { $in: errorsToResync } }) +- .toArray() ++ const results = await db.projectHistoryFailures.find().toArray() + + console.log('number of queues without history store 442 =', results.length) + // now check if the project is truly deleted in mongo diff --git a/server-ce/hotfix/5.0.2/recover_doc_versions.js b/server-ce/hotfix/5.0.2/recover_doc_versions.js new file mode 100644 index 0000000000..32e1ddee4f --- /dev/null +++ b/server-ce/hotfix/5.0.2/recover_doc_versions.js @@ -0,0 +1,243 @@ +const fsPromises = require('fs/promises') +const { ObjectId } = require('mongodb') +const BPromise = require('bluebird') +const logger = require('@overleaf/logger') +const mongodb = require('../lib/mongodb') +const { chunkStore } = require('..') +const Events = require('events') + +// Silence warning. +Events.setMaxListeners(20) + +const BATCH_SIZE = 1000 +const OPTIONS = { + concurrency: parseInt(process.env.DOC_VERSION_RECOVERY_CONCURRENCY, 10) || 20, + force: process.env.DOC_VERSION_RECOVERY_FORCE === 'true', + 'skip-history-failures': + process.env.DOC_VERSION_RECOVERY_SKIP_HISTORY_FAILURES === 'true', + 'resyncs-needed-file': process.env.DOC_VERSION_RECOVERY_RESYNCS_NEEDED_FILE, +} + +const db = { + deletedProjects: mongodb.db.collection('deletedProjects'), + docs: mongodb.db.collection('docs'), + migrations: mongodb.db.collection('migrations'), + projects: mongodb.db.collection('projects'), +} + +const BAD_MIGRATION_NAME = + '20231219081700_move_doc_versions_from_docops_to_docs' + +let loggingChain = Promise.resolve() +const projectIdsThatNeedResyncing = [] + +async function flushLogQueue() { + const logPath = OPTIONS['resyncs-needed-file'] + loggingChain = loggingChain.then(async () => { + const batch = projectIdsThatNeedResyncing.splice(0) + if (batch.length === 0) return + try { + await fsPromises.appendFile(logPath, batch.join('\n') + '\n') + } catch (err) { + projectIdsThatNeedResyncing.push(...batch) + logger.err({ err, logPath, batch }, 'Failed to write to log file') + } + }) + await loggingChain +} +async function recordProjectNeedsResync(projectId) { + if (OPTIONS['resyncs-needed-file']) { + projectIdsThatNeedResyncing.push(projectId) + await flushLogQueue() + } else { + console.log(`Project ${projectId} needs a hard resync.`) + } +} + +async function main() { + const badMigration = await db.migrations.findOne({ name: BAD_MIGRATION_NAME }) + if (OPTIONS.force || badMigration != null) { + console.warn('Need to recover doc versions. This will take a while.') + await runRecovery() + } + await db.migrations.deleteOne({ name: BAD_MIGRATION_NAME }) + console.log('Done.') +} + +async function runRecovery() { + let batch = [] + const summary = { + updated: 0, + ignored: 0, + skipped: 0, + deletedUpdated: 0, + deletedIgnored: 0, + } + const processBatchAndLogProgress = async () => { + try { + await BPromise.map(batch, project => processProject(project, summary), { + concurrency: OPTIONS.concurrency, + }) + } finally { + console.log(`${summary.updated} projects updated`) + console.log(`${summary.ignored} projects had good versions`) + console.log(`${summary.deletedUpdated} deleted projects updated`) + console.log( + `${summary.deletedIgnored} deleted projects had good versions` + ) + console.log(`${summary.skipped} projects skipped`) + } + batch = [] + } + + await printDBStats() + await touchResyncsNeededFile() + for await (const project of getProjects()) { + batch.push(project) + if (batch.length >= BATCH_SIZE) { + await processBatchAndLogProgress() + } + } + + for await (const deletedProject of getDeletedProjects()) { + const project = deletedProject.project + project.isDeleted = true + batch.push(project) + if (batch.length >= BATCH_SIZE) { + await processBatchAndLogProgress() + } + } + + if (batch.length > 0) { + await processBatchAndLogProgress() + } + + await backfillMissingVersions() +} + +async function printDBStats() { + const projects = await db.projects.estimatedDocumentCount() + const docs = await db.docs.estimatedDocumentCount() + console.log( + `Need to check ${projects} projects with a total of ${docs} docs.` + ) +} + +async function touchResyncsNeededFile() { + if (OPTIONS['resyncs-needed-file']) { + await fsPromises.appendFile(OPTIONS['resyncs-needed-file'], '') + } +} + +function getProjects() { + return db.projects.find({}, { projection: { _id: 1, overleaf: 1 } }) +} + +function getDeletedProjects() { + return db.deletedProjects.find( + { project: { $ne: null } }, + { projection: { 'project._id': 1, 'project.overleaf': 1 } } + ) +} + +async function processProject(project, summary) { + const projectId = project._id.toString() + let updated = false + try { + const historyDocVersions = await getHistoryDocVersions(project) + + for (const { docId, version } of historyDocVersions) { + const update = await fixMongoDocVersion(docId, version) + if (update != null) { + updated = true + } + } + + if (project.isDeleted) { + if (updated) { + summary.deletedUpdated += 1 + } else { + summary.deletedIgnored += 1 + } + } else { + await recordProjectNeedsResync(projectId) + if (updated) { + summary.updated += 1 + } else { + summary.ignored += 1 + } + } + } catch (err) { + logger.error({ err, projectId }, 'Failed to process project') + if (OPTIONS['skip-history-failures']) { + summary.skipped += 1 + } else { + throw err + } + } +} + +async function getHistoryDocVersions(project) { + const historyId = project.overleaf.history.id + const chunk = await chunkStore.loadLatest(historyId) + if (chunk == null) { + return [] + } + + const snapshot = chunk.getSnapshot() + const changes = chunk.getChanges() + snapshot.applyAll(changes) + const v2DocVersions = snapshot.getV2DocVersions() + if (v2DocVersions == null) { + return [] + } + return Object.entries(v2DocVersions.data).map(([docId, versionInfo]) => ({ + docId, + version: versionInfo.v, + })) +} + +async function fixMongoDocVersion(docId, historyVersion) { + const docBeforeUpdate = await db.docs.findOneAndUpdate( + { + _id: new ObjectId(docId), + $or: [ + { version: { $lte: historyVersion } }, + { version: { $exists: false } }, + ], + }, + { $set: { version: historyVersion + 1 } } + ) + if (docBeforeUpdate != null) { + return { + previousVersion: docBeforeUpdate.version, + newVersion: historyVersion + 1, + } + } else { + return null + } +} + +/** + * Set all remaining versions to 0 + */ +async function backfillMissingVersions() { + console.log('Defaulting version to 0 for remaining docs.') + await db.docs.updateMany( + { version: { $exists: false } }, + { $set: { version: 0 } } + ) +} + +main() + .finally(async () => { + console.log('Flushing log queue.') + await flushLogQueue() + }) + .then(() => { + process.exit(0) + }) + .catch(err => { + console.error(err) + process.exit(1) + })