From a2322f090a925a3da6faab2c7fd3a966437e7704 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Thu, 10 Aug 2023 17:19:23 +0200 Subject: [PATCH] Merge pull request #14198 from overleaf/jpa-history-migration [server-pro] prepare hotfix 3.5.11 GitOrigin-RevId: c4950a327e472c9cfe1af93599dd5a9a54bb956b --- server-ce/hotfix/3.5.11/Dockerfile | 8 ++ .../hotfix/3.5.11/clean_sl_history_data.js | 70 ++++++++++++++ server-ce/hotfix/3.5.11/pr_14200.patch | 95 +++++++++++++++++++ server-ce/hotfix/4.0.6/Dockerfile | 4 + .../hotfix/4.0.6/clean_sl_history_data.js | 70 ++++++++++++++ .../scripts/history/clean_sl_history_data.js | 70 ++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 server-ce/hotfix/3.5.11/Dockerfile create mode 100644 server-ce/hotfix/3.5.11/clean_sl_history_data.js create mode 100644 server-ce/hotfix/3.5.11/pr_14200.patch create mode 100644 server-ce/hotfix/4.0.6/Dockerfile create mode 100644 server-ce/hotfix/4.0.6/clean_sl_history_data.js create mode 100644 services/web/scripts/history/clean_sl_history_data.js diff --git a/server-ce/hotfix/3.5.11/Dockerfile b/server-ce/hotfix/3.5.11/Dockerfile new file mode 100644 index 0000000000..650f194412 --- /dev/null +++ b/server-ce/hotfix/3.5.11/Dockerfile @@ -0,0 +1,8 @@ +FROM sharelatex/sharelatex:3.5.10 + +# Patch: Drop the old history collections and increase mongo query timeout +ADD clean_sl_history_data.js /overleaf/services/web/scripts/history/clean_sl_history_data.js + +# Patch: convert large deleted docs to files +COPY pr_14200.patch . +RUN patch -p0 < pr_14200.patch diff --git a/server-ce/hotfix/3.5.11/clean_sl_history_data.js b/server-ce/hotfix/3.5.11/clean_sl_history_data.js new file mode 100644 index 0000000000..a7497c826f --- /dev/null +++ b/server-ce/hotfix/3.5.11/clean_sl_history_data.js @@ -0,0 +1,70 @@ +// Increase default mongo query timeout from 1min to 1h +process.env.MONGO_SOCKET_TIMEOUT = process.env.MONGO_SOCKET_TIMEOUT || '360000' +const { waitForDb, db } = require('../../app/src/infrastructure/mongodb') + +async function main() { + await checkAllProjectsAreMigrated() + await setAllowDowngradeToFalse() + await deleteHistoryCollections() + console.log('Legacy history data cleaned up successfully') + process.exit(0) +} + +async function checkAllProjectsAreMigrated() { + console.log('checking all projects are migrated to Full Project History') + + const count = await db.projects.countDocuments({ + 'overleaf.history.display': { $ne: true }, + }) + + if (count === 0) { + console.log('All projects are migrated to Full Project History') + } else { + console.error( + `There are ${count} projects that are not migrated to Full Project History` + + ` please complete the migration before running this script again.` + ) + process.exit(1) + } +} + +async function setAllowDowngradeToFalse() { + console.log('unsetting `allowDowngrade` flag in all projects') + await db.projects.updateMany( + { + 'overleaf.history.id': { $exists: true }, + 'overleaf.history.allowDowngrade': true, + }, + { $unset: { 'overleaf.history.allowDowngrade': 1 } } + ) + console.log('unsetting `allowDowngrade` flag in all projects - Done') +} + +async function deleteHistoryCollections() { + await gracefullyDropCollection(db.docHistory) + await gracefullyDropCollection(db.docHistoryIndex) + await gracefullyDropCollection(db.projectHistoryMetaData) +} + +async function gracefullyDropCollection(collection) { + const collectionName = collection.collectionName + console.log(`removing \`${collectionName}\` data`) + try { + await collection.drop() + } catch (err) { + if (err.code === 26) { + // collection already deleted + console.log(`removing \`${collectionName}\` data - Already removed`) + } else { + throw err + } + } + console.log(`removing \`${collectionName}\` data - Done`) +} + +waitForDb() + .then(main) + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/server-ce/hotfix/3.5.11/pr_14200.patch b/server-ce/hotfix/3.5.11/pr_14200.patch new file mode 100644 index 0000000000..57dd5f9d13 --- /dev/null +++ b/server-ce/hotfix/3.5.11/pr_14200.patch @@ -0,0 +1,95 @@ +--- services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js ++++ services/web/modules/history-migration/app/src/HistoryUpgradeHelper.js +@@ -1,6 +1,9 @@ ++const _ = require('lodash') ++const fs = require('fs') + const { ReadPreference, ObjectId } = require('mongodb') + const { db } = require('../../../../app/src/infrastructure/mongodb') + const Settings = require('@overleaf/settings') ++const logger = require('@overleaf/logger') + + const ProjectHistoryHandler = require('../../../../app/src/Features/Project/ProjectHistoryHandler') + const HistoryManager = require('../../../../app/src/Features/History/HistoryManager') +@@ -8,6 +11,8 @@ const ProjectHistoryController = require('./ProjectHistoryController') + const ProjectEntityHandler = require('../../../../app/src/Features/Project/ProjectEntityHandler') + const ProjectEntityUpdateHandler = require('../../../../app/src/Features/Project/ProjectEntityUpdateHandler') + const DocumentUpdaterHandler = require('../../../../app/src/Features/DocumentUpdater/DocumentUpdaterHandler') ++const { Doc } = require('../../../../app/src/models/Doc') ++const FileWriter = require('../../../../app/src/infrastructure/FileWriter') + + // Timestamp of when 'Enable history for SL in background' release + const ID_WHEN_FULL_PROJECT_HISTORY_ENABLED = +@@ -340,9 +345,33 @@ async function anyDocHistoryIndexExists(project) { + ) + } + ++async function convertDeletedDocToFile(projectId, docId, userId, source, doc) { ++ // write the doc to a temporary file and upload to filestore ++ const tmpFilePath = await FileWriter.promises.writeLinesToDisk( ++ projectId, ++ doc.lines ++ ) ++ await ProjectEntityUpdateHandler.promises.upsertFileWithPath( ++ projectId, ++ `/_deleted/${docId}/${doc.name}`, ++ tmpFilePath, ++ null, ++ userId, ++ source ++ ) ++ // hard delete the original doc, otherwise it will get picked up again ++ // by readDeletedDocs in ProjectHistoryController and the final ++ // resync of the history will fail. ++ await db.docs.deleteOne({ _id: docId }) ++ await db.docOps.deleteOne({ doc_id: docId }) ++ // clean up the temporary file ++ await fs.promises.unlink(tmpFilePath) ++} ++ + async function convertLargeDocsToFile(projectId, userId) { +- const docs = await ProjectEntityHandler.promises.getAllDocs(projectId) + let convertedDocCount = 0 ++ const docs = await ProjectEntityHandler.promises.getAllDocs(projectId) ++ // Convert large docs to files + for (const doc of Object.values(docs)) { + const sizeBound = JSON.stringify(doc.lines) + if (docIsTooLarge(sizeBound, doc.lines, Settings.max_doc_length)) { +@@ -355,6 +384,39 @@ async function convertLargeDocsToFile(projectId, userId) { + convertedDocCount++ + } + } ++ // Convert deleted docs to files, these cannot be converted by ++ // ProjectEntityUpdateHandler so we do it manually ++ const docsCursor = Doc.find({ ++ project_id: ObjectId(projectId), ++ }) ++ .lean() ++ .cursor() ++ for await (const doc of docsCursor) { ++ // check whether the doc is present in the filetree instead of ++ // relying on the deletedAt property ++ const docExistsInFiletree = _.find(docs, existingDoc => ++ existingDoc._id.equals(doc._id) ++ ) ++ if (docExistsInFiletree || doc.inS3) { ++ continue ++ } ++ const sizeBound = JSON.stringify(doc.lines) ++ if (docIsTooLarge(sizeBound, doc.lines, Settings.max_doc_length)) { ++ const docId = doc._id.toString() ++ if (!_.isEmpty(doc.ranges)) { ++ throw new Error(`found too large deleted doc with ranges: ${docId}`) ++ } ++ logger.warn({ projectId, docId }, 'converting large deleted doc') ++ await convertDeletedDocToFile( ++ projectId, ++ doc._id, ++ userId, ++ 'history-migration', ++ doc ++ ) ++ convertedDocCount++ ++ } ++ } + return convertedDocCount + } diff --git a/server-ce/hotfix/4.0.6/Dockerfile b/server-ce/hotfix/4.0.6/Dockerfile new file mode 100644 index 0000000000..dcd5c6d6dc --- /dev/null +++ b/server-ce/hotfix/4.0.6/Dockerfile @@ -0,0 +1,4 @@ +FROM sharelatex/sharelatex:4.0.5 + +# Patch: Drop the old history collections and increase mongo query timeout +ADD clean_sl_history_data.js /overleaf/services/web/scripts/history/clean_sl_history_data.js diff --git a/server-ce/hotfix/4.0.6/clean_sl_history_data.js b/server-ce/hotfix/4.0.6/clean_sl_history_data.js new file mode 100644 index 0000000000..a7497c826f --- /dev/null +++ b/server-ce/hotfix/4.0.6/clean_sl_history_data.js @@ -0,0 +1,70 @@ +// Increase default mongo query timeout from 1min to 1h +process.env.MONGO_SOCKET_TIMEOUT = process.env.MONGO_SOCKET_TIMEOUT || '360000' +const { waitForDb, db } = require('../../app/src/infrastructure/mongodb') + +async function main() { + await checkAllProjectsAreMigrated() + await setAllowDowngradeToFalse() + await deleteHistoryCollections() + console.log('Legacy history data cleaned up successfully') + process.exit(0) +} + +async function checkAllProjectsAreMigrated() { + console.log('checking all projects are migrated to Full Project History') + + const count = await db.projects.countDocuments({ + 'overleaf.history.display': { $ne: true }, + }) + + if (count === 0) { + console.log('All projects are migrated to Full Project History') + } else { + console.error( + `There are ${count} projects that are not migrated to Full Project History` + + ` please complete the migration before running this script again.` + ) + process.exit(1) + } +} + +async function setAllowDowngradeToFalse() { + console.log('unsetting `allowDowngrade` flag in all projects') + await db.projects.updateMany( + { + 'overleaf.history.id': { $exists: true }, + 'overleaf.history.allowDowngrade': true, + }, + { $unset: { 'overleaf.history.allowDowngrade': 1 } } + ) + console.log('unsetting `allowDowngrade` flag in all projects - Done') +} + +async function deleteHistoryCollections() { + await gracefullyDropCollection(db.docHistory) + await gracefullyDropCollection(db.docHistoryIndex) + await gracefullyDropCollection(db.projectHistoryMetaData) +} + +async function gracefullyDropCollection(collection) { + const collectionName = collection.collectionName + console.log(`removing \`${collectionName}\` data`) + try { + await collection.drop() + } catch (err) { + if (err.code === 26) { + // collection already deleted + console.log(`removing \`${collectionName}\` data - Already removed`) + } else { + throw err + } + } + console.log(`removing \`${collectionName}\` data - Done`) +} + +waitForDb() + .then(main) + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/services/web/scripts/history/clean_sl_history_data.js b/services/web/scripts/history/clean_sl_history_data.js new file mode 100644 index 0000000000..a7497c826f --- /dev/null +++ b/services/web/scripts/history/clean_sl_history_data.js @@ -0,0 +1,70 @@ +// Increase default mongo query timeout from 1min to 1h +process.env.MONGO_SOCKET_TIMEOUT = process.env.MONGO_SOCKET_TIMEOUT || '360000' +const { waitForDb, db } = require('../../app/src/infrastructure/mongodb') + +async function main() { + await checkAllProjectsAreMigrated() + await setAllowDowngradeToFalse() + await deleteHistoryCollections() + console.log('Legacy history data cleaned up successfully') + process.exit(0) +} + +async function checkAllProjectsAreMigrated() { + console.log('checking all projects are migrated to Full Project History') + + const count = await db.projects.countDocuments({ + 'overleaf.history.display': { $ne: true }, + }) + + if (count === 0) { + console.log('All projects are migrated to Full Project History') + } else { + console.error( + `There are ${count} projects that are not migrated to Full Project History` + + ` please complete the migration before running this script again.` + ) + process.exit(1) + } +} + +async function setAllowDowngradeToFalse() { + console.log('unsetting `allowDowngrade` flag in all projects') + await db.projects.updateMany( + { + 'overleaf.history.id': { $exists: true }, + 'overleaf.history.allowDowngrade': true, + }, + { $unset: { 'overleaf.history.allowDowngrade': 1 } } + ) + console.log('unsetting `allowDowngrade` flag in all projects - Done') +} + +async function deleteHistoryCollections() { + await gracefullyDropCollection(db.docHistory) + await gracefullyDropCollection(db.docHistoryIndex) + await gracefullyDropCollection(db.projectHistoryMetaData) +} + +async function gracefullyDropCollection(collection) { + const collectionName = collection.collectionName + console.log(`removing \`${collectionName}\` data`) + try { + await collection.drop() + } catch (err) { + if (err.code === 26) { + // collection already deleted + console.log(`removing \`${collectionName}\` data - Already removed`) + } else { + throw err + } + } + console.log(`removing \`${collectionName}\` data - Done`) +} + +waitForDb() + .then(main) + .catch(err => { + console.error(err) + process.exit(1) + })