From cb6fc31225f88d3ddb4338df4e47a7e757108b65 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 7 Dec 2021 10:49:05 +0100 Subject: [PATCH] Merge pull request #6017 from overleaf/tm-history-upgrade-v1-with-conversion Add script for upgrading v1 projects requiring conversion GitOrigin-RevId: 4f3cec528a0b9ad56db652d831052caa91e62521 --- ...pgrade_v1_with_conversion_if_sl_history.js | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js diff --git a/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js b/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js new file mode 100644 index 0000000000..ab66543c4f --- /dev/null +++ b/services/web/scripts/history/upgrade_v1_with_conversion_if_sl_history.js @@ -0,0 +1,230 @@ +const SCRIPT_VERSION = 1 +const VERBOSE_LOGGING = process.env.VERBOSE_LOGGING === 'true' +const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 100 +const DRY_RUN = process.env.DRY_RUN !== 'false' +const USE_QUERY_HINT = process.env.USE_QUERY_HINT !== 'false' +const RETRY_FAILED = process.env.RETRY_FAILED === 'true' +const MAX_UPGRADES_TO_ATTEMPT = + parseInt(process.env.MAX_UPGRADES_TO_ATTEMPT, 10) || false +const MAX_FAILURES = parseInt(process.env.MAX_FAILURES, 10) || 50 +// persist fallback in order to keep batchedUpdate in-sync +process.env.BATCH_SIZE = BATCH_SIZE +// raise mongo timeout to 1hr if otherwise unspecified +process.env.MONGO_SOCKET_TIMEOUT = + parseInt(process.env.MONGO_SOCKET_TIMEOUT, 10) || 3600000 + +const { ReadPreference } = require('mongodb') +const { db } = require('../../app/src/infrastructure/mongodb') +const { promiseMapWithLimit } = require('../../app/src/util/promises') +const { batchedUpdate } = require('../helpers/batchedUpdate') +const ProjectHistoryController = require('../../modules/admin-panel/app/src/ProjectHistoryController') + +console.log({ + DRY_RUN, + VERBOSE_LOGGING, + WRITE_CONCURRENCY, + BATCH_SIZE, + MAX_UPGRADES_TO_ATTEMPT, + MAX_FAILURES, + USE_QUERY_HINT, + RETRY_FAILED, +}) + +const RESULT = { + DRY_RUN, + attempted: 0, + projectsUpgraded: 0, + failed: 0, + continueFrom: null, +} + +let INTERRUPT = false + +async function processBatch(_, projects) { + if (projects.length && projects[0]._id) { + RESULT.continueFrom = projects[0]._id + } + await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) + console.log(RESULT) + if (INTERRUPT) { + // ctrl+c + console.log('Terminated by SIGINT') + process.exit(0) + } + if (RESULT.failed >= MAX_FAILURES) { + console.log(`MAX_FAILURES limit (${MAX_FAILURES}) reached. Stopping.`) + process.exit(0) + } + if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { + console.log( + `MAX_UPGRADES_TO_ATTEMPT limit (${MAX_UPGRADES_TO_ATTEMPT}) reached. Stopping.` + ) + process.exit(0) + } +} + +async function processProject(project) { + if (INTERRUPT) { + return + } + // skip safety check if we want to retry failed upgrades + if (!RETRY_FAILED) { + if (project.overleaf && project.overleaf.history) { + if ( + project.overleaf.history.conversionFailed || + project.overleaf.history.upgradeFailed + ) { + // we don't want to attempt upgrade on projects + // that have been previously attempted and failed + return + } + } + } + const preserveHistory = await shouldPreserveHistory(project) + if (preserveHistory) { + const anyDocHistory = await anyDocHistoryExists(project) + if (anyDocHistory) { + return await doUpgradeForV1WithConversion(project) + } + const anyDocHistoryIndex = await anyDocHistoryIndexExists(project) + if (anyDocHistoryIndex) { + return await doUpgradeForV1WithConversion(project) + } + } +} + +async function doUpgradeForV1WithConversion(project) { + if (RESULT.failed >= MAX_FAILURES) { + return + } + if (MAX_UPGRADES_TO_ATTEMPT && RESULT.attempted >= MAX_UPGRADES_TO_ATTEMPT) { + return + } + RESULT.attempted += 1 + const projectId = project._id + // migrateProjectHistory expects project id as a string + const projectIdString = project._id.toString() + if (!DRY_RUN) { + try { + // We treat these essentially as None projects, the V1 history is irrelevant, + // so we will delete it, and do a conversion as if we're a None project + await ProjectHistoryController.deleteProjectHistory(projectIdString) + if (VERBOSE_LOGGING) { + console.log( + `project ${projectId} existing full project history deleted` + ) + } + await ProjectHistoryController.migrateProjectHistory(projectIdString) + } catch (err) { + // if migrateProjectHistory fails, it cleans up by deleting + // the history and unsetting the history id + // therefore a failed project will still look like a 'None with conversion' project + RESULT.failed += 1 + console.error(`project ${projectId} FAILED with error: `, err) + // We set a failed flag so future runs of the script don't automatically retry + await db.projects.updateOne( + { _id: projectId }, + { + $set: { + 'overleaf.history.conversionFailed': true, + }, + } + ) + return + } + await db.projects.updateOne( + { _id: projectId }, + { + $set: { + 'overleaf.history.upgradeReason': `v1-with-conversion/${SCRIPT_VERSION}`, + }, + $unset: { + 'overleaf.history.upgradeFailed': true, + 'overleaf.history.conversionFailed': true, + }, + } + ) + } + if (VERBOSE_LOGGING) { + console.log( + `project ${projectId} converted and upgraded to full project history` + ) + } + RESULT.projectsUpgraded += 1 +} + +async function shouldPreserveHistory(project) { + return await db.projectHistoryMetaData.findOne( + { + $and: [ + { project_id: { $eq: project._id } }, + { preserveHistory: { $eq: true } }, + ], + }, + { readPreference: ReadPreference.SECONDARY } + ) +} + +async function anyDocHistoryExists(project) { + return await db.docHistory.findOne( + { project_id: { $eq: project._id } }, + { + projection: { _id: 1 }, + readPreference: ReadPreference.SECONDARY, + } + ) +} + +async function anyDocHistoryIndexExists(project) { + return await db.docHistoryIndex.findOne( + { project_id: { $eq: project._id } }, + { + projection: { _id: 1 }, + readPreference: ReadPreference.SECONDARY, + } + ) +} + +async function main() { + const projection = { + _id: 1, + overleaf: 1, + } + const options = {} + if (USE_QUERY_HINT) { + options.hint = { _id: 1 } + } + await batchedUpdate( + 'projects', + { + $and: [ + { 'overleaf.history.display': { $ne: true } }, + { 'overleaf.history.id': { $exists: true } }, + ], + }, + processBatch, + projection, + options + ) + console.log('Final') + console.log(RESULT) +} + +// Upgrading history is not atomic, if we quit out mid-initialisation +// then history could get into a broken state +// Instead, skip any unprocessed projects and exit() at end of the batch. +process.on('SIGINT', function () { + console.log('Caught SIGINT, waiting for in process upgrades to complete') + INTERRUPT = true +}) + +main() + .then(() => { + console.error('Done.') + process.exit(0) + }) + .catch(error => { + console.error({ error }) + process.exit(1) + })