diff --git a/services/docstore/app.js b/services/docstore/app.js index 20c7c2fced..51ad785065 100644 --- a/services/docstore/app.js +++ b/services/docstore/app.js @@ -50,6 +50,7 @@ app.param('doc_id', function (req, res, next, docId) { app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs) app.get('/project/:project_id/doc', HttpController.getAllDocs) app.get('/project/:project_id/ranges', HttpController.getAllRanges) +app.get('/project/:project_id/has-ranges', HttpController.projectHasRanges) app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc) app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted) app.get('/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc) diff --git a/services/docstore/app/js/DocManager.js b/services/docstore/app/js/DocManager.js index 4fd1acbb16..777f4ab6a4 100644 --- a/services/docstore/app/js/DocManager.js +++ b/services/docstore/app/js/DocManager.js @@ -131,6 +131,25 @@ const DocManager = { return docs }, + async projectHasRanges(projectId) { + const docs = await MongoManager.promises.getProjectsDocs( + projectId, + {}, + { _id: 1 } + ) + const docIds = docs.map(doc => doc._id) + for (const docId of docIds) { + const doc = await DocManager.peekDoc(projectId, docId) + if ( + (doc.ranges?.comments != null && doc.ranges.comments.length > 0) || + (doc.ranges?.changes != null && doc.ranges.changes.length > 0) + ) { + return true + } + } + return false + }, + async updateDoc(projectId, docId, lines, version, ranges) { const MAX_ATTEMPTS = 2 for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { diff --git a/services/docstore/app/js/HttpController.js b/services/docstore/app/js/HttpController.js index 59137aeb0b..1c4e137033 100644 --- a/services/docstore/app/js/HttpController.js +++ b/services/docstore/app/js/HttpController.js @@ -130,6 +130,16 @@ function getAllRanges(req, res, next) { ) } +function projectHasRanges(req, res, next) { + const { project_id: projectId } = req.params + DocManager.projectHasRanges(projectId, (err, projectHasRanges) => { + if (err) { + return next(err) + } + res.json({ projectHasRanges }) + }) +} + function updateDoc(req, res, next) { const { doc_id: docId, project_id: projectId } = req.params const lines = req.body?.lines @@ -298,6 +308,7 @@ module.exports = { getAllDocs, getAllDeletedDocs, getAllRanges, + projectHasRanges, updateDoc, patchDoc, archiveAllDocs, diff --git a/services/web/app/src/Features/Docstore/DocstoreManager.js b/services/web/app/src/Features/Docstore/DocstoreManager.js index 13f81a0261..b0e799e43d 100644 --- a/services/web/app/src/Features/Docstore/DocstoreManager.js +++ b/services/web/app/src/Features/Docstore/DocstoreManager.js @@ -219,6 +219,31 @@ function updateDoc(projectId, docId, lines, version, ranges, callback) { ) } +/** + * Asks docstore whether any doc in the project has ranges + * + * @param {string} proejctId + * @param {Callback} callback + */ +function projectHasRanges(projectId, callback) { + const url = `${settings.apis.docstore.url}/project/${projectId}/has-ranges` + request.get({ url, timeout: TIMEOUT, json: true }, (err, res, body) => { + if (err) { + return callback(err) + } + if (res.statusCode >= 200 && res.statusCode < 300) { + callback(null, body.projectHasRanges) + } else { + callback( + new OError( + `docstore api responded with non-success code: ${res.statusCode}`, + { projectId } + ) + ) + } + }) +} + function archiveProject(projectId, callback) { _operateOnProject(projectId, 'archive', callback) } @@ -266,6 +291,7 @@ module.exports = { getDoc, isDocDeleted, updateDoc, + projectHasRanges, archiveProject, unarchiveProject, destroyProject, @@ -277,6 +303,7 @@ module.exports = { getDoc: promisifyMultiResult(getDoc, ['lines', 'rev', 'version', 'ranges']), isDocDeleted: promisify(isDocDeleted), updateDoc: promisifyMultiResult(updateDoc, ['modified', 'rev']), + projectHasRanges: promisify(projectHasRanges), archiveProject: promisify(archiveProject), unarchiveProject: promisify(unarchiveProject), destroyProject: promisify(destroyProject), diff --git a/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js b/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js index cfb7676d15..a00046a349 100644 --- a/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js +++ b/services/web/app/src/Features/DocumentUpdater/DocumentUpdaterHandler.js @@ -247,6 +247,46 @@ function resyncProjectHistory( ) } +/** + * Block a project from being loaded in docupdater + * + * @param {string} projectId + * @param {Callback} callback + */ +function blockProject(projectId, callback) { + _makeRequest( + { path: `/project/${projectId}/block`, method: 'POST', json: true }, + projectId, + 'block-project', + (err, body) => { + if (err) { + return callback(err) + } + callback(null, body.blocked) + } + ) +} + +/** + * Unblock a previously blocked project + * + * @param {string} projectId + * @param {Callback} callback + */ +function unblockProject(projectId, callback) { + _makeRequest( + { path: `/project/${projectId}/unblock`, method: 'POST', json: true }, + projectId, + 'unblock-project', + (err, body) => { + if (err) { + return callback(err) + } + callback(null, body.wasBlocked) + } + ) +} + function updateProjectStructure( projectId, projectHistoryId, @@ -460,6 +500,8 @@ module.exports = { reopenThread, deleteThread, resyncProjectHistory, + blockProject, + unblockProject, updateProjectStructure, promises: { flushProjectToMongo: promisify(flushProjectToMongo), @@ -481,6 +523,8 @@ module.exports = { reopenThread: promisify(reopenThread), deleteThread: promisify(deleteThread), resyncProjectHistory: promisify(resyncProjectHistory), + blockProject: promisify(blockProject), + unblockProject: promisify(unblockProject), updateProjectStructure: promisify(updateProjectStructure), }, } diff --git a/services/web/app/src/Features/History/HistoryRangesSupportMigration.js b/services/web/app/src/Features/History/HistoryRangesSupportMigration.js index 863b631442..c9529b182f 100644 --- a/services/web/app/src/Features/History/HistoryRangesSupportMigration.js +++ b/services/web/app/src/Features/History/HistoryRangesSupportMigration.js @@ -2,8 +2,12 @@ const { callbackify } = require('util') const { ObjectId } = require('mongodb') +const OError = require('@overleaf/o-error') const logger = require('@overleaf/logger') const HistoryManager = require('../History/HistoryManager') +const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler') +const DocstoreManager = require('../Docstore/DocstoreManager') +const ProjectOptionsHandler = require('../Project/ProjectOptionsHandler') const { db } = require('../../infrastructure/mongodb') /** @@ -18,6 +22,7 @@ const { db } = require('../../infrastructure/mongodb') * @param {"forwards" | "backwards"} [opts.direction] * @param {boolean} [opts.force] * @param {boolean} [opts.stopOnError] + * @param {boolean} [opts.quickOnly] */ async function migrateProjects(opts = {}) { const { @@ -29,6 +34,7 @@ async function migrateProjects(opts = {}) { direction = 'forwards', force = false, stopOnError = false, + quickOnly = false, } = opts const clauses = [] @@ -76,11 +82,22 @@ async function migrateProjects(opts = {}) { } const startTimeMs = Date.now() + let quickMigrationSuccess try { - await migrateProject(projectId, direction) + quickMigrationSuccess = await quickMigration(projectId, direction) + if (!quickMigrationSuccess) { + if (quickOnly) { + logger.info( + { projectId, direction }, + 'Quick migration failed, skipping project' + ) + } else { + await migrateProject(projectId, direction) + } + } } catch (err) { logger.error( - { projectId, direction, projectsProcessed }, + { err, projectId, direction, projectsProcessed }, 'Failed to migrate history ranges support' ) projectsProcessed += 1 @@ -93,12 +110,73 @@ async function migrateProjects(opts = {}) { const elapsedMs = Date.now() - startTimeMs projectsProcessed += 1 logger.info( - { projectId, direction, projectsProcessed, elapsedMs }, + { + projectId, + direction, + projectsProcessed, + elapsedMs, + quick: quickMigrationSuccess, + }, 'Migrated history ranges support' ) } } +/** + * Attempt a quick migration (without resync) + * + * @param {string} projectId + * @param {"forwards" | "backwards"} direction + * @return {Promise} whether or not the quick migration was a success + */ +async function quickMigration(projectId, direction = 'forwards') { + const blockSuccess = + await DocumentUpdaterHandler.promises.blockProject(projectId) + if (!blockSuccess) { + return false + } + + let projectHasRanges + try { + projectHasRanges = + await DocstoreManager.promises.projectHasRanges(projectId) + } catch (err) { + await DocumentUpdaterHandler.promises.unblockProject(projectId) + throw err + } + if (projectHasRanges) { + await DocumentUpdaterHandler.promises.unblockProject(projectId) + return false + } + + try { + await ProjectOptionsHandler.promises.setHistoryRangesSupport( + projectId, + direction === 'forwards' + ) + } catch (err) { + await DocumentUpdaterHandler.promises.unblockProject(projectId) + await hardResyncProject(projectId) + throw err + } + + let wasBlocked + try { + wasBlocked = await DocumentUpdaterHandler.promises.unblockProject(projectId) + } catch (err) { + await hardResyncProject(projectId) + throw err + } + if (!wasBlocked) { + await hardResyncProject(projectId) + throw new OError('Tried to unblock project but it was not blocked', { + projectId, + }) + } + + return true +} + /** * Migrate a single project * @@ -112,6 +190,19 @@ async function migrateProject(projectId, direction = 'forwards') { }) } +/** + * Hard resync a project + * + * This is used when something goes wrong with the quick migration after we've + * changed the history ranges support flag on a project. + * + * @param {string} projectId + */ +async function hardResyncProject(projectId) { + await HistoryManager.promises.flushProject(projectId) + await HistoryManager.promises.resyncProject(projectId, { force: true }) +} + module.exports = { migrateProjects: callbackify(migrateProjects), migrateProject: callbackify(migrateProject), diff --git a/services/web/scripts/history/migrate_ranges_support.js b/services/web/scripts/history/migrate_ranges_support.js index e368e3c5ec..5d9d98acd7 100644 --- a/services/web/scripts/history/migrate_ranges_support.js +++ b/services/web/scripts/history/migrate_ranges_support.js @@ -13,6 +13,7 @@ async function main() { direction, force, stopOnError, + quickOnly, } = parseArgs() await HistoryRangesSupportMigration.promises.migrateProjects({ projectIds, @@ -23,6 +24,7 @@ async function main() { direction, force, stopOnError, + quickOnly, }) } @@ -41,12 +43,13 @@ Options: --backwards Disable history ranges support for selected project ids --force Migrate projects even if they were already migrated --stop-on-error Stop after first migration error + --quick-only Do not try a resync migration if quick migration fails `) } function parseArgs() { const args = minimist(process.argv.slice(2), { - boolean: ['backwards', 'help', 'all', 'force'], + boolean: ['backwards', 'help', 'all', 'force', 'quick-only'], string: ['owner-id', 'project-id', 'min-id', 'max-id'], }) @@ -63,6 +66,7 @@ function parseArgs() { const maxCount = args['max-count'] const force = args.force const stopOnError = args['stop-on-error'] + const quickOnly = args['quick-only'] const all = args.all if ( @@ -89,6 +93,7 @@ function parseArgs() { direction, force, stopOnError, + quickOnly, } }