Merge pull request #19151 from overleaf/em-history-ranges-quick-migration

Add quick history ranges support migration

GitOrigin-RevId: 8446beb6bcd7384c32fc1b216e4b72d8f5d91500
This commit is contained in:
Eric Mc Sween 2024-07-02 11:07:12 -04:00 committed by Copybot
parent e8e31dbdb5
commit a95c0bbfc3
7 changed files with 202 additions and 4 deletions

View file

@ -50,6 +50,7 @@ app.param('doc_id', function (req, res, next, docId) {
app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs)
app.get('/project/:project_id/doc', HttpController.getAllDocs)
app.get('/project/:project_id/ranges', HttpController.getAllRanges)
app.get('/project/:project_id/has-ranges', HttpController.projectHasRanges)
app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted)
app.get('/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc)

View file

@ -131,6 +131,25 @@ const DocManager = {
return docs
},
async projectHasRanges(projectId) {
const docs = await MongoManager.promises.getProjectsDocs(
projectId,
{},
{ _id: 1 }
)
const docIds = docs.map(doc => doc._id)
for (const docId of docIds) {
const doc = await DocManager.peekDoc(projectId, docId)
if (
(doc.ranges?.comments != null && doc.ranges.comments.length > 0) ||
(doc.ranges?.changes != null && doc.ranges.changes.length > 0)
) {
return true
}
}
return false
},
async updateDoc(projectId, docId, lines, version, ranges) {
const MAX_ATTEMPTS = 2
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {

View file

@ -130,6 +130,16 @@ function getAllRanges(req, res, next) {
)
}
function projectHasRanges(req, res, next) {
const { project_id: projectId } = req.params
DocManager.projectHasRanges(projectId, (err, projectHasRanges) => {
if (err) {
return next(err)
}
res.json({ projectHasRanges })
})
}
function updateDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
const lines = req.body?.lines
@ -298,6 +308,7 @@ module.exports = {
getAllDocs,
getAllDeletedDocs,
getAllRanges,
projectHasRanges,
updateDoc,
patchDoc,
archiveAllDocs,

View file

@ -219,6 +219,31 @@ function updateDoc(projectId, docId, lines, version, ranges, callback) {
)
}
/**
* Asks docstore whether any doc in the project has ranges
*
* @param {string} proejctId
* @param {Callback} callback
*/
function projectHasRanges(projectId, callback) {
const url = `${settings.apis.docstore.url}/project/${projectId}/has-ranges`
request.get({ url, timeout: TIMEOUT, json: true }, (err, res, body) => {
if (err) {
return callback(err)
}
if (res.statusCode >= 200 && res.statusCode < 300) {
callback(null, body.projectHasRanges)
} else {
callback(
new OError(
`docstore api responded with non-success code: ${res.statusCode}`,
{ projectId }
)
)
}
})
}
function archiveProject(projectId, callback) {
_operateOnProject(projectId, 'archive', callback)
}
@ -266,6 +291,7 @@ module.exports = {
getDoc,
isDocDeleted,
updateDoc,
projectHasRanges,
archiveProject,
unarchiveProject,
destroyProject,
@ -277,6 +303,7 @@ module.exports = {
getDoc: promisifyMultiResult(getDoc, ['lines', 'rev', 'version', 'ranges']),
isDocDeleted: promisify(isDocDeleted),
updateDoc: promisifyMultiResult(updateDoc, ['modified', 'rev']),
projectHasRanges: promisify(projectHasRanges),
archiveProject: promisify(archiveProject),
unarchiveProject: promisify(unarchiveProject),
destroyProject: promisify(destroyProject),

View file

@ -247,6 +247,46 @@ function resyncProjectHistory(
)
}
/**
* Block a project from being loaded in docupdater
*
* @param {string} projectId
* @param {Callback} callback
*/
function blockProject(projectId, callback) {
_makeRequest(
{ path: `/project/${projectId}/block`, method: 'POST', json: true },
projectId,
'block-project',
(err, body) => {
if (err) {
return callback(err)
}
callback(null, body.blocked)
}
)
}
/**
* Unblock a previously blocked project
*
* @param {string} projectId
* @param {Callback} callback
*/
function unblockProject(projectId, callback) {
_makeRequest(
{ path: `/project/${projectId}/unblock`, method: 'POST', json: true },
projectId,
'unblock-project',
(err, body) => {
if (err) {
return callback(err)
}
callback(null, body.wasBlocked)
}
)
}
function updateProjectStructure(
projectId,
projectHistoryId,
@ -460,6 +500,8 @@ module.exports = {
reopenThread,
deleteThread,
resyncProjectHistory,
blockProject,
unblockProject,
updateProjectStructure,
promises: {
flushProjectToMongo: promisify(flushProjectToMongo),
@ -481,6 +523,8 @@ module.exports = {
reopenThread: promisify(reopenThread),
deleteThread: promisify(deleteThread),
resyncProjectHistory: promisify(resyncProjectHistory),
blockProject: promisify(blockProject),
unblockProject: promisify(unblockProject),
updateProjectStructure: promisify(updateProjectStructure),
},
}

View file

@ -2,8 +2,12 @@
const { callbackify } = require('util')
const { ObjectId } = require('mongodb')
const OError = require('@overleaf/o-error')
const logger = require('@overleaf/logger')
const HistoryManager = require('../History/HistoryManager')
const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
const DocstoreManager = require('../Docstore/DocstoreManager')
const ProjectOptionsHandler = require('../Project/ProjectOptionsHandler')
const { db } = require('../../infrastructure/mongodb')
/**
@ -18,6 +22,7 @@ const { db } = require('../../infrastructure/mongodb')
* @param {"forwards" | "backwards"} [opts.direction]
* @param {boolean} [opts.force]
* @param {boolean} [opts.stopOnError]
* @param {boolean} [opts.quickOnly]
*/
async function migrateProjects(opts = {}) {
const {
@ -29,6 +34,7 @@ async function migrateProjects(opts = {}) {
direction = 'forwards',
force = false,
stopOnError = false,
quickOnly = false,
} = opts
const clauses = []
@ -76,11 +82,22 @@ async function migrateProjects(opts = {}) {
}
const startTimeMs = Date.now()
let quickMigrationSuccess
try {
await migrateProject(projectId, direction)
quickMigrationSuccess = await quickMigration(projectId, direction)
if (!quickMigrationSuccess) {
if (quickOnly) {
logger.info(
{ projectId, direction },
'Quick migration failed, skipping project'
)
} else {
await migrateProject(projectId, direction)
}
}
} catch (err) {
logger.error(
{ projectId, direction, projectsProcessed },
{ err, projectId, direction, projectsProcessed },
'Failed to migrate history ranges support'
)
projectsProcessed += 1
@ -93,12 +110,73 @@ async function migrateProjects(opts = {}) {
const elapsedMs = Date.now() - startTimeMs
projectsProcessed += 1
logger.info(
{ projectId, direction, projectsProcessed, elapsedMs },
{
projectId,
direction,
projectsProcessed,
elapsedMs,
quick: quickMigrationSuccess,
},
'Migrated history ranges support'
)
}
}
/**
* Attempt a quick migration (without resync)
*
* @param {string} projectId
* @param {"forwards" | "backwards"} direction
* @return {Promise<boolean>} whether or not the quick migration was a success
*/
async function quickMigration(projectId, direction = 'forwards') {
const blockSuccess =
await DocumentUpdaterHandler.promises.blockProject(projectId)
if (!blockSuccess) {
return false
}
let projectHasRanges
try {
projectHasRanges =
await DocstoreManager.promises.projectHasRanges(projectId)
} catch (err) {
await DocumentUpdaterHandler.promises.unblockProject(projectId)
throw err
}
if (projectHasRanges) {
await DocumentUpdaterHandler.promises.unblockProject(projectId)
return false
}
try {
await ProjectOptionsHandler.promises.setHistoryRangesSupport(
projectId,
direction === 'forwards'
)
} catch (err) {
await DocumentUpdaterHandler.promises.unblockProject(projectId)
await hardResyncProject(projectId)
throw err
}
let wasBlocked
try {
wasBlocked = await DocumentUpdaterHandler.promises.unblockProject(projectId)
} catch (err) {
await hardResyncProject(projectId)
throw err
}
if (!wasBlocked) {
await hardResyncProject(projectId)
throw new OError('Tried to unblock project but it was not blocked', {
projectId,
})
}
return true
}
/**
* Migrate a single project
*
@ -112,6 +190,19 @@ async function migrateProject(projectId, direction = 'forwards') {
})
}
/**
* Hard resync a project
*
* This is used when something goes wrong with the quick migration after we've
* changed the history ranges support flag on a project.
*
* @param {string} projectId
*/
async function hardResyncProject(projectId) {
await HistoryManager.promises.flushProject(projectId)
await HistoryManager.promises.resyncProject(projectId, { force: true })
}
module.exports = {
migrateProjects: callbackify(migrateProjects),
migrateProject: callbackify(migrateProject),

View file

@ -13,6 +13,7 @@ async function main() {
direction,
force,
stopOnError,
quickOnly,
} = parseArgs()
await HistoryRangesSupportMigration.promises.migrateProjects({
projectIds,
@ -23,6 +24,7 @@ async function main() {
direction,
force,
stopOnError,
quickOnly,
})
}
@ -41,12 +43,13 @@ Options:
--backwards Disable history ranges support for selected project ids
--force Migrate projects even if they were already migrated
--stop-on-error Stop after first migration error
--quick-only Do not try a resync migration if quick migration fails
`)
}
function parseArgs() {
const args = minimist(process.argv.slice(2), {
boolean: ['backwards', 'help', 'all', 'force'],
boolean: ['backwards', 'help', 'all', 'force', 'quick-only'],
string: ['owner-id', 'project-id', 'min-id', 'max-id'],
})
@ -63,6 +66,7 @@ function parseArgs() {
const maxCount = args['max-count']
const force = args.force
const stopOnError = args['stop-on-error']
const quickOnly = args['quick-only']
const all = args.all
if (
@ -89,6 +93,7 @@ function parseArgs() {
direction,
force,
stopOnError,
quickOnly,
}
}