mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-07 20:31:06 -05:00
Merge pull request #19151 from overleaf/em-history-ranges-quick-migration
Add quick history ranges support migration GitOrigin-RevId: 8446beb6bcd7384c32fc1b216e4b72d8f5d91500
This commit is contained in:
parent
e8e31dbdb5
commit
a95c0bbfc3
7 changed files with 202 additions and 4 deletions
|
@ -50,6 +50,7 @@ app.param('doc_id', function (req, res, next, docId) {
|
|||
app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs)
|
||||
app.get('/project/:project_id/doc', HttpController.getAllDocs)
|
||||
app.get('/project/:project_id/ranges', HttpController.getAllRanges)
|
||||
app.get('/project/:project_id/has-ranges', HttpController.projectHasRanges)
|
||||
app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
|
||||
app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted)
|
||||
app.get('/project/:project_id/doc/:doc_id/raw', HttpController.getRawDoc)
|
||||
|
|
|
@ -131,6 +131,25 @@ const DocManager = {
|
|||
return docs
|
||||
},
|
||||
|
||||
async projectHasRanges(projectId) {
|
||||
const docs = await MongoManager.promises.getProjectsDocs(
|
||||
projectId,
|
||||
{},
|
||||
{ _id: 1 }
|
||||
)
|
||||
const docIds = docs.map(doc => doc._id)
|
||||
for (const docId of docIds) {
|
||||
const doc = await DocManager.peekDoc(projectId, docId)
|
||||
if (
|
||||
(doc.ranges?.comments != null && doc.ranges.comments.length > 0) ||
|
||||
(doc.ranges?.changes != null && doc.ranges.changes.length > 0)
|
||||
) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
},
|
||||
|
||||
async updateDoc(projectId, docId, lines, version, ranges) {
|
||||
const MAX_ATTEMPTS = 2
|
||||
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
||||
|
|
|
@ -130,6 +130,16 @@ function getAllRanges(req, res, next) {
|
|||
)
|
||||
}
|
||||
|
||||
function projectHasRanges(req, res, next) {
|
||||
const { project_id: projectId } = req.params
|
||||
DocManager.projectHasRanges(projectId, (err, projectHasRanges) => {
|
||||
if (err) {
|
||||
return next(err)
|
||||
}
|
||||
res.json({ projectHasRanges })
|
||||
})
|
||||
}
|
||||
|
||||
function updateDoc(req, res, next) {
|
||||
const { doc_id: docId, project_id: projectId } = req.params
|
||||
const lines = req.body?.lines
|
||||
|
@ -298,6 +308,7 @@ module.exports = {
|
|||
getAllDocs,
|
||||
getAllDeletedDocs,
|
||||
getAllRanges,
|
||||
projectHasRanges,
|
||||
updateDoc,
|
||||
patchDoc,
|
||||
archiveAllDocs,
|
||||
|
|
|
@ -219,6 +219,31 @@ function updateDoc(projectId, docId, lines, version, ranges, callback) {
|
|||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Asks docstore whether any doc in the project has ranges
|
||||
*
|
||||
* @param {string} proejctId
|
||||
* @param {Callback} callback
|
||||
*/
|
||||
function projectHasRanges(projectId, callback) {
|
||||
const url = `${settings.apis.docstore.url}/project/${projectId}/has-ranges`
|
||||
request.get({ url, timeout: TIMEOUT, json: true }, (err, res, body) => {
|
||||
if (err) {
|
||||
return callback(err)
|
||||
}
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||||
callback(null, body.projectHasRanges)
|
||||
} else {
|
||||
callback(
|
||||
new OError(
|
||||
`docstore api responded with non-success code: ${res.statusCode}`,
|
||||
{ projectId }
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function archiveProject(projectId, callback) {
|
||||
_operateOnProject(projectId, 'archive', callback)
|
||||
}
|
||||
|
@ -266,6 +291,7 @@ module.exports = {
|
|||
getDoc,
|
||||
isDocDeleted,
|
||||
updateDoc,
|
||||
projectHasRanges,
|
||||
archiveProject,
|
||||
unarchiveProject,
|
||||
destroyProject,
|
||||
|
@ -277,6 +303,7 @@ module.exports = {
|
|||
getDoc: promisifyMultiResult(getDoc, ['lines', 'rev', 'version', 'ranges']),
|
||||
isDocDeleted: promisify(isDocDeleted),
|
||||
updateDoc: promisifyMultiResult(updateDoc, ['modified', 'rev']),
|
||||
projectHasRanges: promisify(projectHasRanges),
|
||||
archiveProject: promisify(archiveProject),
|
||||
unarchiveProject: promisify(unarchiveProject),
|
||||
destroyProject: promisify(destroyProject),
|
||||
|
|
|
@ -247,6 +247,46 @@ function resyncProjectHistory(
|
|||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Block a project from being loaded in docupdater
|
||||
*
|
||||
* @param {string} projectId
|
||||
* @param {Callback} callback
|
||||
*/
|
||||
function blockProject(projectId, callback) {
|
||||
_makeRequest(
|
||||
{ path: `/project/${projectId}/block`, method: 'POST', json: true },
|
||||
projectId,
|
||||
'block-project',
|
||||
(err, body) => {
|
||||
if (err) {
|
||||
return callback(err)
|
||||
}
|
||||
callback(null, body.blocked)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Unblock a previously blocked project
|
||||
*
|
||||
* @param {string} projectId
|
||||
* @param {Callback} callback
|
||||
*/
|
||||
function unblockProject(projectId, callback) {
|
||||
_makeRequest(
|
||||
{ path: `/project/${projectId}/unblock`, method: 'POST', json: true },
|
||||
projectId,
|
||||
'unblock-project',
|
||||
(err, body) => {
|
||||
if (err) {
|
||||
return callback(err)
|
||||
}
|
||||
callback(null, body.wasBlocked)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function updateProjectStructure(
|
||||
projectId,
|
||||
projectHistoryId,
|
||||
|
@ -460,6 +500,8 @@ module.exports = {
|
|||
reopenThread,
|
||||
deleteThread,
|
||||
resyncProjectHistory,
|
||||
blockProject,
|
||||
unblockProject,
|
||||
updateProjectStructure,
|
||||
promises: {
|
||||
flushProjectToMongo: promisify(flushProjectToMongo),
|
||||
|
@ -481,6 +523,8 @@ module.exports = {
|
|||
reopenThread: promisify(reopenThread),
|
||||
deleteThread: promisify(deleteThread),
|
||||
resyncProjectHistory: promisify(resyncProjectHistory),
|
||||
blockProject: promisify(blockProject),
|
||||
unblockProject: promisify(unblockProject),
|
||||
updateProjectStructure: promisify(updateProjectStructure),
|
||||
},
|
||||
}
|
||||
|
|
|
@ -2,8 +2,12 @@
|
|||
|
||||
const { callbackify } = require('util')
|
||||
const { ObjectId } = require('mongodb')
|
||||
const OError = require('@overleaf/o-error')
|
||||
const logger = require('@overleaf/logger')
|
||||
const HistoryManager = require('../History/HistoryManager')
|
||||
const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
|
||||
const DocstoreManager = require('../Docstore/DocstoreManager')
|
||||
const ProjectOptionsHandler = require('../Project/ProjectOptionsHandler')
|
||||
const { db } = require('../../infrastructure/mongodb')
|
||||
|
||||
/**
|
||||
|
@ -18,6 +22,7 @@ const { db } = require('../../infrastructure/mongodb')
|
|||
* @param {"forwards" | "backwards"} [opts.direction]
|
||||
* @param {boolean} [opts.force]
|
||||
* @param {boolean} [opts.stopOnError]
|
||||
* @param {boolean} [opts.quickOnly]
|
||||
*/
|
||||
async function migrateProjects(opts = {}) {
|
||||
const {
|
||||
|
@ -29,6 +34,7 @@ async function migrateProjects(opts = {}) {
|
|||
direction = 'forwards',
|
||||
force = false,
|
||||
stopOnError = false,
|
||||
quickOnly = false,
|
||||
} = opts
|
||||
|
||||
const clauses = []
|
||||
|
@ -76,11 +82,22 @@ async function migrateProjects(opts = {}) {
|
|||
}
|
||||
|
||||
const startTimeMs = Date.now()
|
||||
let quickMigrationSuccess
|
||||
try {
|
||||
await migrateProject(projectId, direction)
|
||||
quickMigrationSuccess = await quickMigration(projectId, direction)
|
||||
if (!quickMigrationSuccess) {
|
||||
if (quickOnly) {
|
||||
logger.info(
|
||||
{ projectId, direction },
|
||||
'Quick migration failed, skipping project'
|
||||
)
|
||||
} else {
|
||||
await migrateProject(projectId, direction)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
{ projectId, direction, projectsProcessed },
|
||||
{ err, projectId, direction, projectsProcessed },
|
||||
'Failed to migrate history ranges support'
|
||||
)
|
||||
projectsProcessed += 1
|
||||
|
@ -93,12 +110,73 @@ async function migrateProjects(opts = {}) {
|
|||
const elapsedMs = Date.now() - startTimeMs
|
||||
projectsProcessed += 1
|
||||
logger.info(
|
||||
{ projectId, direction, projectsProcessed, elapsedMs },
|
||||
{
|
||||
projectId,
|
||||
direction,
|
||||
projectsProcessed,
|
||||
elapsedMs,
|
||||
quick: quickMigrationSuccess,
|
||||
},
|
||||
'Migrated history ranges support'
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt a quick migration (without resync)
|
||||
*
|
||||
* @param {string} projectId
|
||||
* @param {"forwards" | "backwards"} direction
|
||||
* @return {Promise<boolean>} whether or not the quick migration was a success
|
||||
*/
|
||||
async function quickMigration(projectId, direction = 'forwards') {
|
||||
const blockSuccess =
|
||||
await DocumentUpdaterHandler.promises.blockProject(projectId)
|
||||
if (!blockSuccess) {
|
||||
return false
|
||||
}
|
||||
|
||||
let projectHasRanges
|
||||
try {
|
||||
projectHasRanges =
|
||||
await DocstoreManager.promises.projectHasRanges(projectId)
|
||||
} catch (err) {
|
||||
await DocumentUpdaterHandler.promises.unblockProject(projectId)
|
||||
throw err
|
||||
}
|
||||
if (projectHasRanges) {
|
||||
await DocumentUpdaterHandler.promises.unblockProject(projectId)
|
||||
return false
|
||||
}
|
||||
|
||||
try {
|
||||
await ProjectOptionsHandler.promises.setHistoryRangesSupport(
|
||||
projectId,
|
||||
direction === 'forwards'
|
||||
)
|
||||
} catch (err) {
|
||||
await DocumentUpdaterHandler.promises.unblockProject(projectId)
|
||||
await hardResyncProject(projectId)
|
||||
throw err
|
||||
}
|
||||
|
||||
let wasBlocked
|
||||
try {
|
||||
wasBlocked = await DocumentUpdaterHandler.promises.unblockProject(projectId)
|
||||
} catch (err) {
|
||||
await hardResyncProject(projectId)
|
||||
throw err
|
||||
}
|
||||
if (!wasBlocked) {
|
||||
await hardResyncProject(projectId)
|
||||
throw new OError('Tried to unblock project but it was not blocked', {
|
||||
projectId,
|
||||
})
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrate a single project
|
||||
*
|
||||
|
@ -112,6 +190,19 @@ async function migrateProject(projectId, direction = 'forwards') {
|
|||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Hard resync a project
|
||||
*
|
||||
* This is used when something goes wrong with the quick migration after we've
|
||||
* changed the history ranges support flag on a project.
|
||||
*
|
||||
* @param {string} projectId
|
||||
*/
|
||||
async function hardResyncProject(projectId) {
|
||||
await HistoryManager.promises.flushProject(projectId)
|
||||
await HistoryManager.promises.resyncProject(projectId, { force: true })
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
migrateProjects: callbackify(migrateProjects),
|
||||
migrateProject: callbackify(migrateProject),
|
||||
|
|
|
@ -13,6 +13,7 @@ async function main() {
|
|||
direction,
|
||||
force,
|
||||
stopOnError,
|
||||
quickOnly,
|
||||
} = parseArgs()
|
||||
await HistoryRangesSupportMigration.promises.migrateProjects({
|
||||
projectIds,
|
||||
|
@ -23,6 +24,7 @@ async function main() {
|
|||
direction,
|
||||
force,
|
||||
stopOnError,
|
||||
quickOnly,
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -41,12 +43,13 @@ Options:
|
|||
--backwards Disable history ranges support for selected project ids
|
||||
--force Migrate projects even if they were already migrated
|
||||
--stop-on-error Stop after first migration error
|
||||
--quick-only Do not try a resync migration if quick migration fails
|
||||
`)
|
||||
}
|
||||
|
||||
function parseArgs() {
|
||||
const args = minimist(process.argv.slice(2), {
|
||||
boolean: ['backwards', 'help', 'all', 'force'],
|
||||
boolean: ['backwards', 'help', 'all', 'force', 'quick-only'],
|
||||
string: ['owner-id', 'project-id', 'min-id', 'max-id'],
|
||||
})
|
||||
|
||||
|
@ -63,6 +66,7 @@ function parseArgs() {
|
|||
const maxCount = args['max-count']
|
||||
const force = args.force
|
||||
const stopOnError = args['stop-on-error']
|
||||
const quickOnly = args['quick-only']
|
||||
const all = args.all
|
||||
|
||||
if (
|
||||
|
@ -89,6 +93,7 @@ function parseArgs() {
|
|||
direction,
|
||||
force,
|
||||
stopOnError,
|
||||
quickOnly,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in a new issue