overleaf/services/document-updater/app/js/HistoryManager.js
Eric Mc Sween e73fdfba63 Merge pull request #18906 from overleaf/em-migrate-existing-histories-2
History ranges migration script - second attempt

GitOrigin-RevId: 60a2c04e2a72e76a58e9e179fefc4186a96fde32
2024-06-19 08:05:02 +00:00

141 lines
4 KiB
JavaScript

const async = require('async')
const logger = require('@overleaf/logger')
const { promisifyAll } = require('@overleaf/promise-utils')
const request = require('request')
const Settings = require('@overleaf/settings')
const ProjectHistoryRedisManager = require('./ProjectHistoryRedisManager')
const metrics = require('./Metrics')
const HistoryManager = {
// flush changes in the background
flushProjectChangesAsync(projectId) {
HistoryManager.flushProjectChanges(
projectId,
{ background: true },
function () {}
)
},
// flush changes and callback (for when we need to know the queue is flushed)
flushProjectChanges(projectId, options, callback) {
if (callback == null) {
callback = function () {}
}
if (options.skip_history_flush) {
logger.debug({ projectId }, 'skipping flush of project history')
return callback()
}
metrics.inc('history-flush', 1, { status: 'project-history' })
const url = `${Settings.apis.project_history.url}/project/${projectId}/flush`
const qs = {}
if (options.background) {
qs.background = true
} // pass on the background flush option if present
logger.debug({ projectId, url, qs }, 'flushing doc in project history api')
request.post({ url, qs }, function (error, res, body) {
if (error) {
logger.error({ error, projectId }, 'project history api request failed')
callback(error)
} else if (res.statusCode < 200 && res.statusCode >= 300) {
logger.error(
{ projectId },
`project history api returned a failure status code: ${res.statusCode}`
)
callback(error)
} else {
callback()
}
})
},
FLUSH_DOC_EVERY_N_OPS: 100,
FLUSH_PROJECT_EVERY_N_OPS: 500,
recordAndFlushHistoryOps(projectId, ops, projectOpsLength) {
if (ops == null) {
ops = []
}
if (ops.length === 0) {
return
}
// record updates for project history
if (
HistoryManager.shouldFlushHistoryOps(
projectOpsLength,
ops.length,
HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
)
) {
// Do this in the background since it uses HTTP and so may be too
// slow to wait for when processing a doc update.
logger.debug(
{ projectOpsLength, projectId },
'flushing project history api'
)
HistoryManager.flushProjectChangesAsync(projectId)
}
},
shouldFlushHistoryOps(length, opsLength, threshold) {
if (!length) {
return false
} // don't flush unless we know the length
// We want to flush every 100 ops, i.e. 100, 200, 300, etc
// Find out which 'block' (i.e. 0-99, 100-199) we were in before and after pushing these
// ops. If we've changed, then we've gone over a multiple of 100 and should flush.
// (Most of the time, we will only hit 100 and then flushing will put us back to 0)
const previousLength = length - opsLength
const prevBlock = Math.floor(previousLength / threshold)
const newBlock = Math.floor(length / threshold)
return newBlock !== prevBlock
},
MAX_PARALLEL_REQUESTS: 4,
resyncProjectHistory(
projectId,
projectHistoryId,
docs,
files,
opts,
callback
) {
ProjectHistoryRedisManager.queueResyncProjectStructure(
projectId,
projectHistoryId,
docs,
files,
function (error) {
if (error) {
return callback(error)
}
const DocumentManager = require('./DocumentManager')
const resyncDoc = (doc, cb) => {
DocumentManager.resyncDocContentsWithLock(
projectId,
doc.doc,
doc.path,
opts,
cb
)
}
async.eachLimit(
docs,
HistoryManager.MAX_PARALLEL_REQUESTS,
resyncDoc,
callback
)
}
)
},
}
module.exports = HistoryManager
module.exports.promises = promisifyAll(HistoryManager, {
without: [
'flushProjectChangesAsync',
'recordAndFlushHistoryOps',
'shouldFlushHistoryOps',
],
})