overleaf/services/document-updater/scripts/flush_track_changes.js

100 lines
3 KiB
JavaScript
Raw Normal View History

// Flush docs found in docsWithHistory sets in the project-history redis.
// We normally write docsWithHistory in the main redis, but some keys
// got into the project-history redis.
// Usage: DRY_RUN=false node scripts/flush_track_changes.js
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const rclientProjectHistory = require('@overleaf/redis-wrapper').createClient(
Settings.redis.history
)
const ProjectFlusher = require('../app/js/ProjectFlusher')
const keys = Settings.redis.history.key_schema
const request = require('request')
const util = require('util')
async function findAndProcessDocs(options) {
logger.info({ options }, 'flushing docs with pending track-changes')
let cursor = 0
do {
const [newCursor, projectKeys] = await rclientProjectHistory.scan(
cursor,
'MATCH',
keys.docsWithHistoryOps({ project_id: '*' }),
'COUNT',
options.limit
)
await flushProjects(projectKeys, options)
cursor = newCursor
} while (cursor !== '0')
}
async function flushProjects(projectKeys, options) {
const projectIds = ProjectFlusher._extractIds(projectKeys)
for (const projectId of projectIds) {
const docIds = await rclientProjectHistory.smembers(
keys.docsWithHistoryOps({ project_id: projectId })
)
logger.debug({ projectId, docIds }, 'checking docs')
if (docIds) {
for (const docId of docIds) {
try {
await flushDoc(projectId, docId, options)
} catch (err) {
logger.error({ projectId, docId, err }, 'error flushing doc')
}
}
}
}
}
function makeRequest(url, callback) {
request.post(url, function (err, res, body) {
if (err) {
return callback(err)
} else {
return callback(err, { res, body })
}
})
}
const makeRequestPromise = util.promisify(makeRequest)
async function flushDoc(projectId, docId, options) {
const url = `${Settings.apis.trackchanges.url}/project/${projectId}/doc/${docId}/flush`
logger.debug(
{ projectId, docId, url, options },
'flushing doc in track changes api'
)
if (options.dryRun) {
return
}
const { res, body } = await makeRequestPromise(url)
if (res && res.statusCode === 204) {
logger.debug(
{ projectId, docId, statusCode: res.statusCode },
'successful flush in track changes api'
)
// on a successful flush the doc id is removed from docsWithHistoryOps by track-changes
// in deleteAppliedDocUpdates BUT it will do it for the main redis, not the project history redis.
// So we need to remove the docId here:
await rclientProjectHistory.srem(
keys.docsWithHistoryOps({ project_id: projectId }),
docId
)
} else {
logger.debug(
{ projectId, docId, res, body },
'problem flushing in track changes api'
)
}
}
findAndProcessDocs({ limit: 1000, dryRun: process.env.DRY_RUN !== 'false' })
.then(result => {
rclientProjectHistory.quit()
console.log('DONE')
})
.catch(function (error) {
console.error(error)
process.exit(1)
})