overleaf/services/project-history/scripts/clear_deleted.js

137 lines
3.6 KiB
JavaScript
Raw Normal View History

#!/usr/bin/env node
import async from 'async'
import logger from '@overleaf/logger'
import Settings from '@overleaf/settings'
import redis from '@overleaf/redis-wrapper'
import { db, ObjectId } from '../app/js/mongodb.js'
logger.logger.level('fatal')
const rclient = redis.createClient(Settings.redis.project_history)
const Keys = Settings.redis.project_history.key_schema
const argv = process.argv.slice(2)
const limit = parseInt(argv[0], 10) || null
const force = argv[1] === 'force' || false
let delay = 0
function checkAndClear(project, callback) {
const projectId = project.project_id
function checkDeleted(cb) {
db.projects.findOne(
{ _id: new ObjectId(projectId) },
{ projection: { _id: 1 } },
(err, result) => {
if (err) {
cb(err)
} else if (!result) {
// project not found, but we still need to look at deletedProjects
cb()
} else {
console.log(`Project ${projectId} found in projects`)
cb(new Error('error: project still exists'))
}
}
)
}
function checkRecoverable(cb) {
db.deletedProjects.findOne(
{
// this condition makes use of the index
'deleterData.deletedProjectId': new ObjectId(projectId),
// this condition checks if the deleted project has expired
'project._id': new ObjectId(projectId),
},
{ projection: { _id: 1 } },
(err, result) => {
if (err) {
cb(err)
} else if (!result) {
console.log(
`project ${projectId} has been deleted - safe to clear queue`
)
cb()
} else {
console.log(`Project ${projectId} found in deletedProjects`)
cb(new Error('error: project still exists'))
}
}
)
}
function clearRedisQueue(cb) {
const key = Keys.projectHistoryOps({ project_id: projectId })
delay++
if (force) {
console.log('setting redis key', key, 'to expire in', delay, 'seconds')
// use expire to allow redis to delete the key in the background
rclient.expire(key, delay, err => {
cb(err)
})
} else {
console.log(
'dry run, would set key',
key,
'to expire in',
delay,
'seconds'
)
cb()
}
}
function clearMongoEntry(cb) {
if (force) {
console.log('deleting key in mongo projectHistoryFailures', projectId)
db.projectHistoryFailures.deleteOne({ project_id: projectId }, cb)
} else {
console.log('would delete failure record for', projectId, 'from mongo')
cb()
}
}
// do the checks and deletions
async.waterfall(
[checkDeleted, checkRecoverable, clearRedisQueue, clearMongoEntry],
err => {
if (!err || err.message === 'error: project still exists') {
callback()
} else {
console.log('error:', err)
callback(err)
}
}
)
}
// find all the broken projects from the failure records
async function main() {
const results = await db.projectHistoryFailures.find({}).toArray()
processFailures(results)
}
main().catch(error => {
console.error(error)
process.exit(1)
})
function processFailures(results) {
if (argv.length === 0) {
console.log(`
Usage: node clear_deleted.js [QUEUES] [FORCE]
where
QUEUES is the number of queues to process
FORCE is the string "force" when we're ready to delete the queues. Without it, this script does a dry-run
`)
}
console.log('number of stuck projects', results.length)
// now check if the project is truly deleted in mongo
async.eachSeries(results.slice(0, limit), checkAndClear, err => {
console.log('DONE', err)
process.exit()
})
}