const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 const { batchedUpdate } = require('./helpers/batchedUpdate') const { promiseMapWithLimit, promisify } = require('../app/src/util/promises') const { db } = require('../app/src/infrastructure/mongodb') const sleep = promisify(setTimeout) const PERFORM_CLEANUP = process.argv.includes('--perform-cleanup') const FIX_PARTIAL_INSERTS = process.argv.includes('--fix-partial-inserts') const LET_USER_DOUBLE_CHECK_INPUTS_FOR = parseInt( process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000, 10 ) async function main() { await letUserDoubleCheckInputs() await batchedUpdate( 'projects', // array is not empty ~ array has one item { 'deletedFiles.0': { $exists: true } }, processBatch, { _id: 1, deletedFiles: 1 } ) } main() .then(() => { process.exit(0) }) .catch(error => { console.error({ error }) process.exit(1) }) async function processBatch(_, projects) { await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) } async function processProject(project) { await backFillFiles(project) if (PERFORM_CLEANUP) { await cleanupProject(project) } } async function backFillFiles(project) { const projectId = project._id filterDuplicatesInPlace(project) project.deletedFiles.forEach(file => { file.projectId = projectId }) if (FIX_PARTIAL_INSERTS) { await fixPartialInserts(project) } else { await db.deletedFiles.insertMany(project.deletedFiles) } } function filterDuplicatesInPlace(project) { const fileIds = new Set() project.deletedFiles = project.deletedFiles.filter(file => { const id = file._id.toString() if (fileIds.has(id)) return false fileIds.add(id) return true }) } async function fixPartialInserts(project) { const seenFileIds = new Set( ( await db.deletedFiles .find( { _id: { $in: project.deletedFiles.map(file => file._id) } }, { projection: { _id: 1 } } ) .toArray() ).map(file => file._id.toString()) ) project.deletedFiles = project.deletedFiles.filter(file => { const id = file._id.toString() if (seenFileIds.has(id)) return false seenFileIds.add(id) return true }) if (project.deletedFiles.length > 0) { await db.deletedFiles.insertMany(project.deletedFiles) } } async function cleanupProject(project) { await db.projects.updateOne( { _id: project._id }, { $set: { deletedFiles: [] } } ) } async function letUserDoubleCheckInputs() { if (PERFORM_CLEANUP) { console.error('BACK FILLING AND PERFORMING CLEANUP') } else { console.error( 'BACK FILLING ONLY - You will need to rerun with --perform-cleanup' ) } console.error( 'Waiting for you to double check inputs for', LET_USER_DOUBLE_CHECK_INPUTS_FOR, 'ms' ) await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR) }