From ec7cd9fc3eece1ac3f9567111918a143c232ae78 Mon Sep 17 00:00:00 2001 From: Jakob Ackermann Date: Tue, 6 Apr 2021 13:13:28 +0200 Subject: [PATCH] Merge pull request #3656 from overleaf/jpa-script-back-fill-doc-name [scripts] add new script for back filling the names of deleted docs GitOrigin-RevId: c3a7ad8ba1306728bc1a433bec5dc847651bf94d --- .../back_fill_doc_name_for_deleted_docs.js | 74 +++++++++++ .../src/BackFillDocNameForDeletedDocsTests.js | 118 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 services/web/scripts/back_fill_doc_name_for_deleted_docs.js create mode 100644 services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.js diff --git a/services/web/scripts/back_fill_doc_name_for_deleted_docs.js b/services/web/scripts/back_fill_doc_name_for_deleted_docs.js new file mode 100644 index 0000000000..805699f05b --- /dev/null +++ b/services/web/scripts/back_fill_doc_name_for_deleted_docs.js @@ -0,0 +1,74 @@ +const WRITE_CONCURRENCY = parseInt(process.env.WRITE_CONCURRENCY, 10) || 10 + +const { batchedUpdate } = require('./helpers/batchedUpdate') +const { promiseMapWithLimit, promisify } = require('../app/src/util/promises') +const { db } = require('../app/src/infrastructure/mongodb') +const sleep = promisify(setTimeout) + +const PERFORM_CLEANUP = process.argv.pop() === '--perform-cleanup' +const LET_USER_DOUBLE_CHECK_INPUTS_FOR = parseInt( + process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000, + 10 +) + +async function main() { + await letUserDoubleCheckInputs() + + await batchedUpdate( + 'projects', + // array is not empty ~ array has one item + { 'deletedDocs.0': { $exists: true } }, + processBatch, + { _id: 1, deletedDocs: 1 } + ) +} + +main() + .then(() => { + process.exit(0) + }) + .catch(error => { + console.error({ error }) + process.exit(1) + }) + +async function processBatch(_, projects) { + await promiseMapWithLimit(WRITE_CONCURRENCY, projects, processProject) +} + +async function processProject(project) { + for (const doc of project.deletedDocs) { + await backFillDoc(doc) + } + if (PERFORM_CLEANUP) { + await cleanupProject(project) + } +} + +async function backFillDoc(doc) { + const { name, deletedAt } = doc + await db.docs.updateOne({ _id: doc._id }, { $set: { name, deletedAt } }) +} + +async function cleanupProject(project) { + await db.projects.updateOne( + { _id: project._id }, + { $set: { deletedDocs: [] } } + ) +} + +async function letUserDoubleCheckInputs() { + if (PERFORM_CLEANUP) { + console.error('BACK FILLING AND PERFORMING CLEANUP') + } else { + console.error( + 'BACK FILLING ONLY - You will need to rerun with --perform-cleanup' + ) + } + console.error( + 'Waiting for you to double check inputs for', + LET_USER_DOUBLE_CHECK_INPUTS_FOR, + 'ms' + ) + await sleep(LET_USER_DOUBLE_CHECK_INPUTS_FOR) +} diff --git a/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.js b/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.js new file mode 100644 index 0000000000..53a0bee31c --- /dev/null +++ b/services/web/test/acceptance/src/BackFillDocNameForDeletedDocsTests.js @@ -0,0 +1,118 @@ +const { exec } = require('child_process') +const { promisify } = require('util') +const { expect } = require('chai') +const logger = require('logger-sharelatex') +const { db, ObjectId } = require('../../../app/src/infrastructure/mongodb') +const User = require('./helpers/User').promises + +async function getDeletedDocs(projectId) { + return (await db.projects.findOne({ _id: projectId })).deletedDocs +} + +async function setDeletedDocs(projectId, deletedDocs) { + await db.projects.updateOne({ _id: projectId }, { $set: { deletedDocs } }) +} + +describe('BackFillDocNameForDeletedDocs', function() { + let user, projectId1, projectId2, docId1, docId2, docId3 + beforeEach('create projects', async function() { + user = new User() + await user.login() + + projectId1 = ObjectId(await user.createProject('project1')) + projectId2 = ObjectId(await user.createProject('project2')) + }) + beforeEach('create docs', async function() { + docId1 = ObjectId( + await user.createDocInProject(projectId1, null, 'doc1.tex') + ) + docId2 = ObjectId( + await user.createDocInProject(projectId1, null, 'doc2.tex') + ) + docId3 = ObjectId( + await user.createDocInProject(projectId2, null, 'doc3.tex') + ) + }) + beforeEach('deleted docs', async function() { + await user.deleteItemInProject(projectId1, 'doc', docId1) + await user.deleteItemInProject(projectId1, 'doc', docId2) + await user.deleteItemInProject(projectId2, 'doc', docId3) + }) + beforeEach('insert doc stubs into docs collection', async function() { + await db.docs.insertMany([ + { _id: docId1, deleted: true }, + { _id: docId2, deleted: true }, + { _id: docId3, deleted: true } + ]) + }) + let deletedDocs1, deletedDocs2 + let deletedAt1, deletedAt2, deletedAt3 + beforeEach('set deletedDocs details', async function() { + deletedAt1 = new Date() + deletedAt2 = new Date() + deletedAt3 = new Date() + deletedDocs1 = [ + { _id: docId1, name: 'doc1.tex', deletedAt: deletedAt1 }, + { _id: docId2, name: 'doc2.tex', deletedAt: deletedAt2 } + ] + deletedDocs2 = [{ _id: docId3, name: 'doc3.tex', deletedAt: deletedAt3 }] + await setDeletedDocs(projectId1, deletedDocs1) + await setDeletedDocs(projectId2, deletedDocs2) + }) + + async function runScript(args = []) { + let result + try { + result = await promisify(exec)( + ['LET_USER_DOUBLE_CHECK_INPUTS_FOR=1'] + .concat(['node', 'scripts/back_fill_doc_name_for_deleted_docs']) + .concat(args) + .join(' ') + ) + } catch (error) { + // dump details like exit code, stdErr and stdOut + logger.error({ error }, 'script failed') + throw error + } + const { stderr: stdErr, stdout: stdOut } = result + expect(stdOut).to.include(projectId1.toString()) + expect(stdOut).to.include(projectId2.toString()) + + expect(stdErr).to.include(`Completed batch ending ${projectId2}`) + } + + function checkDocsBackFilled() { + it('should back fill names and deletedAt dates into docs', async function() { + const docs = await db.docs.find({}).toArray() + expect(docs).to.deep.equal([ + { _id: docId1, deleted: true, name: 'doc1.tex', deletedAt: deletedAt1 }, + { _id: docId2, deleted: true, name: 'doc2.tex', deletedAt: deletedAt2 }, + { _id: docId3, deleted: true, name: 'doc3.tex', deletedAt: deletedAt3 } + ]) + }) + } + + describe('back fill only', function() { + beforeEach('run script', runScript) + + checkDocsBackFilled() + + it('should leave the deletedDocs as is', async function() { + expect(await getDeletedDocs(projectId1)).to.deep.equal(deletedDocs1) + expect(await getDeletedDocs(projectId2)).to.deep.equal(deletedDocs2) + }) + }) + + describe('back fill and cleanup', function() { + beforeEach('run script with cleanup flag', async function() { + await runScript(['--perform-cleanup']) + }) + + checkDocsBackFilled() + + it('should cleanup the deletedDocs', async function() { + expect(await getDeletedDocs(projectId1)).to.deep.equal([]) + expect(await getDeletedDocs(projectId2)).to.deep.equal([]) + }) + }) +})