github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3701 from overleaf/jpa-script-back-fill-deleted-files-stage-2

Browse source 
[scripts] back_fill_deleted_files: optionally fix partial inserts

GitOrigin-RevId: 7d48d311db1af0b38222725ca86cdba8951b16b3
This commit is contained in:
Shane Kilkelly 2021-04-01 09:18:28 +01:00 committed by Copybot
parent b9b4ca224c
commit ae212feec4
2 changed files with 58 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
services/web/scripts/back_fill_deleted_files.js
View file

@ -5,7 +5,8 @@ const { promiseMapWithLimit, promisify } = require('../app/src/util/promises')
const { db } = require('../app/src/infrastructure/mongodb')
const sleep = promisify(setTimeout)
const PERFORM_CLEANUP = process.argv.pop() === '--perform-cleanup'
const PERFORM_CLEANUP = process.argv.includes('--perform-cleanup')
const FIX_PARTIAL_INSERTS = process.argv.includes('--fix-partial-inserts')
const LET_USER_DOUBLE_CHECK_INPUTS_FOR = parseInt(
process.env.LET_USER_DOUBLE_CHECK_INPUTS_FOR || 10 * 1000,
10
@ -50,7 +51,12 @@ async function backFillFiles(project) {
project.deletedFiles.forEach(file => {
file.projectId = projectId
})
await db.deletedFiles.insertMany(project.deletedFiles)
if (FIX_PARTIAL_INSERTS) {
await fixPartialInserts(project)
} else {
await db.deletedFiles.insertMany(project.deletedFiles)
}
}
function filterDuplicatesInPlace(project) {
@ -63,6 +69,28 @@ function filterDuplicatesInPlace(project) {
})
}
async function fixPartialInserts(project) {
const seenFileIds = new Set(
(
await db.deletedFiles
.find(
{ _id: { $in: project.deletedFiles.map(file => file._id) } },
{ projection: { _id: 1 } }
)
.toArray()
).map(file => file._id.toString())
)
project.deletedFiles = project.deletedFiles.filter(file => {
const id = file._id.toString()
if (seenFileIds.has(id)) return false
seenFileIds.add(id)
return true
})
if (project.deletedFiles.length > 0) {
await db.deletedFiles.insertMany(project.deletedFiles)
}
}
async function cleanupProject(project) {
await db.projects.updateOne(
{ _id: project._id },

28
services/web/test/acceptance/src/BackFillDeletedFilesTests.js
View file

@ -131,4 +131,32 @@ describe('BackFillDeletedFiles', function() {
expect(await getDeletedFiles(projectId5)).to.deep.equal([])
})
})
describe('fix partial inserts and cleanup', function() {
beforeEach('simulate one missing insert', async function() {
await setDeletedFiles(projectId1, [deletedFiles1[0]])
})
beforeEach('run script with cleanup flag', async function() {
await runScript(['--perform-cleanup'])
})
beforeEach('add case for one missing file', async function() {
await setDeletedFiles(projectId1, deletedFiles1)
})
beforeEach('add cases for no more files to insert', async function() {
await setDeletedFiles(projectId2, deletedFiles2)
await setDeletedFiles(projectId5, deletedFiles3)
})
beforeEach('fixing partial insert and cleanup', async function() {
await runScript(['--fix-partial-inserts', '--perform-cleanup'])
})
checkAreFilesBackFilled()
it('should cleanup the deletedFiles', async function() {
expect(await getDeletedFiles(projectId1)).to.deep.equal([])
expect(await getDeletedFiles(projectId2)).to.deep.equal([])
expect(await getDeletedFiles(projectId5)).to.deep.equal([])
})
})
})