github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #3678 from overleaf/jpa-back-fill-deleted-files-handle-duplicates

Browse source 
[scripts] back_fill_deleted_files: handle duplicate deleted files

GitOrigin-RevId: 3f2f1b662727d61d2da2800ad0d635b65562164b
This commit is contained in:
Jakob Ackermann 2021-02-19 09:54:40 +00:00 committed by Copybot
parent 450d744f9a
commit 30cf22c8a9
2 changed files with 27 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
services/web/scripts/back_fill_deleted_files.js
View file

@ -46,12 +46,23 @@ async function processProject(project) {
async function backFillFiles(project) { async function backFillFiles(project) {
const projectId = project._id const projectId = project._id
filterDuplicatesInPlace(project)
project.deletedFiles.forEach(file => { project.deletedFiles.forEach(file => {
file.projectId = projectId file.projectId = projectId
}) })
await db.deletedFiles.insertMany(project.deletedFiles) await db.deletedFiles.insertMany(project.deletedFiles)
} }
function filterDuplicatesInPlace(project) {
const fileIds = new Set()
project.deletedFiles = project.deletedFiles.filter(file => {
const id = file._id.toString()
if (fileIds.has(id)) return false
fileIds.add(id)
return true
})
}
async function cleanupProject(project) { async function cleanupProject(project) {
await db.projects.updateOne( await db.projects.updateOne(
{ _id: project._id }, { _id: project._id },

21
services/web/test/acceptance/src/BackFillDeletedFilesTests.js
View file

@ -21,7 +21,7 @@ async function unsetDeletedFiles(projectId) {
} }
describe('BackFillDeletedFiles', function() { describe('BackFillDeletedFiles', function() {
let user, projectId1, projectId2, projectId3, projectId4 let user, projectId1, projectId2, projectId3, projectId4, projectId5
beforeEach('create projects', async function() { beforeEach('create projects', async function() {
user = new User() user = new User()
@ -31,14 +31,16 @@ describe('BackFillDeletedFiles', function() {
projectId2 = ObjectId(await user.createProject('project2')) projectId2 = ObjectId(await user.createProject('project2'))
projectId3 = ObjectId(await user.createProject('project3')) projectId3 = ObjectId(await user.createProject('project3'))
projectId4 = ObjectId(await user.createProject('project4')) projectId4 = ObjectId(await user.createProject('project4'))
projectId5 = ObjectId(await user.createProject('project5'))
}) })
let fileId1, fileId2, fileId3 let fileId1, fileId2, fileId3, fileId4
beforeEach('create files', function() { beforeEach('create files', function() {
// take a short cut and just allocate file ids // take a short cut and just allocate file ids
fileId1 = ObjectId() fileId1 = ObjectId()
fileId2 = ObjectId() fileId2 = ObjectId()
fileId3 = ObjectId() fileId3 = ObjectId()
fileId4 = ObjectId()
}) })
const otherFileDetails = { const otherFileDetails = {
name: 'universe.jpg', name: 'universe.jpg',
@ -47,7 +49,7 @@ describe('BackFillDeletedFiles', function() {
deletedAt: new Date(), deletedAt: new Date(),
__v: 0 __v: 0
} }
let deletedFiles1, deletedFiles2 let deletedFiles1, deletedFiles2, deletedFiles3
beforeEach('set deletedFiles details', async function() { beforeEach('set deletedFiles details', async function() {
deletedFiles1 = [ deletedFiles1 = [
{ _id: fileId1, ...otherFileDetails }, { _id: fileId1, ...otherFileDetails },
@ -61,6 +63,12 @@ describe('BackFillDeletedFiles', function() {
await setDeletedFiles(projectId3, []) await setDeletedFiles(projectId3, [])
// a project without deletedFiles array // a project without deletedFiles array
await unsetDeletedFiles(projectId4) await unsetDeletedFiles(projectId4)
// duplicate entry
deletedFiles3 = [
{ _id: fileId4, ...otherFileDetails },
{ _id: fileId4, ...otherFileDetails }
]
await setDeletedFiles(projectId5, deletedFiles3)
}) })
async function runScript(args = []) { async function runScript(args = []) {
@ -81,7 +89,7 @@ describe('BackFillDeletedFiles', function() {
expect(stdOut).to.include(projectId1.toString()) expect(stdOut).to.include(projectId1.toString())
expect(stdOut).to.include(projectId2.toString()) expect(stdOut).to.include(projectId2.toString())
expect(stdErr).to.include(`Completed batch ending ${projectId2}`) expect(stdErr).to.include(`Completed batch ending ${projectId5}`)
} }
function checkAreFilesBackFilled() { function checkAreFilesBackFilled() {
@ -92,7 +100,8 @@ describe('BackFillDeletedFiles', function() {
expect(docs).to.deep.equal([ expect(docs).to.deep.equal([
{ _id: fileId1, projectId: projectId1, ...otherFileDetails }, { _id: fileId1, projectId: projectId1, ...otherFileDetails },
{ _id: fileId2, projectId: projectId1, ...otherFileDetails }, { _id: fileId2, projectId: projectId1, ...otherFileDetails },
{ _id: fileId3, projectId: projectId2, ...otherFileDetails } { _id: fileId3, projectId: projectId2, ...otherFileDetails },
{ _id: fileId4, projectId: projectId5, ...otherFileDetails }
]) ])
}) })
} }
@ -105,6 +114,7 @@ describe('BackFillDeletedFiles', function() {
it('should leave the deletedFiles as is', async function() { it('should leave the deletedFiles as is', async function() {
expect(await getDeletedFiles(projectId1)).to.deep.equal(deletedFiles1) expect(await getDeletedFiles(projectId1)).to.deep.equal(deletedFiles1)
expect(await getDeletedFiles(projectId2)).to.deep.equal(deletedFiles2) expect(await getDeletedFiles(projectId2)).to.deep.equal(deletedFiles2)
expect(await getDeletedFiles(projectId5)).to.deep.equal(deletedFiles3)
}) })
}) })
@ -118,6 +128,7 @@ describe('BackFillDeletedFiles', function() {
it('should cleanup the deletedFiles', async function() { it('should cleanup the deletedFiles', async function() {
expect(await getDeletedFiles(projectId1)).to.deep.equal([]) expect(await getDeletedFiles(projectId1)).to.deep.equal([])
expect(await getDeletedFiles(projectId2)).to.deep.equal([]) expect(await getDeletedFiles(projectId2)).to.deep.equal([])
expect(await getDeletedFiles(projectId5)).to.deep.equal([])
}) })
}) })
}) })