github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #2361 from overleaf/em-project-imports-2

Browse source 
Import full folder structure in a single Mongo update

GitOrigin-RevId: 623d2a098b2084fdd0193e1593c1c55c08a2d92d
This commit is contained in:
Eric Mc Sween 2019-11-18 09:03:04 -05:00 committed by sharelatex
parent bf9473fb41
commit 27504d7b9d
8 changed files with 810 additions and 2262 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
services/web/app/src/Features/Project/FolderStructureBuilder.js Normal file
View file

@ -0,0 +1,73 @@
const Path = require('path')
const OError = require('@overleaf/o-error')
const { ObjectId } = require('mongodb')
module.exports = { buildFolderStructure }
function buildFolderStructure(docUploads, fileUploads) {
const builder = new FolderStructureBuilder()
for (const docUpload of docUploads) {
builder.addDocUpload(docUpload)
}
for (const fileUpload of fileUploads) {
builder.addFileUpload(fileUpload)
}
return builder.rootFolder
}
class FolderStructureBuilder {
constructor() {
this.foldersByPath = new Map()
this.entityPaths = new Set()
this.rootFolder = this.createFolder('rootFolder')
this.foldersByPath.set('/', this.rootFolder)
this.entityPaths.add('/')
}
addDocUpload(docUpload) {
this.recordEntityPath(Path.join(docUpload.dirname, docUpload.doc.name))
const folder = this.mkdirp(docUpload.dirname)
folder.docs.push(docUpload.doc)
}
addFileUpload(fileUpload) {
this.recordEntityPath(
Path.join(fileUpload.dirname, fileUpload.fileRef.name)
)
const folder = this.mkdirp(fileUpload.dirname)
folder.fileRefs.push(fileUpload.fileRef)
}
mkdirp(path) {
const existingFolder = this.foldersByPath.get(path)
if (existingFolder != null) {
return existingFolder
}
// Folder not found, create it.
this.recordEntityPath(path)
const dirname = Path.dirname(path)
const basename = Path.basename(path)
const parentFolder = this.mkdirp(dirname)
const newFolder = this.createFolder(basename)
parentFolder.folders.push(newFolder)
this.foldersByPath.set(path, newFolder)
return newFolder
}
recordEntityPath(path) {
if (this.entityPaths.has(path)) {
throw new OError({ message: 'entity already exists', info: { path } })
}
this.entityPaths.add(path)
}
createFolder(name) {
return {
_id: ObjectId(),
name,
folders: [],
docs: [],
fileRefs: []
}
}
}

2
services/web/app/src/Features/Project/ProjectDuplicator.js
View file

@ -35,7 +35,7 @@ module.exports = ProjectDuplicator = {
callback
) {
const setRootDoc = _.once(doc_id =>
ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id)
ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id, () => {})
)
const docs = originalFolder.docs || []
const jobs = docs.map(

36
services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js
View file

@ -10,6 +10,7 @@ const logger = require('logger-sharelatex')
const path = require('path')
const { ObjectId } = require('mongodb')
const Settings = require('settings-sharelatex')
const OError = require('@overleaf/o-error')
const CooldownManager = require('../Cooldown/CooldownManager')
const Errors = require('../Errors/Errors')
const { Folder } = require('../../models/Folder')
@ -18,6 +19,7 @@ const { Project } = require('../../models/Project')
const ProjectEntityHandler = require('./ProjectEntityHandler')
const ProjectGetter = require('./ProjectGetter')
const ProjectLocator = require('./ProjectLocator')
const FolderStructureBuilder = require('./FolderStructureBuilder')
const SafePath = require('./SafePath')
const LOCK_NAMESPACE = 'mongoTransaction'
@ -69,6 +71,7 @@ module.exports = {
'rev',
'changes'
]),
createNewFolderStructure: callbackify(wrapWithLock(createNewFolderStructure)),
_insertDeletedDocReference: callbackify(_insertDeletedDocReference),
_insertDeletedFileReference: callbackify(_insertDeletedFileReference),
_putElement: callbackifyMultiResult(_putElement, ['result', 'project']),
@ -82,6 +85,7 @@ module.exports = {
moveEntity: wrapWithLock(moveEntity),
deleteEntity: wrapWithLock(deleteEntity),
renameEntity: wrapWithLock(renameEntity),
createNewFolderStructure: wrapWithLock(createNewFolderStructure),
_insertDeletedDocReference,
_insertDeletedFileReference,
_putElement
@ -606,3 +610,35 @@ async function _checkValidMove(
}
}
}
async function createNewFolderStructure(projectId, docUploads, fileUploads) {
try {
const rootFolder = FolderStructureBuilder.buildFolderStructure(
docUploads,
fileUploads
)
const result = await Project.updateOne(
{
_id: projectId,
'rootFolder.0.folders.0': { $exists: false },
'rootFolder.0.docs.0': { $exists: false },
'rootFolder.0.files.0': { $exists: false }
},
{
$set: { rootFolder: [rootFolder] },
$inc: { version: 1 }
}
).exec()
if (result.n !== 1) {
throw new OError({
message: 'project not found or folder structure already exists',
info: { projectId }
})
}
} catch (err) {
throw new OError({
message: 'failed to create folder structure',
info: { projectId }
}).withCause(err)
}
}

1138
services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js
View file

File diff suppressed because it is too large Load diff

1634
services/web/app/src/Features/Project/ProjectEntityUpdateHandlerCanary.js
View file

File diff suppressed because it is too large Load diff

115
services/web/test/unit/src/Project/FolderStructureBuilderTests.js Normal file
View file

@ -0,0 +1,115 @@
const { expect } = require('chai')
const sinon = require('sinon')
const SandboxedModule = require('sandboxed-module')
const MODULE_PATH =
'../../../../app/src/Features/Project/FolderStructureBuilder'
const MOCK_OBJECT_ID = 'MOCK_OBJECT_ID'
describe('FolderStructureBuilder', function() {
beforeEach(function() {
this.ObjectId = sinon.stub().returns(MOCK_OBJECT_ID)
this.FolderStructureBuilder = SandboxedModule.require(MODULE_PATH, {
requires: {
mongodb: { ObjectId: this.ObjectId }
}
})
})
describe('buildFolderStructure', function() {
describe('when given no documents at all', function() {
beforeEach(function() {
this.result = this.FolderStructureBuilder.buildFolderStructure([], [])
})
it('returns an empty root folder', function() {
expect(this.result).to.deep.equal({
_id: MOCK_OBJECT_ID,
name: 'rootFolder',
folders: [],
docs: [],
fileRefs: []
})
})
})
describe('when given documents and files', function() {
beforeEach(function() {
const docUploads = [
{ dirname: '/', doc: { _id: 'doc-1', name: 'main.tex' } },
{ dirname: '/foo', doc: { _id: 'doc-2', name: 'other.tex' } },
{ dirname: '/foo', doc: { _id: 'doc-3', name: 'other.bib' } },
{
dirname: '/foo/foo1/foo2',
doc: { _id: 'doc-4', name: 'another.tex' }
}
]
const fileUploads = [
{ dirname: '/', fileRef: { _id: 'file-1', name: 'aaa.jpg' } },
{ dirname: '/foo', fileRef: { _id: 'file-2', name: 'bbb.jpg' } },
{ dirname: '/bar', fileRef: { _id: 'file-3', name: 'ccc.jpg' } }
]
this.result = this.FolderStructureBuilder.buildFolderStructure(
docUploads,
fileUploads
)
})
it('returns a full folder structure', function() {
expect(this.result).to.deep.equal({
_id: MOCK_OBJECT_ID,
name: 'rootFolder',
docs: [{ _id: 'doc-1', name: 'main.tex' }],
fileRefs: [{ _id: 'file-1', name: 'aaa.jpg' }],
folders: [
{
_id: MOCK_OBJECT_ID,
name: 'foo',
docs: [
{ _id: 'doc-2', name: 'other.tex' },
{ _id: 'doc-3', name: 'other.bib' }
],
fileRefs: [{ _id: 'file-2', name: 'bbb.jpg' }],
folders: [
{
_id: MOCK_OBJECT_ID,
name: 'foo1',
docs: [],
fileRefs: [],
folders: [
{
_id: MOCK_OBJECT_ID,
name: 'foo2',
docs: [{ _id: 'doc-4', name: 'another.tex' }],
fileRefs: [],
folders: []
}
]
}
]
},
{
_id: MOCK_OBJECT_ID,
name: 'bar',
docs: [],
fileRefs: [{ _id: 'file-3', name: 'ccc.jpg' }],
folders: []
}
]
})
})
})
describe('when given duplicate files', function() {
it('throws an error', function() {
const docUploads = [
{ dirname: '/foo', doc: { _id: 'doc-1', name: 'doc.tex' } },
{ dirname: '/foo', doc: { _id: 'doc-2', name: 'doc.tex' } }
]
expect(() =>
this.FolderStructureBuilder.buildFolderStructure(docUploads, [])
).to.throw()
})
})
})
})

58
services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js
View file

@ -177,6 +177,10 @@ describe('ProjectEntityMongoUpdateHandler', function() {
}
}
this.FolderStructureBuilder = {
buildFolderStructure: sinon.stub()
}
this.subject = SandboxedModule.require(MODULE_PATH, {
globals: {
console: console
@ -191,6 +195,7 @@ describe('ProjectEntityMongoUpdateHandler', function() {
'./ProjectEntityHandler': this.ProjectEntityHandler,
'./ProjectLocator': this.ProjectLocator,
'./ProjectGetter': this.ProjectGetter,
'./FolderStructureBuilder': this.FolderStructureBuilder,
// We need to provide Errors here to make instance check work
'../Errors/Errors': Errors
}
@ -1031,4 +1036,57 @@ describe('ProjectEntityMongoUpdateHandler', function() {
this.ProjectMock.verify()
})
})
describe('createNewFolderStructure', function() {
beforeEach(function() {
this.mockRootFolder = 'MOCK_ROOT_FOLDER'
this.docUploads = ['MOCK_DOC_UPLOAD']
this.fileUploads = ['MOCK_FILE_UPLOAD']
this.FolderStructureBuilder.buildFolderStructure
.withArgs(this.docUploads, this.fileUploads)
.returns(this.mockRootFolder)
this.updateExpectation = this.ProjectMock.expects('updateOne')
.withArgs(
{
_id: this.project._id,
'rootFolder.0.folders.0': { $exists: false },
'rootFolder.0.docs.0': { $exists: false },
'rootFolder.0.files.0': { $exists: false }
},
{ $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } }
)
.chain('exec')
})
describe('happy path', function() {
beforeEach(async function() {
this.updateExpectation.resolves({ n: 1 })
await this.subject.promises.createNewFolderStructure(
this.project._id,
this.docUploads,
this.fileUploads
)
})
it('updates the database', function() {
this.ProjectMock.verify()
})
})
describe("when the update doesn't find a matching document", function() {
beforeEach(async function() {
this.updateExpectation.resolves({ n: 0 })
})
it('throws an error', async function() {
await expect(
this.subject.promises.createNewFolderStructure(
this.project._id,
this.docUploads,
this.fileUploads
)
).to.be.rejected
})
})
})
})

16
services/web/test/unit/src/Project/ProjectEntityUpdateHandlerTests.js
View file

@ -453,8 +453,8 @@ describe('ProjectEntityUpdateHandler', function() {
return this.logger.warn
.calledWith(
{
project_id,
doc_id,
projectId: project_id,
docId: doc_id,
lines: this.docLines
},
'doc not found while updating doc lines'
@ -504,7 +504,11 @@ describe('ProjectEntityUpdateHandler', function() {
.stub()
.yields(null, `/main.tex`)
this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
this.ProjectEntityUpdateHandler.setRootDoc(
project_id,
this.rootDoc_id,
() => {}
)
return this.ProjectModel.update
.calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
.should.equal(true)
@ -516,7 +520,11 @@ describe('ProjectEntityUpdateHandler', function() {
.stub()
.yields(Errors.NotFoundError)
this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
this.ProjectEntityUpdateHandler.setRootDoc(
project_id,
this.rootDoc_id,
() => {}
)
return this.ProjectModel.update
.calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
.should.equal(false)