mirror of
https://github.com/overleaf/overleaf.git
synced 2024-11-21 20:47:08 -05:00
Merge pull request #2361 from overleaf/em-project-imports-2
Import full folder structure in a single Mongo update GitOrigin-RevId: 623d2a098b2084fdd0193e1593c1c55c08a2d92d
This commit is contained in:
parent
bf9473fb41
commit
27504d7b9d
8 changed files with 810 additions and 2262 deletions
|
@ -0,0 +1,73 @@
|
|||
const Path = require('path')
|
||||
const OError = require('@overleaf/o-error')
|
||||
const { ObjectId } = require('mongodb')
|
||||
|
||||
module.exports = { buildFolderStructure }
|
||||
|
||||
function buildFolderStructure(docUploads, fileUploads) {
|
||||
const builder = new FolderStructureBuilder()
|
||||
for (const docUpload of docUploads) {
|
||||
builder.addDocUpload(docUpload)
|
||||
}
|
||||
for (const fileUpload of fileUploads) {
|
||||
builder.addFileUpload(fileUpload)
|
||||
}
|
||||
return builder.rootFolder
|
||||
}
|
||||
|
||||
class FolderStructureBuilder {
|
||||
constructor() {
|
||||
this.foldersByPath = new Map()
|
||||
this.entityPaths = new Set()
|
||||
this.rootFolder = this.createFolder('rootFolder')
|
||||
this.foldersByPath.set('/', this.rootFolder)
|
||||
this.entityPaths.add('/')
|
||||
}
|
||||
|
||||
addDocUpload(docUpload) {
|
||||
this.recordEntityPath(Path.join(docUpload.dirname, docUpload.doc.name))
|
||||
const folder = this.mkdirp(docUpload.dirname)
|
||||
folder.docs.push(docUpload.doc)
|
||||
}
|
||||
|
||||
addFileUpload(fileUpload) {
|
||||
this.recordEntityPath(
|
||||
Path.join(fileUpload.dirname, fileUpload.fileRef.name)
|
||||
)
|
||||
const folder = this.mkdirp(fileUpload.dirname)
|
||||
folder.fileRefs.push(fileUpload.fileRef)
|
||||
}
|
||||
|
||||
mkdirp(path) {
|
||||
const existingFolder = this.foldersByPath.get(path)
|
||||
if (existingFolder != null) {
|
||||
return existingFolder
|
||||
}
|
||||
// Folder not found, create it.
|
||||
this.recordEntityPath(path)
|
||||
const dirname = Path.dirname(path)
|
||||
const basename = Path.basename(path)
|
||||
const parentFolder = this.mkdirp(dirname)
|
||||
const newFolder = this.createFolder(basename)
|
||||
parentFolder.folders.push(newFolder)
|
||||
this.foldersByPath.set(path, newFolder)
|
||||
return newFolder
|
||||
}
|
||||
|
||||
recordEntityPath(path) {
|
||||
if (this.entityPaths.has(path)) {
|
||||
throw new OError({ message: 'entity already exists', info: { path } })
|
||||
}
|
||||
this.entityPaths.add(path)
|
||||
}
|
||||
|
||||
createFolder(name) {
|
||||
return {
|
||||
_id: ObjectId(),
|
||||
name,
|
||||
folders: [],
|
||||
docs: [],
|
||||
fileRefs: []
|
||||
}
|
||||
}
|
||||
}
|
|
@ -35,7 +35,7 @@ module.exports = ProjectDuplicator = {
|
|||
callback
|
||||
) {
|
||||
const setRootDoc = _.once(doc_id =>
|
||||
ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id)
|
||||
ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id, () => {})
|
||||
)
|
||||
const docs = originalFolder.docs || []
|
||||
const jobs = docs.map(
|
||||
|
|
|
@ -10,6 +10,7 @@ const logger = require('logger-sharelatex')
|
|||
const path = require('path')
|
||||
const { ObjectId } = require('mongodb')
|
||||
const Settings = require('settings-sharelatex')
|
||||
const OError = require('@overleaf/o-error')
|
||||
const CooldownManager = require('../Cooldown/CooldownManager')
|
||||
const Errors = require('../Errors/Errors')
|
||||
const { Folder } = require('../../models/Folder')
|
||||
|
@ -18,6 +19,7 @@ const { Project } = require('../../models/Project')
|
|||
const ProjectEntityHandler = require('./ProjectEntityHandler')
|
||||
const ProjectGetter = require('./ProjectGetter')
|
||||
const ProjectLocator = require('./ProjectLocator')
|
||||
const FolderStructureBuilder = require('./FolderStructureBuilder')
|
||||
const SafePath = require('./SafePath')
|
||||
|
||||
const LOCK_NAMESPACE = 'mongoTransaction'
|
||||
|
@ -69,6 +71,7 @@ module.exports = {
|
|||
'rev',
|
||||
'changes'
|
||||
]),
|
||||
createNewFolderStructure: callbackify(wrapWithLock(createNewFolderStructure)),
|
||||
_insertDeletedDocReference: callbackify(_insertDeletedDocReference),
|
||||
_insertDeletedFileReference: callbackify(_insertDeletedFileReference),
|
||||
_putElement: callbackifyMultiResult(_putElement, ['result', 'project']),
|
||||
|
@ -82,6 +85,7 @@ module.exports = {
|
|||
moveEntity: wrapWithLock(moveEntity),
|
||||
deleteEntity: wrapWithLock(deleteEntity),
|
||||
renameEntity: wrapWithLock(renameEntity),
|
||||
createNewFolderStructure: wrapWithLock(createNewFolderStructure),
|
||||
_insertDeletedDocReference,
|
||||
_insertDeletedFileReference,
|
||||
_putElement
|
||||
|
@ -606,3 +610,35 @@ async function _checkValidMove(
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createNewFolderStructure(projectId, docUploads, fileUploads) {
|
||||
try {
|
||||
const rootFolder = FolderStructureBuilder.buildFolderStructure(
|
||||
docUploads,
|
||||
fileUploads
|
||||
)
|
||||
const result = await Project.updateOne(
|
||||
{
|
||||
_id: projectId,
|
||||
'rootFolder.0.folders.0': { $exists: false },
|
||||
'rootFolder.0.docs.0': { $exists: false },
|
||||
'rootFolder.0.files.0': { $exists: false }
|
||||
},
|
||||
{
|
||||
$set: { rootFolder: [rootFolder] },
|
||||
$inc: { version: 1 }
|
||||
}
|
||||
).exec()
|
||||
if (result.n !== 1) {
|
||||
throw new OError({
|
||||
message: 'project not found or folder structure already exists',
|
||||
info: { projectId }
|
||||
})
|
||||
}
|
||||
} catch (err) {
|
||||
throw new OError({
|
||||
message: 'failed to create folder structure',
|
||||
info: { projectId }
|
||||
}).withCause(err)
|
||||
}
|
||||
}
|
||||
|
|
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
@ -0,0 +1,115 @@
|
|||
const { expect } = require('chai')
|
||||
const sinon = require('sinon')
|
||||
const SandboxedModule = require('sandboxed-module')
|
||||
|
||||
const MODULE_PATH =
|
||||
'../../../../app/src/Features/Project/FolderStructureBuilder'
|
||||
const MOCK_OBJECT_ID = 'MOCK_OBJECT_ID'
|
||||
|
||||
describe('FolderStructureBuilder', function() {
|
||||
beforeEach(function() {
|
||||
this.ObjectId = sinon.stub().returns(MOCK_OBJECT_ID)
|
||||
this.FolderStructureBuilder = SandboxedModule.require(MODULE_PATH, {
|
||||
requires: {
|
||||
mongodb: { ObjectId: this.ObjectId }
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('buildFolderStructure', function() {
|
||||
describe('when given no documents at all', function() {
|
||||
beforeEach(function() {
|
||||
this.result = this.FolderStructureBuilder.buildFolderStructure([], [])
|
||||
})
|
||||
|
||||
it('returns an empty root folder', function() {
|
||||
expect(this.result).to.deep.equal({
|
||||
_id: MOCK_OBJECT_ID,
|
||||
name: 'rootFolder',
|
||||
folders: [],
|
||||
docs: [],
|
||||
fileRefs: []
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when given documents and files', function() {
|
||||
beforeEach(function() {
|
||||
const docUploads = [
|
||||
{ dirname: '/', doc: { _id: 'doc-1', name: 'main.tex' } },
|
||||
{ dirname: '/foo', doc: { _id: 'doc-2', name: 'other.tex' } },
|
||||
{ dirname: '/foo', doc: { _id: 'doc-3', name: 'other.bib' } },
|
||||
{
|
||||
dirname: '/foo/foo1/foo2',
|
||||
doc: { _id: 'doc-4', name: 'another.tex' }
|
||||
}
|
||||
]
|
||||
const fileUploads = [
|
||||
{ dirname: '/', fileRef: { _id: 'file-1', name: 'aaa.jpg' } },
|
||||
{ dirname: '/foo', fileRef: { _id: 'file-2', name: 'bbb.jpg' } },
|
||||
{ dirname: '/bar', fileRef: { _id: 'file-3', name: 'ccc.jpg' } }
|
||||
]
|
||||
this.result = this.FolderStructureBuilder.buildFolderStructure(
|
||||
docUploads,
|
||||
fileUploads
|
||||
)
|
||||
})
|
||||
|
||||
it('returns a full folder structure', function() {
|
||||
expect(this.result).to.deep.equal({
|
||||
_id: MOCK_OBJECT_ID,
|
||||
name: 'rootFolder',
|
||||
docs: [{ _id: 'doc-1', name: 'main.tex' }],
|
||||
fileRefs: [{ _id: 'file-1', name: 'aaa.jpg' }],
|
||||
folders: [
|
||||
{
|
||||
_id: MOCK_OBJECT_ID,
|
||||
name: 'foo',
|
||||
docs: [
|
||||
{ _id: 'doc-2', name: 'other.tex' },
|
||||
{ _id: 'doc-3', name: 'other.bib' }
|
||||
],
|
||||
fileRefs: [{ _id: 'file-2', name: 'bbb.jpg' }],
|
||||
folders: [
|
||||
{
|
||||
_id: MOCK_OBJECT_ID,
|
||||
name: 'foo1',
|
||||
docs: [],
|
||||
fileRefs: [],
|
||||
folders: [
|
||||
{
|
||||
_id: MOCK_OBJECT_ID,
|
||||
name: 'foo2',
|
||||
docs: [{ _id: 'doc-4', name: 'another.tex' }],
|
||||
fileRefs: [],
|
||||
folders: []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
_id: MOCK_OBJECT_ID,
|
||||
name: 'bar',
|
||||
docs: [],
|
||||
fileRefs: [{ _id: 'file-3', name: 'ccc.jpg' }],
|
||||
folders: []
|
||||
}
|
||||
]
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('when given duplicate files', function() {
|
||||
it('throws an error', function() {
|
||||
const docUploads = [
|
||||
{ dirname: '/foo', doc: { _id: 'doc-1', name: 'doc.tex' } },
|
||||
{ dirname: '/foo', doc: { _id: 'doc-2', name: 'doc.tex' } }
|
||||
]
|
||||
expect(() =>
|
||||
this.FolderStructureBuilder.buildFolderStructure(docUploads, [])
|
||||
).to.throw()
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
|
@ -177,6 +177,10 @@ describe('ProjectEntityMongoUpdateHandler', function() {
|
|||
}
|
||||
}
|
||||
|
||||
this.FolderStructureBuilder = {
|
||||
buildFolderStructure: sinon.stub()
|
||||
}
|
||||
|
||||
this.subject = SandboxedModule.require(MODULE_PATH, {
|
||||
globals: {
|
||||
console: console
|
||||
|
@ -191,6 +195,7 @@ describe('ProjectEntityMongoUpdateHandler', function() {
|
|||
'./ProjectEntityHandler': this.ProjectEntityHandler,
|
||||
'./ProjectLocator': this.ProjectLocator,
|
||||
'./ProjectGetter': this.ProjectGetter,
|
||||
'./FolderStructureBuilder': this.FolderStructureBuilder,
|
||||
// We need to provide Errors here to make instance check work
|
||||
'../Errors/Errors': Errors
|
||||
}
|
||||
|
@ -1031,4 +1036,57 @@ describe('ProjectEntityMongoUpdateHandler', function() {
|
|||
this.ProjectMock.verify()
|
||||
})
|
||||
})
|
||||
|
||||
describe('createNewFolderStructure', function() {
|
||||
beforeEach(function() {
|
||||
this.mockRootFolder = 'MOCK_ROOT_FOLDER'
|
||||
this.docUploads = ['MOCK_DOC_UPLOAD']
|
||||
this.fileUploads = ['MOCK_FILE_UPLOAD']
|
||||
this.FolderStructureBuilder.buildFolderStructure
|
||||
.withArgs(this.docUploads, this.fileUploads)
|
||||
.returns(this.mockRootFolder)
|
||||
this.updateExpectation = this.ProjectMock.expects('updateOne')
|
||||
.withArgs(
|
||||
{
|
||||
_id: this.project._id,
|
||||
'rootFolder.0.folders.0': { $exists: false },
|
||||
'rootFolder.0.docs.0': { $exists: false },
|
||||
'rootFolder.0.files.0': { $exists: false }
|
||||
},
|
||||
{ $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } }
|
||||
)
|
||||
.chain('exec')
|
||||
})
|
||||
|
||||
describe('happy path', function() {
|
||||
beforeEach(async function() {
|
||||
this.updateExpectation.resolves({ n: 1 })
|
||||
await this.subject.promises.createNewFolderStructure(
|
||||
this.project._id,
|
||||
this.docUploads,
|
||||
this.fileUploads
|
||||
)
|
||||
})
|
||||
|
||||
it('updates the database', function() {
|
||||
this.ProjectMock.verify()
|
||||
})
|
||||
})
|
||||
|
||||
describe("when the update doesn't find a matching document", function() {
|
||||
beforeEach(async function() {
|
||||
this.updateExpectation.resolves({ n: 0 })
|
||||
})
|
||||
|
||||
it('throws an error', async function() {
|
||||
await expect(
|
||||
this.subject.promises.createNewFolderStructure(
|
||||
this.project._id,
|
||||
this.docUploads,
|
||||
this.fileUploads
|
||||
)
|
||||
).to.be.rejected
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
@ -453,8 +453,8 @@ describe('ProjectEntityUpdateHandler', function() {
|
|||
return this.logger.warn
|
||||
.calledWith(
|
||||
{
|
||||
project_id,
|
||||
doc_id,
|
||||
projectId: project_id,
|
||||
docId: doc_id,
|
||||
lines: this.docLines
|
||||
},
|
||||
'doc not found while updating doc lines'
|
||||
|
@ -504,7 +504,11 @@ describe('ProjectEntityUpdateHandler', function() {
|
|||
.stub()
|
||||
.yields(null, `/main.tex`)
|
||||
|
||||
this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
|
||||
this.ProjectEntityUpdateHandler.setRootDoc(
|
||||
project_id,
|
||||
this.rootDoc_id,
|
||||
() => {}
|
||||
)
|
||||
return this.ProjectModel.update
|
||||
.calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
|
||||
.should.equal(true)
|
||||
|
@ -516,7 +520,11 @@ describe('ProjectEntityUpdateHandler', function() {
|
|||
.stub()
|
||||
.yields(Errors.NotFoundError)
|
||||
|
||||
this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
|
||||
this.ProjectEntityUpdateHandler.setRootDoc(
|
||||
project_id,
|
||||
this.rootDoc_id,
|
||||
() => {}
|
||||
)
|
||||
return this.ProjectModel.update
|
||||
.calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
|
||||
.should.equal(false)
|
||||
|
|
Loading…
Reference in a new issue