Merge pull request #2361 from overleaf/em-project-imports-2

Import full folder structure in a single Mongo update

GitOrigin-RevId: 623d2a098b2084fdd0193e1593c1c55c08a2d92d

services/web/app/src/Features/Project/FolderStructureBuilder.js

@@ -0,0 +1,73 @@
+const Path = require('path')
+const OError = require('@overleaf/o-error')
+const { ObjectId } = require('mongodb')
+
+module.exports = { buildFolderStructure }
+
+function buildFolderStructure(docUploads, fileUploads) {
+  const builder = new FolderStructureBuilder()
+  for (const docUpload of docUploads) {
+    builder.addDocUpload(docUpload)
+  }
+  for (const fileUpload of fileUploads) {
+    builder.addFileUpload(fileUpload)
+  }
+  return builder.rootFolder
+}
+
+class FolderStructureBuilder {
+  constructor() {
+    this.foldersByPath = new Map()
+    this.entityPaths = new Set()
+    this.rootFolder = this.createFolder('rootFolder')
+    this.foldersByPath.set('/', this.rootFolder)
+    this.entityPaths.add('/')
+  }
+
+  addDocUpload(docUpload) {
+    this.recordEntityPath(Path.join(docUpload.dirname, docUpload.doc.name))
+    const folder = this.mkdirp(docUpload.dirname)
+    folder.docs.push(docUpload.doc)
+  }
+
+  addFileUpload(fileUpload) {
+    this.recordEntityPath(
+      Path.join(fileUpload.dirname, fileUpload.fileRef.name)
+    )
+    const folder = this.mkdirp(fileUpload.dirname)
+    folder.fileRefs.push(fileUpload.fileRef)
+  }
+
+  mkdirp(path) {
+    const existingFolder = this.foldersByPath.get(path)
+    if (existingFolder != null) {
+      return existingFolder
+    }
+    // Folder not found, create it.
+    this.recordEntityPath(path)
+    const dirname = Path.dirname(path)
+    const basename = Path.basename(path)
+    const parentFolder = this.mkdirp(dirname)
+    const newFolder = this.createFolder(basename)
+    parentFolder.folders.push(newFolder)
+    this.foldersByPath.set(path, newFolder)
+    return newFolder
+  }
+
+  recordEntityPath(path) {
+    if (this.entityPaths.has(path)) {
+      throw new OError({ message: 'entity already exists', info: { path } })
+    }
+    this.entityPaths.add(path)
+  }
+
+  createFolder(name) {
+    return {
+      _id: ObjectId(),
+      name,
+      folders: [],
+      docs: [],
+      fileRefs: []
+    }
+  }
+}

services/web/app/src/Features/Project/ProjectDuplicator.js

@@ -35,7 +35,7 @@ module.exports = ProjectDuplicator = {
     callback
   ) {
     const setRootDoc = _.once(doc_id =>
-      ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id)
+      ProjectEntityUpdateHandler.setRootDoc(newProject._id, doc_id, () => {})
     )
     const docs = originalFolder.docs || []
     const jobs = docs.map(

services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js

@@ -10,6 +10,7 @@ const logger = require('logger-sharelatex')
 const path = require('path')
 const { ObjectId } = require('mongodb')
 const Settings = require('settings-sharelatex')
+const OError = require('@overleaf/o-error')
 const CooldownManager = require('../Cooldown/CooldownManager')
 const Errors = require('../Errors/Errors')
 const { Folder } = require('../../models/Folder')
@@ -18,6 +19,7 @@ const { Project } = require('../../models/Project')
 const ProjectEntityHandler = require('./ProjectEntityHandler')
 const ProjectGetter = require('./ProjectGetter')
 const ProjectLocator = require('./ProjectLocator')
+const FolderStructureBuilder = require('./FolderStructureBuilder')
 const SafePath = require('./SafePath')
 
 const LOCK_NAMESPACE = 'mongoTransaction'
@@ -69,6 +71,7 @@ module.exports = {
     'rev',
     'changes'
   ]),
+  createNewFolderStructure: callbackify(wrapWithLock(createNewFolderStructure)),
   _insertDeletedDocReference: callbackify(_insertDeletedDocReference),
   _insertDeletedFileReference: callbackify(_insertDeletedFileReference),
   _putElement: callbackifyMultiResult(_putElement, ['result', 'project']),
@@ -82,6 +85,7 @@ module.exports = {
     moveEntity: wrapWithLock(moveEntity),
     deleteEntity: wrapWithLock(deleteEntity),
     renameEntity: wrapWithLock(renameEntity),
+    createNewFolderStructure: wrapWithLock(createNewFolderStructure),
     _insertDeletedDocReference,
     _insertDeletedFileReference,
     _putElement
@@ -606,3 +610,35 @@ async function _checkValidMove(
     }
   }
 }
+
+async function createNewFolderStructure(projectId, docUploads, fileUploads) {
+  try {
+    const rootFolder = FolderStructureBuilder.buildFolderStructure(
+      docUploads,
+      fileUploads
+    )
+    const result = await Project.updateOne(
+      {
+        _id: projectId,
+        'rootFolder.0.folders.0': { $exists: false },
+        'rootFolder.0.docs.0': { $exists: false },
+        'rootFolder.0.files.0': { $exists: false }
+      },
+      {
+        $set: { rootFolder: [rootFolder] },
+        $inc: { version: 1 }
+      }
+    ).exec()
+    if (result.n !== 1) {
+      throw new OError({
+        message: 'project not found or folder structure already exists',
+        info: { projectId }
+      })
+    }
+  } catch (err) {
+    throw new OError({
+      message: 'failed to create folder structure',
+      info: { projectId }
+    }).withCause(err)
+  }
+}

services/web/test/unit/src/Project/FolderStructureBuilderTests.js

@@ -0,0 +1,115 @@
+const { expect } = require('chai')
+const sinon = require('sinon')
+const SandboxedModule = require('sandboxed-module')
+
+const MODULE_PATH =
+  '../../../../app/src/Features/Project/FolderStructureBuilder'
+const MOCK_OBJECT_ID = 'MOCK_OBJECT_ID'
+
+describe('FolderStructureBuilder', function() {
+  beforeEach(function() {
+    this.ObjectId = sinon.stub().returns(MOCK_OBJECT_ID)
+    this.FolderStructureBuilder = SandboxedModule.require(MODULE_PATH, {
+      requires: {
+        mongodb: { ObjectId: this.ObjectId }
+      }
+    })
+  })
+
+  describe('buildFolderStructure', function() {
+    describe('when given no documents at all', function() {
+      beforeEach(function() {
+        this.result = this.FolderStructureBuilder.buildFolderStructure([], [])
+      })
+
+      it('returns an empty root folder', function() {
+        expect(this.result).to.deep.equal({
+          _id: MOCK_OBJECT_ID,
+          name: 'rootFolder',
+          folders: [],
+          docs: [],
+          fileRefs: []
+        })
+      })
+    })
+
+    describe('when given documents and files', function() {
+      beforeEach(function() {
+        const docUploads = [
+          { dirname: '/', doc: { _id: 'doc-1', name: 'main.tex' } },
+          { dirname: '/foo', doc: { _id: 'doc-2', name: 'other.tex' } },
+          { dirname: '/foo', doc: { _id: 'doc-3', name: 'other.bib' } },
+          {
+            dirname: '/foo/foo1/foo2',
+            doc: { _id: 'doc-4', name: 'another.tex' }
+          }
+        ]
+        const fileUploads = [
+          { dirname: '/', fileRef: { _id: 'file-1', name: 'aaa.jpg' } },
+          { dirname: '/foo', fileRef: { _id: 'file-2', name: 'bbb.jpg' } },
+          { dirname: '/bar', fileRef: { _id: 'file-3', name: 'ccc.jpg' } }
+        ]
+        this.result = this.FolderStructureBuilder.buildFolderStructure(
+          docUploads,
+          fileUploads
+        )
+      })
+
+      it('returns a full folder structure', function() {
+        expect(this.result).to.deep.equal({
+          _id: MOCK_OBJECT_ID,
+          name: 'rootFolder',
+          docs: [{ _id: 'doc-1', name: 'main.tex' }],
+          fileRefs: [{ _id: 'file-1', name: 'aaa.jpg' }],
+          folders: [
+            {
+              _id: MOCK_OBJECT_ID,
+              name: 'foo',
+              docs: [
+                { _id: 'doc-2', name: 'other.tex' },
+                { _id: 'doc-3', name: 'other.bib' }
+              ],
+              fileRefs: [{ _id: 'file-2', name: 'bbb.jpg' }],
+              folders: [
+                {
+                  _id: MOCK_OBJECT_ID,
+                  name: 'foo1',
+                  docs: [],
+                  fileRefs: [],
+                  folders: [
+                    {
+                      _id: MOCK_OBJECT_ID,
+                      name: 'foo2',
+                      docs: [{ _id: 'doc-4', name: 'another.tex' }],
+                      fileRefs: [],
+                      folders: []
+                    }
+                  ]
+                }
+              ]
+            },
+            {
+              _id: MOCK_OBJECT_ID,
+              name: 'bar',
+              docs: [],
+              fileRefs: [{ _id: 'file-3', name: 'ccc.jpg' }],
+              folders: []
+            }
+          ]
+        })
+      })
+    })
+
+    describe('when given duplicate files', function() {
+      it('throws an error', function() {
+        const docUploads = [
+          { dirname: '/foo', doc: { _id: 'doc-1', name: 'doc.tex' } },
+          { dirname: '/foo', doc: { _id: 'doc-2', name: 'doc.tex' } }
+        ]
+        expect(() =>
+          this.FolderStructureBuilder.buildFolderStructure(docUploads, [])
+        ).to.throw()
+      })
+    })
+  })
+})

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js

@@ -177,6 +177,10 @@ describe('ProjectEntityMongoUpdateHandler', function() {
       }
     }
 
+    this.FolderStructureBuilder = {
+      buildFolderStructure: sinon.stub()
+    }
+
     this.subject = SandboxedModule.require(MODULE_PATH, {
       globals: {
         console: console
@@ -191,6 +195,7 @@ describe('ProjectEntityMongoUpdateHandler', function() {
         './ProjectEntityHandler': this.ProjectEntityHandler,
         './ProjectLocator': this.ProjectLocator,
         './ProjectGetter': this.ProjectGetter,
+        './FolderStructureBuilder': this.FolderStructureBuilder,
         // We need to provide Errors here to make instance check work
         '../Errors/Errors': Errors
       }
@@ -1031,4 +1036,57 @@ describe('ProjectEntityMongoUpdateHandler', function() {
       this.ProjectMock.verify()
     })
   })
+
+  describe('createNewFolderStructure', function() {
+    beforeEach(function() {
+      this.mockRootFolder = 'MOCK_ROOT_FOLDER'
+      this.docUploads = ['MOCK_DOC_UPLOAD']
+      this.fileUploads = ['MOCK_FILE_UPLOAD']
+      this.FolderStructureBuilder.buildFolderStructure
+        .withArgs(this.docUploads, this.fileUploads)
+        .returns(this.mockRootFolder)
+      this.updateExpectation = this.ProjectMock.expects('updateOne')
+        .withArgs(
+          {
+            _id: this.project._id,
+            'rootFolder.0.folders.0': { $exists: false },
+            'rootFolder.0.docs.0': { $exists: false },
+            'rootFolder.0.files.0': { $exists: false }
+          },
+          { $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } }
+        )
+        .chain('exec')
+    })
+
+    describe('happy path', function() {
+      beforeEach(async function() {
+        this.updateExpectation.resolves({ n: 1 })
+        await this.subject.promises.createNewFolderStructure(
+          this.project._id,
+          this.docUploads,
+          this.fileUploads
+        )
+      })
+
+      it('updates the database', function() {
+        this.ProjectMock.verify()
+      })
+    })
+
+    describe("when the update doesn't find a matching document", function() {
+      beforeEach(async function() {
+        this.updateExpectation.resolves({ n: 0 })
+      })
+
+      it('throws an error', async function() {
+        await expect(
+          this.subject.promises.createNewFolderStructure(
+            this.project._id,
+            this.docUploads,
+            this.fileUploads
+          )
+        ).to.be.rejected
+      })
+    })
+  })
 })

services/web/test/unit/src/Project/ProjectEntityUpdateHandlerTests.js

@@ -453,8 +453,8 @@ describe('ProjectEntityUpdateHandler', function() {
         return this.logger.warn
           .calledWith(
             {
-              project_id,
-              doc_id,
+              projectId: project_id,
+              docId: doc_id,
               lines: this.docLines
             },
             'doc not found while updating doc lines'
@@ -504,7 +504,11 @@ describe('ProjectEntityUpdateHandler', function() {
         .stub()
         .yields(null, `/main.tex`)
 
-      this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
+      this.ProjectEntityUpdateHandler.setRootDoc(
+        project_id,
+        this.rootDoc_id,
+        () => {}
+      )
       return this.ProjectModel.update
         .calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
         .should.equal(true)
@@ -516,7 +520,11 @@ describe('ProjectEntityUpdateHandler', function() {
         .stub()
         .yields(Errors.NotFoundError)
 
-      this.ProjectEntityUpdateHandler.setRootDoc(project_id, this.rootDoc_id)
+      this.ProjectEntityUpdateHandler.setRootDoc(
+        project_id,
+        this.rootDoc_id,
+        () => {}
+      )
       return this.ProjectModel.update
         .calledWith({ _id: project_id }, { rootDoc_id: this.rootDoc_id })
         .should.equal(false)