Merge pull request #2639 from overleaf/em-convert-doc-to-file

Endpoint for converting a doc to a file

GitOrigin-RevId: 0a3bd46a7a78537b0f64dc577402277cbe81fecb

services/web/app/src/Features/Editor/EditorHttpController.js

@@ -1,3 +1,4 @@
+const HttpErrors = require('@overleaf/o-error/http')
 const ProjectDeleter = require('../Project/ProjectDeleter')
 const EditorController = require('./EditorController')
 const ProjectGetter = require('../Project/ProjectGetter')
@@ -11,6 +12,7 @@ const PrivilegeLevels = require('../Authorization/PrivilegeLevels')
 const TokenAccessHandler = require('../TokenAccess/TokenAccessHandler')
 const AuthenticationController = require('../Authentication/AuthenticationController')
 const Errors = require('../Errors/Errors')
+const ProjectEntityUpdateHandler = require('../Project/ProjectEntityUpdateHandler')
 const { expressify } = require('../../util/promises')
 
 module.exports = {
@@ -23,6 +25,7 @@ module.exports = {
   deleteFile: expressify(deleteFile),
   deleteFolder: expressify(deleteFolder),
   deleteEntity: expressify(deleteEntity),
+  convertDocToFile: expressify(convertDocToFile),
   _nameIsAcceptableLength
 }
 
@@ -221,3 +224,25 @@ async function deleteEntity(req, res, next) {
   )
   res.sendStatus(204)
 }
+
+async function convertDocToFile(req, res, next) {
+  const projectId = req.params.Project_id
+  const docId = req.params.entity_id
+  const { userId } = req.body
+  try {
+    const fileRef = await ProjectEntityUpdateHandler.promises.convertDocToFile(
+      projectId,
+      docId,
+      userId
+    )
+    res.json({ fileId: fileRef._id.toString() })
+  } catch (err) {
+    if (err instanceof Errors.NotFoundError) {
+      throw new HttpErrors.NotFoundError({
+        info: { public: { message: 'Document not found' } }
+      })
+    } else {
+      throw err
+    }
+  }
+}

services/web/app/src/Features/Editor/EditorRouter.js

@@ -2,6 +2,7 @@ const EditorHttpController = require('./EditorHttpController')
 const AuthenticationController = require('../Authentication/AuthenticationController')
 const AuthorizationMiddleware = require('../Authorization/AuthorizationMiddleware')
 const RateLimiterMiddleware = require('../Security/RateLimiterMiddleware')
+const { Joi, validate } = require('../../infrastructure/Validation')
 
 module.exports = {
   apply(webRouter, apiRouter) {
@@ -54,6 +55,16 @@ module.exports = {
       AuthorizationMiddleware.ensureUserCanWriteProjectContent,
       EditorHttpController.deleteFolder
     )
+    apiRouter.post(
+      '/project/:Project_id/doc/:entity_id/convert-to-file',
+      AuthenticationController.httpAuth,
+      validate({
+        body: Joi.object({
+          userId: Joi.objectId().required()
+        })
+      }),
+      EditorHttpController.convertDocToFile
+    )
 
     // Called by the real-time API to load up the current project state.
     // This is a post request because it's more than just a getting of data. We take actions

services/web/app/src/Features/Project/ProjectEntityHandler.js

@@ -224,6 +224,7 @@ module.exports = ProjectEntityHandler
 module.exports.promises = promisifyAll(ProjectEntityHandler, {
   multiResult: {
     getAllEntities: ['docs', 'files'],
-    getAllEntitiesFromProject: ['docs', 'files']
+    getAllEntitiesFromProject: ['docs', 'files'],
+    getDoc: ['lines', 'rev', 'version', 'ranges']
   }
 })

services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js

@@ -47,6 +47,7 @@ module.exports = {
     'path',
     'newProject'
   ]),
+  replaceDocWithFile: callbackify(replaceDocWithFile),
   mkdirp: callbackifyMultiResult(wrapWithLock(mkdirp), [
     'newFolders',
     'folder'
@@ -81,6 +82,7 @@ module.exports = {
     addFile: wrapWithLock(addFile),
     addFolder: wrapWithLock(addFolder),
     replaceFileWithNew: wrapWithLock(replaceFileWithNew),
+    replaceDocWithFile: wrapWithLock(replaceDocWithFile),
     mkdirp: wrapWithLock(mkdirp),
     moveEntity: wrapWithLock(moveEntity),
     deleteEntity: wrapWithLock(deleteEntity),
@@ -186,6 +188,33 @@ async function replaceFileWithNew(projectId, fileId, newFileRef) {
   return { oldFileRef: fileRef, project, path, newProject }
 }
 
+async function replaceDocWithFile(projectId, docId, fileRef) {
+  const project = await ProjectGetter.promises.getProjectWithoutLock(
+    projectId,
+    { rootFolder: true, name: true, overleaf: true }
+  )
+  const { path } = await ProjectLocator.promises.findElement({
+    project,
+    element_id: docId,
+    type: 'doc'
+  })
+  const folderMongoPath = _getParentMongoPath(path.mongo)
+  const newProject = await Project.findOneAndUpdate(
+    { _id: project._id },
+    {
+      $pull: {
+        [`${folderMongoPath}.docs`]: { _id: docId }
+      },
+      $push: {
+        [`${folderMongoPath}.fileRefs`]: fileRef
+      },
+      $inc: { version: 1 }
+    },
+    { new: true }
+  ).exec()
+  return newProject
+}
+
 async function mkdirp(projectId, path, options = {}) {
   // defaults to case insensitive paths, use options {exactCaseMatch:true}
   // to make matching case-sensitive
@@ -637,3 +666,14 @@ async function createNewFolderStructure(projectId, docUploads, fileUploads) {
     }).withCause(err)
   }
 }
+
+/**
+ * Given a Mongo path to an entity, return the Mongo path to the parent folder
+ */
+function _getParentMongoPath(mongoPath) {
+  const segments = mongoPath.split('.')
+  if (segments.length <= 2) {
+    throw new Error('Root folder has no parents')
+  }
+  return segments.slice(0, -2).join('.')
+}

services/web/app/src/Features/Project/ProjectEntityUpdateHandler.js

@@ -3,6 +3,7 @@ const async = require('async')
 const logger = require('logger-sharelatex')
 const Settings = require('settings-sharelatex')
 const Path = require('path')
+const fs = require('fs')
 const { Doc } = require('../../models/Doc')
 const DocstoreManager = require('../Docstore/DocstoreManager')
 const DocumentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler')
@@ -18,6 +19,9 @@ const ProjectUpdateHandler = require('./ProjectUpdateHandler')
 const ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandler')
 const SafePath = require('./SafePath')
 const TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender')
+const FileWriter = require('../../infrastructure/FileWriter')
+const EditorRealTimeController = require('../Editor/EditorRealTimeController')
+const { promisifyAll } = require('../../util/promises')
 
 const LOCK_NAMESPACE = 'sequentialProjectStructureUpdateLock'
 const VALID_ROOT_DOC_EXTENSIONS = Settings.validRootDocExtensions
@@ -1512,7 +1516,146 @@ const ProjectEntityUpdateHandler = {
     })
 
     async.series(jobs, callback)
-  }
+  },
+
+  convertDocToFile: wrapWithLock({
+    beforeLock(next) {
+      return function(projectId, docId, userId, callback) {
+        DocumentUpdaterHandler.deleteDoc(projectId, docId, err => {
+          if (err) {
+            return callback(err)
+          }
+          ProjectLocator.findElement(
+            { project_id: projectId, element_id: docId, type: 'doc' },
+            (err, doc, path) => {
+              const docPath = path.fileSystem
+              if (err) {
+                return callback(err)
+              }
+              DocstoreManager.getDoc(projectId, docId, (err, docLines) => {
+                if (err) {
+                  return callback(err)
+                }
+                FileWriter.writeLinesToDisk(
+                  projectId,
+                  docLines,
+                  (err, fsPath) => {
+                    if (err) {
+                      return callback(err)
+                    }
+                    FileStoreHandler.uploadFileFromDisk(
+                      projectId,
+                      { name: doc.name },
+                      fsPath,
+                      (err, fileStoreUrl, fileRef) => {
+                        if (err) {
+                          return callback(err)
+                        }
+                        fs.unlink(fsPath, err => {
+                          if (err) {
+                            logger.warn(
+                              { err, path: fsPath },
+                              'failed to clean up temporary file'
+                            )
+                          }
+                          next(
+                            projectId,
+                            doc,
+                            docPath,
+                            fileRef,
+                            fileStoreUrl,
+                            userId,
+                            callback
+                          )
+                        })
+                      }
+                    )
+                  }
+                )
+              })
+            }
+          )
+        })
+      }
+    },
+    withLock(projectId, doc, path, fileRef, fileStoreUrl, userId, callback) {
+      ProjectEntityMongoUpdateHandler.replaceDocWithFile(
+        projectId,
+        doc._id,
+        fileRef,
+        (err, project) => {
+          if (err) {
+            return callback(err)
+          }
+          const projectHistoryId =
+            project.overleaf &&
+            project.overleaf.history &&
+            project.overleaf.history.id
+          DocumentUpdaterHandler.updateProjectStructure(
+            projectId,
+            projectHistoryId,
+            userId,
+            {
+              oldDocs: [{ doc, path }],
+              newFiles: [{ file: fileRef, path, url: fileStoreUrl }],
+              newProject: project
+            },
+            err => {
+              if (err) {
+                return callback(err)
+              }
+              ProjectLocator.findElement(
+                {
+                  project_id: projectId,
+                  element_id: fileRef._id,
+                  type: 'file'
+                },
+                (err, element, path, folder) => {
+                  if (err) {
+                    return callback(err)
+                  }
+                  EditorRealTimeController.emitToRoom(
+                    projectId,
+                    'removeEntity',
+                    doc._id,
+                    'convertDocToFile'
+                  )
+                  EditorRealTimeController.emitToRoom(
+                    projectId,
+                    'reciveNewFile',
+                    folder._id,
+                    fileRef,
+                    'convertDocToFile',
+                    null
+                  )
+                  callback(null, fileRef)
+                }
+              )
+            }
+          )
+        }
+      )
+    }
+  })
 }
 
 module.exports = ProjectEntityUpdateHandler
+module.exports.promises = promisifyAll(ProjectEntityUpdateHandler, {
+  without: ['isPathValidForRootDoc'],
+  multiResult: {
+    copyFileFromExistingProjectWithProject: ['fileRef', 'folderId'],
+    _addDocAndSendToTpds: ['result', 'project'],
+    addDoc: ['doc', 'folderId'],
+    addDocWithRanges: ['doc', 'folderId'],
+    _uploadFile: ['fileStoreUrl', 'fileRef'],
+    _addFileAndSendToTpds: ['result', 'project'],
+    addFile: ['fileRef', 'folderId'],
+    upsertDoc: ['doc', 'isNew'],
+    upsertFile: ['fileRef', 'isNew', 'oldFileRef'],
+    upsertDocWithPath: ['doc', 'isNew', 'newFolders', 'folder'],
+    upsertFileWithPath: ['fileRef', 'isNew', 'oldFile', 'newFolders', 'folder'],
+    mkdirp: ['newFolders', 'folder'],
+    mkdirpWithExactCase: ['newFolders', 'folder'],
+    addFolder: ['folder', 'parentFolderId']
+  }
+})

services/web/app/src/infrastructure/FileWriter.js

@@ -10,15 +10,15 @@
  * DS207: Consider shorter variations of null checks
  * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
  */
-let FileWriter
 const fs = require('fs')
 const logger = require('logger-sharelatex')
 const uuid = require('uuid')
 const _ = require('underscore')
 const Settings = require('settings-sharelatex')
 const request = require('request')
+const { promisifyAll } = require('../util/promises')
 
-module.exports = FileWriter = {
+const FileWriter = {
   ensureDumpFolderExists(callback) {
     if (callback == null) {
       callback = function(error) {}
@@ -123,3 +123,6 @@ module.exports = FileWriter = {
     })
   }
 }
+
+module.exports = FileWriter
+module.exports.promises = promisifyAll(FileWriter)

services/web/test/unit/src/Editor/EditorHttpControllerTests.js

@@ -29,8 +29,9 @@ describe('EditorHttpController', function() {
       _id: this.projectView._id,
       owner: { _id: this.projectView.owner._id }
     }
-    this.doc = { mock: 'doc' }
+    this.doc = { _id: new ObjectId(), name: 'excellent-original-idea.tex' }
-    this.folder = { mock: 'folder' }
+    this.file = { _id: new ObjectId() }
+    this.folder = { _id: new ObjectId() }
 
     this.parentFolderId = 'mock-folder-id'
     this.req = { i18n: { translate: string => string } }
@@ -42,6 +43,7 @@ describe('EditorHttpController', function() {
     }
     this.next = sinon.stub()
     this.token = null
+    this.docLines = ['hello', 'overleaf']
 
     this.AuthorizationManager = {
       isRestrictedUser: sinon.stub().returns(false),
@@ -82,6 +84,7 @@ describe('EditorHttpController', function() {
     this.EditorController = {
       promises: {
         addDoc: sinon.stub().resolves(this.doc),
+        addFile: sinon.stub().resolves(this.file),
         addFolder: sinon.stub().resolves(this.folder),
         renameEntity: sinon.stub().resolves(),
         moveEntity: sinon.stub().resolves(),
@@ -113,6 +116,11 @@ describe('EditorHttpController', function() {
     this.AuthenticationController = {
       getLoggedInUserId: sinon.stub().returns(this.user._id)
     }
+    this.ProjectEntityUpdateHandler = {
+      promises: {
+        convertDocToFile: sinon.stub().resolves(this.file)
+      }
+    }
     this.EditorHttpController = SandboxedModule.require(MODULE_PATH, {
       globals: {
         console: console
@@ -132,6 +140,9 @@ describe('EditorHttpController', function() {
         '../TokenAccess/TokenAccessHandler': this.TokenAccessHandler,
         '../Authentication/AuthenticationController': this
           .AuthenticationController,
+        '../../infrastructure/FileWriter': this.FileWriter,
+        '../Project/ProjectEntityUpdateHandler': this
+          .ProjectEntityUpdateHandler,
         '../Errors/Errors': Errors
       }
     })
@@ -496,4 +507,32 @@ describe('EditorHttpController', function() {
       expect(this.res.sendStatus).to.have.been.calledWith(204)
     })
   })
+
+  describe('convertDocToFile', function() {
+    beforeEach(function(done) {
+      this.req.params = {
+        Project_id: this.project._id.toString(),
+        entity_id: this.doc._id.toString()
+      }
+      this.req.body = { userId: this.user._id.toString() }
+      this.res.json.callsFake(() => done())
+      this.EditorHttpController.convertDocToFile(this.req, this.res)
+    })
+
+    it('should convert the doc to a file', function() {
+      expect(
+        this.ProjectEntityUpdateHandler.promises.convertDocToFile
+      ).to.have.been.calledWith(
+        this.project._id.toString(),
+        this.doc._id.toString(),
+        this.user._id.toString()
+      )
+    })
+
+    it('should return the file id in the response', function() {
+      expect(this.res.json).to.have.been.calledWith({
+        fileId: this.file._id.toString()
+      })
+    })
+  })
 })

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js

@@ -1089,4 +1089,27 @@ describe('ProjectEntityMongoUpdateHandler', function() {
       })
     })
   })
+
+  describe('replaceDocWithFile', function() {
+    it('should simultaneously remove the doc and add the file', async function() {
+      this.ProjectMock.expects('findOneAndUpdate')
+        .withArgs(
+          { _id: this.project._id },
+          {
+            $pull: { 'rootFolder.0.docs': { _id: this.doc._id } },
+            $push: { 'rootFolder.0.fileRefs': this.file },
+            $inc: { version: 1 }
+          },
+          { new: true }
+        )
+        .chain('exec')
+        .resolves(this.project)
+      await this.subject.promises.replaceDocWithFile(
+        this.project._id,
+        this.doc._id,
+        this.file
+      )
+      this.ProjectMock.verify()
+    })
+  })
 })