overleaf/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js

const _ = require('underscore')
const async = require('async')
const logger = require('logger-sharelatex')
const path = require('path')
const settings = require('settings-sharelatex')
const CooldownManager = require('../Cooldown/CooldownManager')
const Errors = require('../Errors/Errors')
const { Folder } = require('../../models/Folder')
const LockManager = require('../../infrastructure/LockManager')
const { Project } = require('../../models/Project')
const ProjectEntityHandler = require('./ProjectEntityHandler')
const ProjectGetter = require('./ProjectGetter')
const ProjectLocator = require('./ProjectLocator')
const SafePath = require('./SafePath')

const LOCK_NAMESPACE = 'mongoTransaction'

function wrapWithLock(methodWithoutLock) {
  // This lock is used whenever we read or write to an existing project's
  // structure. Some operations to project structure cannot be done atomically
  // in mongo, this lock is used to prevent reading the structure between two
  // parts of a staged update.
  function methodWithLock(projectId, ...rest) {
    const adjustedLength = Math.max(rest.length, 1)
    const args = rest.slice(0, adjustedLength - 1)
    const callback = rest[adjustedLength - 1]
    LockManager.runWithLock(
      LOCK_NAMESPACE,
      projectId,
      cb => methodWithoutLock(projectId, ...args, cb),
      callback
    )
  }
  methodWithLock.withoutLock = methodWithoutLock
  return methodWithLock
}

const ProjectEntityMongoUpdateHandler = {
  LOCK_NAMESPACE,

  addDoc: wrapWithLock(function(projectId, folderId, doc, callback) {
    ProjectGetter.getProjectWithoutLock(
      projectId,
      { rootFolder: true, name: true, overleaf: true },
      (err, project) => {
        if (err != null) {
          logger.warn({ projectId, err }, 'error getting project for add doc')
          return callback(err)
        }
        logger.log(
          { projectId, folderId, doc_name: doc.name },
          'adding doc to project with project'
        )
        ProjectEntityMongoUpdateHandler._confirmFolder(
          project,
          folderId,
          folderId => {
            ProjectEntityMongoUpdateHandler._putElement(
              project,
              folderId,
              doc,
              'doc',
              callback
            )
          }
        )
      }
    )
  }),

  addFile: wrapWithLock(function(projectId, folderId, fileRef, callback) {
    ProjectGetter.getProjectWithoutLock(
      projectId,
      { rootFolder: true, name: true, overleaf: true },
      (err, project) => {
        if (err != null) {
          logger.warn({ projectId, err }, 'error getting project for add file')
          return callback(err)
        }
        logger.log(
          { projectId: project._id, folderId, file_name: fileRef.name },
          'adding file'
        )
        ProjectEntityMongoUpdateHandler._confirmFolder(
          project,
          folderId,
          folderId =>
            ProjectEntityMongoUpdateHandler._putElement(
              project,
              folderId,
              fileRef,
              'file',
              callback
            )
        )
      }
    )
  }),

  replaceFileWithNew: wrapWithLock((projectId, fileId, newFileRef, callback) =>
    ProjectGetter.getProjectWithoutLock(
      projectId,
      { rootFolder: true, name: true, overleaf: true },
      (err, project) => {
        if (err != null) {
          return callback(err)
        }
        ProjectLocator.findElement(
          { project, element_id: fileId, type: 'file' },
          (err, fileRef, path) => {
            if (err != null) {
              return callback(err)
            }
            ProjectEntityMongoUpdateHandler._insertDeletedFileReference(
              projectId,
              fileRef,
              err => {
                if (err != null) {
                  return callback(err)
                }
                const conditions = { _id: project._id }
                const inc = {}
                // increment the project structure version as we are adding a new file here
                inc['version'] = 1
                const set = {}
                set[`${path.mongo}._id`] = newFileRef._id
                set[`${path.mongo}.created`] = new Date()
                set[`${path.mongo}.linkedFileData`] = newFileRef.linkedFileData
                inc[`${path.mongo}.rev`] = 1
                set[`${path.mongo}.hash`] = newFileRef.hash
                const update = {
                  $inc: inc,
                  $set: set
                }
                // Note: Mongoose uses new:true to return the modified document
                // https://mongoosejs.com/docs/api.html#model_Model.findOneAndUpdate
                // but Mongo uses returnNewDocument:true instead
                // https://docs.mongodb.com/manual/reference/method/db.collection.findOneAndUpdate/
                // We are using Mongoose here, but if we ever switch to a direct mongo call
                // the next line will need to be updated.
                Project.findOneAndUpdate(
                  conditions,
                  update,
                  { new: true },
                  (err, newProject) => {
                    if (err != null) {
                      return callback(err)
                    }
                    callback(null, fileRef, project, path, newProject)
                  }
                )
              }
            )
          }
        )
      }
    )
  ),

  mkdirp: wrapWithLock(function(projectId, path, options, callback) {
    // defaults to case insensitive paths, use options {exactCaseMatch:true}
    // to make matching case-sensitive
    let folders = path.split('/')
    folders = _.select(folders, folder => folder.length !== 0)

    ProjectGetter.getProjectWithOnlyFolders(projectId, (err, project) => {
      if (err != null) {
        return callback(err)
      }
      if (path === '/') {
        logger.log(
          { projectId: project._id },
          'mkdir is only trying to make path of / so sending back root folder'
        )
        return callback(null, [], project.rootFolder[0])
      }
      logger.log({ projectId: project._id, path, folders }, 'running mkdirp')

      let builtUpPath = ''
      const procesFolder = (previousFolders, folderName, callback) => {
        let parentFolderId
        previousFolders = previousFolders || []
        const parentFolder = previousFolders[previousFolders.length - 1]
        if (parentFolder != null) {
          parentFolderId = parentFolder._id
        }
        builtUpPath = `${builtUpPath}/${folderName}`
        ProjectLocator.findElementByPath(
          {
            project,
            path: builtUpPath,
            exactCaseMatch: options != null ? options.exactCaseMatch : undefined
          },
          (err, foundFolder) => {
            if (err != null) {
              logger.log(
                { path, projectId: project._id, folderName },
                'making folder from mkdirp'
              )
              ProjectEntityMongoUpdateHandler.addFolder.withoutLock(
                projectId,
                parentFolderId,
                folderName,
                (err, newFolder, parentFolderId) => {
                  if (err != null) {
                    return callback(err)
                  }
                  newFolder.parentFolder_id = parentFolderId
                  previousFolders.push(newFolder)
                  callback(null, previousFolders)
                }
              )
            } else {
              foundFolder.filterOut = true
              previousFolders.push(foundFolder)
              callback(null, previousFolders)
            }
          }
        )
      }

      async.reduce(folders, [], procesFolder, (err, folders) => {
        if (err != null) {
          return callback(err)
        }
        const lastFolder = folders[folders.length - 1]
        folders = _.select(folders, folder => !folder.filterOut)
        callback(null, folders, lastFolder)
      })
    })
  }),

  moveEntity: wrapWithLock(function(
    projectId,
    entityId,
    destFolderId,
    entityType,
    callback
  ) {
    ProjectGetter.getProjectWithoutLock(
      projectId,
      { rootFolder: true, name: true, overleaf: true },
      (err, project) => {
        if (err != null) {
          return callback(err)
        }
        ProjectLocator.findElement(
          { project, element_id: entityId, type: entityType },
          (err, entity, entityPath) => {
            if (err != null) {
              return callback(err)
            }
            // Prevent top-level docs/files with reserved names (to match v1 behaviour)
            if (
              ProjectEntityMongoUpdateHandler._blockedFilename(
                entityPath,
                entityType
              )
            ) {
              return callback(
                new Errors.InvalidNameError('blocked element name')
              )
            }
            ProjectEntityMongoUpdateHandler._checkValidMove(
              project,
              entityType,
              entity,
              entityPath,
              destFolderId,
              error => {
                if (error != null) {
                  return callback(error)
                }
                ProjectEntityHandler.getAllEntitiesFromProject(
                  project,
                  (error, oldDocs, oldFiles) => {
                    if (error != null) {
                      return callback(error)
                    }
                    // For safety, insert the entity in the destination
                    // location first, and then remove the original.  If
                    // there is an error the entity may appear twice. This
                    // will cause some breakage but is better than being
                    // lost, which is what happens if this is done in the
                    // opposite order.
                    ProjectEntityMongoUpdateHandler._putElement(
                      project,
                      destFolderId,
                      entity,
                      entityType,
                      (err, result) => {
                        if (err != null) {
                          return callback(err)
                        }
                        // Note: putElement always pushes onto the end of an
                        // array so it will never change an existing mongo
                        // path. Therefore it is safe to remove an element
                        // from the project with an existing path after
                        // calling putElement. But we must be sure that we
                        // have not moved a folder subfolder of itself (which
                        // is done by _checkValidMove above) because that
                        // would lead to it being deleted.
                        ProjectEntityMongoUpdateHandler._removeElementFromMongoArray(
                          Project,
                          projectId,
                          entityPath.mongo,
                          entityId,
                          (err, newProject) => {
                            if (err != null) {
                              return callback(err)
                            }
                            ProjectEntityHandler.getAllEntitiesFromProject(
                              newProject,
                              (err, newDocs, newFiles) => {
                                if (err != null) {
                                  return callback(err)
                                }
                                const startPath = entityPath.fileSystem
                                const endPath = result.path.fileSystem
                                const changes = {
                                  oldDocs,
                                  newDocs,
                                  oldFiles,
                                  newFiles,
                                  newProject
                                }
                                // check that no files have been lost (or duplicated)
                                if (
                                  oldFiles.length !== newFiles.length ||
                                  oldDocs.length !== newDocs.length
                                ) {
                                  logger.warn(
                                    {
                                      projectId,
                                      oldDocs: oldDocs.length,
                                      newDocs: newDocs.length,
                                      oldFiles: oldFiles.length,
                                      newFiles: newFiles.length,
                                      origProject: project,
                                      newProject
                                    },
                                    "project corrupted moving files - shouldn't happen"
                                  )
                                  return callback(
                                    new Error(
                                      'unexpected change in project structure'
                                    )
                                  )
                                }
                                callback(
                                  null,
                                  project,
                                  startPath,
                                  endPath,
                                  entity.rev,
                                  changes,
                                  callback
                                )
                              }
                            )
                          }
                        )
                      }
                    )
                  }
                )
              }
            )
          }
        )
      }
    )
  }),

  deleteEntity: wrapWithLock((projectId, entityId, entityType, callback) =>
    ProjectGetter.getProjectWithoutLock(
      projectId,
      { name: true, rootFolder: true, overleaf: true },
      (error, project) => {
        if (error != null) {
          return callback(error)
        }
        ProjectLocator.findElement(
          { project, element_id: entityId, type: entityType },
          (error, entity, path) => {
            if (error != null) {
              return callback(error)
            }
            ProjectEntityMongoUpdateHandler._removeElementFromMongoArray(
              Project,
              projectId,
              path.mongo,
              entityId,
              (error, newProject) => {
                if (error != null) {
                  return callback(error)
                }
                callback(null, entity, path, project, newProject)
              }
            )
          }
        )
      }
    )
  ),

  renameEntity: wrapWithLock(
    (projectId, entityId, entityType, newName, callback) =>
      ProjectGetter.getProjectWithoutLock(
        projectId,
        { rootFolder: true, name: true, overleaf: true },
        (error, project) => {
          if (error != null) {
            return callback(error)
          }
          ProjectEntityHandler.getAllEntitiesFromProject(
            project,
            (error, oldDocs, oldFiles) => {
              if (error != null) {
                return callback(error)
              }
              ProjectLocator.findElement(
                { project, element_id: entityId, type: entityType },
                (error, entity, entPath, parentFolder) => {
                  if (error != null) {
                    return callback(error)
                  }
                  const endPath = path.join(
                    path.dirname(entPath.fileSystem),
                    newName
                  )
                  // Prevent top-level docs/files with reserved names (to match v1 behaviour)
                  if (
                    ProjectEntityMongoUpdateHandler._blockedFilename(
                      { fileSystem: endPath },
                      entityType
                    )
                  ) {
                    return callback(
                      new Errors.InvalidNameError('blocked element name')
                    )
                  }
                  // check if the new name already exists in the current folder
                  ProjectEntityMongoUpdateHandler._checkValidElementName(
                    parentFolder,
                    newName,
                    error => {
                      if (error != null) {
                        return callback(error)
                      }
                      const conditions = { _id: projectId }
                      const update = { $set: {}, $inc: {} }
                      const namePath = entPath.mongo + '.name'
                      update['$set'][namePath] = newName
                      // we need to increment the project version number for any structure change
                      update['$inc']['version'] = 1
                      Project.findOneAndUpdate(
                        conditions,
                        update,
                        { new: true },
                        (error, newProject) => {
                          if (error != null) {
                            return callback(error)
                          }
                          ProjectEntityHandler.getAllEntitiesFromProject(
                            newProject,
                            (error, newDocs, newFiles) => {
                              if (error != null) {
                                return callback(error)
                              }
                              const startPath = entPath.fileSystem
                              const changes = {
                                oldDocs,
                                newDocs,
                                oldFiles,
                                newFiles,
                                newProject
                              }
                              callback(
                                null,
                                project,
                                startPath,
                                endPath,
                                entity.rev,
                                changes,
                                callback
                              )
                            }
                          )
                        }
                      )
                    }
                  )
                }
              )
            }
          )
        }
      )
  ),

  addFolder: wrapWithLock((projectId, parentFolderId, folderName, callback) =>
    ProjectGetter.getProjectWithoutLock(
      projectId,
      { rootFolder: true, name: true, overleaf: true },
      (err, project) => {
        if (err != null) {
          logger.warn(
            { projectId, err },
            'error getting project for add folder'
          )
          return callback(err)
        }
        ProjectEntityMongoUpdateHandler._confirmFolder(
          project,
          parentFolderId,
          parentFolderId => {
            const folder = new Folder({ name: folderName })
            logger.log(
              { project: project._id, parentFolderId, folderName },
              'adding new folder'
            )
            ProjectEntityMongoUpdateHandler._putElement(
              project,
              parentFolderId,
              folder,
              'folder',
              err => {
                if (err != null) {
                  logger.warn(
                    { err, projectId: project._id },
                    'error adding folder to project'
                  )
                  return callback(err)
                }
                callback(null, folder, parentFolderId)
              }
            )
          }
        )
      }
    )
  ),

  _removeElementFromMongoArray(model, modelId, path, elementId, callback) {
    const conditions = { _id: modelId }
    const pullUpdate = { $pull: {}, $inc: {} }
    const nonArrayPath = path.slice(0, path.lastIndexOf('.'))
    // remove specific element from array by id
    pullUpdate['$pull'][nonArrayPath] = { _id: elementId }
    // we need to increment the project version number for any structure change
    pullUpdate['$inc']['version'] = 1
    model.findOneAndUpdate(conditions, pullUpdate, { new: true }, callback)
  },

  _countElements(project) {
    function countFolder(folder) {
      let total = 0

      for (let subfolder of (folder != null ? folder.folders : undefined) ||
        []) {
        total += countFolder(subfolder)
      }

      if (
        folder != null &&
        folder.folders != null &&
        folder.folders.length > 0
      ) {
        total += folder.folders.length
      }

      if (folder != null && folder.docs != null && folder.docs.length > 0) {
        total += folder.docs.length
      }

      if (
        folder != null &&
        folder.fileRefs != null &&
        folder.fileRefs.length > 0
      ) {
        total += folder.fileRefs.length
      }

      return total
    }

    return countFolder(project.rootFolder[0])
  },

  _putElement(project, folderId, element, type, callback) {
    function sanitizeTypeOfElement(elementType) {
      const lastChar = elementType.slice(-1)
      if (lastChar !== 's') {
        elementType += 's'
      }
      if (elementType === 'files') {
        elementType = 'fileRefs'
      }
      return elementType
    }

    if (element == null || element._id == null) {
      logger.warn(
        { projectId: project._id, folderId, element, type },
        'failed trying to insert element as it was null'
      )
      return callback(new Error('no element passed to be inserted'))
    }
    type = sanitizeTypeOfElement(type)

    // original check path.resolve("/", element.name) isnt "/#{element.name}" or element.name.match("/")
    // check if name is allowed
    if (!SafePath.isCleanFilename(element.name)) {
      logger.warn(
        { projectId: project._id, folderId, element, type },
        'failed trying to insert element as name was invalid'
      )
      return callback(new Errors.InvalidNameError('invalid element name'))
    }

    if (folderId == null) {
      folderId = project.rootFolder[0]._id
    }

    if (
      ProjectEntityMongoUpdateHandler._countElements(project) >
      settings.maxEntitiesPerProject
    ) {
      logger.warn(
        { projectId: project._id },
        'project too big, stopping insertions'
      )
      CooldownManager.putProjectOnCooldown(project._id)
      return callback(new Error('project_has_to_many_files'))
    }

    ProjectLocator.findElement(
      { project, element_id: folderId, type: 'folders' },
      (err, folder, path) => {
        if (err != null) {
          logger.warn(
            { err, projectId: project._id, folderId, type, element },
            'error finding folder for _putElement'
          )
          return callback(err)
        }
        const newPath = {
          fileSystem: `${path.fileSystem}/${element.name}`,
          mongo: path.mongo
        }
        // check if the path would be too long
        if (!SafePath.isAllowedLength(newPath.fileSystem)) {
          return callback(new Errors.InvalidNameError('path too long'))
        }
        // Prevent top-level docs/files with reserved names (to match v1 behaviour)
        if (ProjectEntityMongoUpdateHandler._blockedFilename(newPath, type)) {
          return callback(new Errors.InvalidNameError('blocked element name'))
        }
        ProjectEntityMongoUpdateHandler._checkValidElementName(
          folder,
          element.name,
          err => {
            if (err != null) {
              return callback(err)
            }
            const id = element._id + ''
            element._id = require('mongoose').Types.ObjectId(id)
            const conditions = { _id: project._id }
            const mongopath = `${path.mongo}.${type}`
            const update = { $push: {}, $inc: {} }
            update['$push'][mongopath] = element
            // we need to increment the project version number for any structure change
            update['$inc']['version'] = 1 // increment project version number
            logger.log(
              {
                projectId: project._id,
                element_id: element._id,
                fileType: type,
                folderId,
                mongopath
              },
              'adding element to project'
            )
            // We are using Mongoose here, but if we ever switch to a direct mongo call
            // the next line will need to be updated to {returnNewDocument:true}
            Project.findOneAndUpdate(
              conditions,
              update,
              { new: true },
              (err, newProject) => {
                if (err != null) {
                  logger.warn(
                    { err, projectId: project._id },
                    'error saving in putElement project'
                  )
                  return callback(err)
                }
                callback(err, { path: newPath }, newProject)
              }
            )
          }
        )
      }
    )
  },

  _blockedFilename(entityPath, entityType) {
    // check if name would be blocked in v1
    // javascript reserved names are forbidden for docs and files
    // at the top-level (but folders with reserved names are allowed).
    const isFolder = ['folder', 'folders'].includes(entityType)
    const [dir, file] = [
      path.dirname(entityPath.fileSystem),
      path.basename(entityPath.fileSystem)
    ]
    const isTopLevel = dir === '/'
    if (isTopLevel && !isFolder && SafePath.isBlockedFilename(file)) {
      return true
    } else {
      return false
    }
  },

  _checkValidElementName(folder, name, callback) {
    // check if the name is already taken by a doc, file or
    // folder. If so, return an error "file already exists".
    const err = new Errors.InvalidNameError('file already exists')
    for (let doc of (folder != null ? folder.docs : undefined) || []) {
      if (doc.name === name) {
        return callback(err)
      }
    }
    for (let file of (folder != null ? folder.fileRefs : undefined) || []) {
      if (file.name === name) {
        return callback(err)
      }
    }
    for (folder of (folder != null ? folder.folders : undefined) || []) {
      if (folder.name === name) {
        return callback(err)
      }
    }
    callback()
  },

  _confirmFolder(project, folderId, callback) {
    logger.log(
      { folderId, projectId: project._id },
      'confirming folder in project'
    )
    if (folderId + '' === 'undefined') {
      callback(project.rootFolder[0]._id)
    } else if (folderId !== null) {
      callback(folderId)
    } else {
      callback(project.rootFolder[0]._id)
    }
  },

  _checkValidMove(
    project,
    entityType,
    entity,
    entityPath,
    destFolderId,
    callback
  ) {
    ProjectLocator.findElement(
      { project, element_id: destFolderId, type: 'folder' },
      (err, destEntity, destFolderPath) => {
        if (err != null) {
          return callback(err)
        }
        // check if there is already a doc/file/folder with the same name
        // in the destination folder
        ProjectEntityMongoUpdateHandler._checkValidElementName(
          destEntity,
          entity.name,
          err => {
            if (err != null) {
              return callback(err)
            }
            if (/folder/.test(entityType)) {
              logger.log(
                {
                  destFolderPath: destFolderPath.fileSystem,
                  folderPath: entityPath.fileSystem
                },
                'checking folder is not moving into child folder'
              )
              const isNestedFolder =
                destFolderPath.fileSystem.slice(
                  0,
                  entityPath.fileSystem.length
                ) === entityPath.fileSystem
              if (isNestedFolder) {
                return callback(
                  new Errors.InvalidNameError(
                    'destination folder is a child folder of me'
                  )
                )
              }
            }
            callback()
          }
        )
      }
    )
  },

  _insertDeletedDocReference(projectId, doc, callback) {
    Project.update(
      {
        _id: projectId
      },
      {
        $push: {
          deletedDocs: {
            _id: doc._id,
            name: doc.name,
            deletedAt: new Date()
          }
        }
      },
      {},
      callback
    )
  },

  _insertDeletedFileReference(projectId, fileRef, callback) {
    Project.update(
      {
        _id: projectId
      },
      {
        $push: {
          deletedFiles: {
            _id: fileRef._id,
            name: fileRef.name,
            linkedFileData: fileRef.linkedFileData,
            hash: fileRef.hash,
            deletedAt: new Date()
          }
        }
      },
      {},
      callback
    )
  }
}

module.exports = ProjectEntityMongoUpdateHandler