diff --git a/services/web/app/src/Features/Project/ProjectEntityHandler.js b/services/web/app/src/Features/Project/ProjectEntityHandler.js index 1ba1012c96..aab074432c 100644 --- a/services/web/app/src/Features/Project/ProjectEntityHandler.js +++ b/services/web/app/src/Features/Project/ProjectEntityHandler.js @@ -363,5 +363,10 @@ const ProjectEntityHandler = { } } -ProjectEntityHandler.promises = promisifyAll(ProjectEntityHandler) module.exports = ProjectEntityHandler +module.exports.promises = promisifyAll(ProjectEntityHandler, { + multiResult: { + getAllEntities: ['docs', 'files'], + getAllEntitiesFromProject: ['docs', 'files'] + } +}) diff --git a/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js b/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js new file mode 100644 index 0000000000..b84cda0e74 --- /dev/null +++ b/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary.js @@ -0,0 +1,608 @@ +/* NOTE: this file is an async/await version of + * ProjectEntityMongoUpdateHandler.js. It's temporarily separate from the + * callback-style version so that we can test it in production for some code + * paths only. + */ +const { callbackify } = require('util') +const { callbackifyMultiResult } = require('../../util/promises') +const _ = require('underscore') +const logger = require('logger-sharelatex') +const path = require('path') +const { ObjectId } = require('mongodb') +const Settings = require('settings-sharelatex') +const CooldownManager = require('../Cooldown/CooldownManager') +const Errors = require('../Errors/Errors') +const { Folder } = require('../../models/Folder') +const LockManager = require('../../infrastructure/LockManager') +const { Project } = require('../../models/Project') +const ProjectEntityHandler = require('./ProjectEntityHandler') +const ProjectGetter = require('./ProjectGetter') +const ProjectLocator = require('./ProjectLocator') +const SafePath = require('./SafePath') + +const LOCK_NAMESPACE = 'mongoTransaction' +const ENTITY_TYPE_TO_MONGO_PATH_SEGMENT = { + doc: 'docs', + docs: 'docs', + file: 'fileRefs', + files: 'fileRefs', + fileRefs: 'fileRefs', + folder: 'folders', + folders: 'folders' +} + +module.exports = { + LOCK_NAMESPACE, + addDoc: callbackifyMultiResult(wrapWithLock(addDoc), ['result', 'project']), + addFile: callbackifyMultiResult(wrapWithLock(addFile), ['result', 'project']), + addFolder: callbackifyMultiResult(wrapWithLock(addFolder), [ + 'folder', + 'parentFolderId' + ]), + replaceFileWithNew: callbackifyMultiResult(wrapWithLock(replaceFileWithNew), [ + 'oldFileRef', + 'project', + 'path', + 'newProject' + ]), + mkdirp: callbackifyMultiResult(wrapWithLock(mkdirp), [ + 'newFolders', + 'folder' + ]), + moveEntity: callbackifyMultiResult(wrapWithLock(moveEntity), [ + 'project', + 'startPath', + 'endPath', + 'rev', + 'changes' + ]), + deleteEntity: callbackifyMultiResult(wrapWithLock(deleteEntity), [ + 'entity', + 'path', + 'projectBeforeDeletion', + 'newProject' + ]), + renameEntity: callbackifyMultiResult(wrapWithLock(renameEntity), [ + 'project', + 'startPath', + 'endPath', + 'rev', + 'changes' + ]), + _insertDeletedDocReference: callbackify(_insertDeletedDocReference), + _insertDeletedFileReference: callbackify(_insertDeletedFileReference), + _putElement: callbackifyMultiResult(_putElement, ['result', 'project']), + _confirmFolder, + promises: { + addDoc: wrapWithLock(addDoc), + addFile: wrapWithLock(addFile), + addFolder: wrapWithLock(addFolder), + replaceFileWithNew: wrapWithLock(replaceFileWithNew), + mkdirp: wrapWithLock(mkdirp), + moveEntity: wrapWithLock(moveEntity), + deleteEntity: wrapWithLock(deleteEntity), + renameEntity: wrapWithLock(renameEntity), + _insertDeletedDocReference, + _insertDeletedFileReference, + _putElement + } +} + +function wrapWithLock(methodWithoutLock) { + // This lock is used whenever we read or write to an existing project's + // structure. Some operations to project structure cannot be done atomically + // in mongo, this lock is used to prevent reading the structure between two + // parts of a staged update. + async function methodWithLock(projectId, ...rest) { + return LockManager.promises.runWithLock(LOCK_NAMESPACE, projectId, () => + methodWithoutLock(projectId, ...rest) + ) + } + return methodWithLock +} + +async function addDoc(projectId, folderId, doc) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { + rootFolder: true, + name: true, + overleaf: true + } + ) + folderId = _confirmFolder(project, folderId) + const { result, project: newProject } = await _putElement( + project, + folderId, + doc, + 'doc' + ) + return { result, project: newProject } +} + +async function addFile(projectId, folderId, fileRef) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { rootFolder: true, name: true, overleaf: true } + ) + folderId = _confirmFolder(project, folderId) + const { result, project: newProject } = await _putElement( + project, + folderId, + fileRef, + 'file' + ) + return { result, project: newProject } +} + +async function addFolder(projectId, parentFolderId, folderName) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { rootFolder: true, name: true, overleaf: true } + ) + parentFolderId = _confirmFolder(project, parentFolderId) + const folder = new Folder({ name: folderName }) + await _putElement(project, parentFolderId, folder, 'folder') + return { folder, parentFolderId } +} + +async function replaceFileWithNew(projectId, fileId, newFileRef) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { rootFolder: true, name: true, overleaf: true } + ) + const { element: fileRef, path } = await ProjectLocator.promises.findElement({ + project, + element_id: fileId, + type: 'file' + }) + await _insertDeletedFileReference(projectId, fileRef) + const newProject = await Project.findOneAndUpdate( + { _id: project._id }, + { + $set: { + [`${path.mongo}._id`]: newFileRef._id, + [`${path.mongo}.created`]: new Date(), + [`${path.mongo}.linkedFileData`]: newFileRef.linkedFileData, + [`${path.mongo}.hash`]: newFileRef.hash + }, + $inc: { + version: 1, + [`${path.mongo}.rev`]: 1 + } + }, + { new: true } + ).exec() + // Note: Mongoose uses new:true to return the modified document + // https://mongoosejs.com/docs/api.html#model_Model.findOneAndUpdate + // but Mongo uses returnNewDocument:true instead + // https://docs.mongodb.com/manual/reference/method/db.collection.findOneAndUpdate/ + // We are using Mongoose here, but if we ever switch to a direct mongo call + // the next line will need to be updated. + return { oldFileRef: fileRef, project, path, newProject } +} + +async function mkdirp(projectId, path, options = {}) { + // defaults to case insensitive paths, use options {exactCaseMatch:true} + // to make matching case-sensitive + let folders = path.split('/') + folders = _.select(folders, folder => folder.length !== 0) + + const project = await ProjectGetter.promises.getProjectWithOnlyFolders( + projectId + ) + if (path === '/') { + logger.log( + { projectId: project._id }, + 'mkdir is only trying to make path of / so sending back root folder' + ) + return { newFolders: [], folder: project.rootFolder[0] } + } + + const newFolders = [] + let builtUpPath = '' + let lastFolder = null + for (const folderName of folders) { + builtUpPath += `/${folderName}` + try { + const { + element: foundFolder + } = await ProjectLocator.promises.findElementByPath({ + project, + path: builtUpPath, + exactCaseMatch: options.exactCaseMatch + }) + lastFolder = foundFolder + } catch (err) { + // Folder couldn't be found. Create it. + logger.log( + { path, projectId: project._id, folderName }, + 'making folder from mkdirp' + ) + const parentFolderId = lastFolder && lastFolder._id + const { + folder: newFolder, + parentFolderId: newParentFolderId + } = await addFolder(projectId, parentFolderId, folderName) + newFolder.parentFolder_id = newParentFolderId + lastFolder = newFolder + newFolders.push(newFolder) + } + } + return { folder: lastFolder, newFolders } +} + +async function moveEntity(projectId, entityId, destFolderId, entityType) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { rootFolder: true, name: true, overleaf: true } + ) + const { + element: entity, + path: entityPath + } = await ProjectLocator.promises.findElement({ + project, + element_id: entityId, + type: entityType + }) + // Prevent top-level docs/files with reserved names (to match v1 behaviour) + if (_blockedFilename(entityPath, entityType)) { + throw new Errors.InvalidNameError('blocked element name') + } + await _checkValidMove(project, entityType, entity, entityPath, destFolderId) + const { + docs: oldDocs, + files: oldFiles + } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(project) + // For safety, insert the entity in the destination + // location first, and then remove the original. If + // there is an error the entity may appear twice. This + // will cause some breakage but is better than being + // lost, which is what happens if this is done in the + // opposite order. + const { result } = await _putElement( + project, + destFolderId, + entity, + entityType + ) + // Note: putElement always pushes onto the end of an + // array so it will never change an existing mongo + // path. Therefore it is safe to remove an element + // from the project with an existing path after + // calling putElement. But we must be sure that we + // have not moved a folder subfolder of itself (which + // is done by _checkValidMove above) because that + // would lead to it being deleted. + const newProject = await _removeElementFromMongoArray( + Project, + projectId, + entityPath.mongo, + entityId + ) + const { + docs: newDocs, + files: newFiles + } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(newProject) + const startPath = entityPath.fileSystem + const endPath = result.path.fileSystem + const changes = { + oldDocs, + newDocs, + oldFiles, + newFiles, + newProject + } + // check that no files have been lost (or duplicated) + if ( + oldFiles.length !== newFiles.length || + oldDocs.length !== newDocs.length + ) { + logger.warn( + { + projectId, + oldDocs: oldDocs.length, + newDocs: newDocs.length, + oldFiles: oldFiles.length, + newFiles: newFiles.length, + origProject: project, + newProject + }, + "project corrupted moving files - shouldn't happen" + ) + throw new Error('unexpected change in project structure') + } + return { project, startPath, endPath, rev: entity.rev, changes } +} + +async function deleteEntity(projectId, entityId, entityType, callback) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { name: true, rootFolder: true, overleaf: true } + ) + const { element: entity, path } = await ProjectLocator.promises.findElement({ + project, + element_id: entityId, + type: entityType + }) + const newProject = await _removeElementFromMongoArray( + Project, + projectId, + path.mongo, + entityId + ) + return { entity, path, projectBeforeDeletion: project, newProject } +} + +async function renameEntity( + projectId, + entityId, + entityType, + newName, + callback +) { + const project = await ProjectGetter.promises.getProjectWithoutLock( + projectId, + { rootFolder: true, name: true, overleaf: true } + ) + const { + element: entity, + path: entPath, + folder: parentFolder + } = await ProjectLocator.promises.findElement({ + project, + element_id: entityId, + type: entityType + }) + const startPath = entPath.fileSystem + const endPath = path.join(path.dirname(entPath.fileSystem), newName) + + // Prevent top-level docs/files with reserved names (to match v1 behaviour) + if (_blockedFilename({ fileSystem: endPath }, entityType)) { + throw new Errors.InvalidNameError('blocked element name') + } + + // check if the new name already exists in the current folder + _checkValidElementName(parentFolder, newName) + + const { + docs: oldDocs, + files: oldFiles + } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(project) + + // we need to increment the project version number for any structure change + const newProject = await Project.findOneAndUpdate( + { _id: projectId }, + { $set: { [`${entPath.mongo}.name`]: newName }, $inc: { version: 1 } }, + { new: true } + ).exec() + + const { + docs: newDocs, + files: newFiles + } = await ProjectEntityHandler.promises.getAllEntitiesFromProject(newProject) + return { + project, + startPath, + endPath, + rev: entity.rev, + changes: { oldDocs, newDocs, oldFiles, newFiles, newProject } + } +} + +async function _insertDeletedDocReference(projectId, doc) { + await Project.updateOne( + { _id: projectId }, + { + $push: { + deletedDocs: { _id: doc._id, name: doc.name, deletedAt: new Date() } + } + } + ).exec() +} + +async function _insertDeletedFileReference(projectId, fileRef) { + await Project.updateOne( + { _id: projectId }, + { + $push: { + deletedFiles: { + _id: fileRef._id, + name: fileRef.name, + linkedFileData: fileRef.linkedFileData, + hash: fileRef.hash, + deletedAt: new Date() + } + } + } + ).exec() +} + +async function _removeElementFromMongoArray(model, modelId, path, elementId) { + const nonArrayPath = path.slice(0, path.lastIndexOf('.')) + const newDoc = model + .findOneAndUpdate( + { _id: modelId }, + { + $pull: { [nonArrayPath]: { _id: elementId } }, + $inc: { version: 1 } + }, + { new: true } + ) + .exec() + return newDoc +} + +function _countElements(project) { + function countFolder(folder) { + if (folder == null) { + return 0 + } + + let total = 0 + if (folder.folders) { + total += folder.folders.length + for (const subfolder of folder.folders) { + total += countFolder(subfolder) + } + } + if (folder.docs) { + total += folder.docs.length + } + if (folder.fileRefs) { + total += folder.fileRefs.length + } + return total + } + + return countFolder(project.rootFolder[0]) +} + +async function _putElement(project, folderId, element, type) { + if (element == null || element._id == null) { + logger.warn( + { projectId: project._id, folderId, element, type }, + 'failed trying to insert element as it was null' + ) + throw new Error('no element passed to be inserted') + } + + const pathSegment = _getMongoPathSegmentFromType(type) + + // original check path.resolve("/", element.name) isnt "/#{element.name}" or element.name.match("/") + // check if name is allowed + if (!SafePath.isCleanFilename(element.name)) { + logger.warn( + { projectId: project._id, folderId, element, type }, + 'failed trying to insert element as name was invalid' + ) + throw new Errors.InvalidNameError('invalid element name') + } + + if (folderId == null) { + folderId = project.rootFolder[0]._id + } + + if (_countElements(project) > Settings.maxEntitiesPerProject) { + logger.warn( + { projectId: project._id }, + 'project too big, stopping insertions' + ) + CooldownManager.putProjectOnCooldown(project._id) + throw new Error('project_has_to_many_files') + } + + const { element: folder, path } = await ProjectLocator.promises.findElement({ + project, + element_id: folderId, + type: 'folder' + }) + const newPath = { + fileSystem: `${path.fileSystem}/${element.name}`, + mongo: path.mongo + } + // check if the path would be too long + if (!SafePath.isAllowedLength(newPath.fileSystem)) { + throw new Errors.InvalidNameError('path too long') + } + // Prevent top-level docs/files with reserved names (to match v1 behaviour) + if (_blockedFilename(newPath, type)) { + throw new Errors.InvalidNameError('blocked element name') + } + _checkValidElementName(folder, element.name) + element._id = ObjectId(element._id.toString()) + const mongoPath = `${path.mongo}.${pathSegment}` + const newProject = await Project.findOneAndUpdate( + { _id: project._id }, + { $push: { [mongoPath]: element }, $inc: { version: 1 } }, + { new: true } + ).exec() + return { result: { path: newPath }, project: newProject } +} + +function _blockedFilename(entityPath, entityType) { + // check if name would be blocked in v1 + // javascript reserved names are forbidden for docs and files + // at the top-level (but folders with reserved names are allowed). + const isFolder = entityType === 'folder' + const dir = path.dirname(entityPath.fileSystem) + const file = path.basename(entityPath.fileSystem) + const isTopLevel = dir === '/' + if (isTopLevel && !isFolder && SafePath.isBlockedFilename(file)) { + return true + } else { + return false + } +} + +function _getMongoPathSegmentFromType(type) { + const pathSegment = ENTITY_TYPE_TO_MONGO_PATH_SEGMENT[type] + if (pathSegment == null) { + throw new Error(`Unknown entity type: ${type}`) + } + return pathSegment +} + +/** + * Check if the name is already taken by a doc, file or folder. If so, return an + * error "file already exists". + */ +function _checkValidElementName(folder, name) { + if (folder == null) { + return + } + const elements = [] + .concat(folder.docs || []) + .concat(folder.fileRefs || []) + .concat(folder.folders || []) + for (const element of elements) { + if (element.name === name) { + throw new Errors.InvalidNameError('file already exists') + } + } +} + +function _confirmFolder(project, folderId) { + if (folderId == null) { + return project.rootFolder[0]._id + } else { + return folderId + } +} + +async function _checkValidMove( + project, + entityType, + entity, + entityPath, + destFolderId +) { + const { + element: destEntity, + path: destFolderPath + } = await ProjectLocator.promises.findElement({ + project, + element_id: destFolderId, + type: 'folder' + }) + // check if there is already a doc/file/folder with the same name + // in the destination folder + _checkValidElementName(destEntity, entity.name) + if (/folder/.test(entityType)) { + logger.log( + { + destFolderPath: destFolderPath.fileSystem, + folderPath: entityPath.fileSystem + }, + 'checking folder is not moving into child folder' + ) + const isNestedFolder = + destFolderPath.fileSystem.slice(0, entityPath.fileSystem.length) === + entityPath.fileSystem + if (isNestedFolder) { + throw new Errors.InvalidNameError( + 'destination folder is a child folder of me' + ) + } + } +} diff --git a/services/web/app/src/Features/Project/ProjectEntityUpdateHandlerCanary.js b/services/web/app/src/Features/Project/ProjectEntityUpdateHandlerCanary.js new file mode 100644 index 0000000000..485c7bc444 --- /dev/null +++ b/services/web/app/src/Features/Project/ProjectEntityUpdateHandlerCanary.js @@ -0,0 +1,1634 @@ +/* NOTE: this file is an almost exact copy of ProjectEntityUpdateHandler.js. + * The only difference is that it imports + * ProjectEntityMongoUpdateHandlerCanary.js. It's meant to be a short-lived + * module, so that we can test the async/await code in production for some code + * paths only. + */ + +/* eslint-disable + camelcase, + handle-callback-err, + max-len, + one-var, + standard/no-callback-literal, +*/ + +// TODO: This file was created by bulk-decaffeinate. +// Fix any style issues and re-enable lint. +/* + * decaffeinate suggestions: + * DS101: Remove unnecessary use of Array.from + * DS102: Remove unnecessary code created because of implicit returns + * DS103: Rewrite code to no longer use __guard__ + * DS201: Simplify complex destructure assignments + * DS207: Consider shorter variations of null checks + * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md + */ +let ProjectEntityUpdateHandler, self +const _ = require('lodash') +const async = require('async') +const logger = require('logger-sharelatex') +const Settings = require('settings-sharelatex') +const path = require('path') +const { Doc } = require('../../models/Doc') +const DocstoreManager = require('../Docstore/DocstoreManager') +const DocumentUpdaterHandler = require('../../Features/DocumentUpdater/DocumentUpdaterHandler') +const Errors = require('../Errors/Errors') +const { File } = require('../../models/File') +const FileStoreHandler = require('../FileStore/FileStoreHandler') +const LockManager = require('../../infrastructure/LockManager') +const { Project } = require('../../models/Project') +const ProjectEntityHandler = require('./ProjectEntityHandler') +const ProjectGetter = require('./ProjectGetter') +const ProjectLocator = require('./ProjectLocator') +const ProjectUpdateHandler = require('./ProjectUpdateHandler') +const ProjectEntityMongoUpdateHandler = require('./ProjectEntityMongoUpdateHandlerCanary') +const SafePath = require('./SafePath') +const TpdsUpdateSender = require('../ThirdPartyDataStore/TpdsUpdateSender') + +const LOCK_NAMESPACE = 'sequentialProjectStructureUpdateLock' + +const validRootDocExtensions = Settings.validRootDocExtensions +const validRootDocRegExp = new RegExp( + `^\\.(${validRootDocExtensions.join('|')})$`, + 'i' +) + +const wrapWithLock = function(methodWithoutLock) { + // This lock is used to make sure that the project structure updates are made + // sequentially. In particular the updates must be made in mongo and sent to + // the doc-updater in the same order. + let methodWithLock + if (typeof methodWithoutLock === 'function') { + methodWithLock = function(project_id, ...rest) { + const adjustedLength = Math.max(rest.length, 1), + args = rest.slice(0, adjustedLength - 1), + callback = rest[adjustedLength - 1] + return LockManager.runWithLock( + LOCK_NAMESPACE, + project_id, + cb => methodWithoutLock(project_id, ...Array.from(args), cb), + callback + ) + } + methodWithLock.withoutLock = methodWithoutLock + return methodWithLock + } else { + // handle case with separate setup and locked stages + const wrapWithSetup = methodWithoutLock.beforeLock // a function to set things up before the lock + const mainTask = methodWithoutLock.withLock // function to execute inside the lock + methodWithLock = wrapWithSetup(function(project_id, ...rest) { + const adjustedLength = Math.max(rest.length, 1), + args = rest.slice(0, adjustedLength - 1), + callback = rest[adjustedLength - 1] + return LockManager.runWithLock( + LOCK_NAMESPACE, + project_id, + cb => mainTask(project_id, ...Array.from(args), cb), + callback + ) + }) + methodWithLock.withoutLock = wrapWithSetup(mainTask) + methodWithLock.beforeLock = methodWithoutLock.beforeLock + methodWithLock.mainTask = methodWithoutLock.withLock + return methodWithLock + } +} + +module.exports = ProjectEntityUpdateHandler = self = { + copyFileFromExistingProjectWithProject: wrapWithLock({ + beforeLock(next) { + return function( + project_id, + project, + folder_id, + originalProject_id, + origonalFileRef, + userId, + callback + ) { + if (callback == null) { + callback = function(error, fileRef, folder_id) {} + } + logger.log( + { project_id, folder_id, originalProject_id, origonalFileRef }, + 'copying file in s3 with project' + ) + folder_id = ProjectEntityMongoUpdateHandler._confirmFolder( + project, + folder_id + ) + if (origonalFileRef == null) { + logger.err( + { project_id, folder_id, originalProject_id, origonalFileRef }, + 'file trying to copy is null' + ) + return callback() + } + // convert any invalid characters in original file to '_' + const fileProperties = { + name: SafePath.clean(origonalFileRef.name) + } + if (origonalFileRef.linkedFileData != null) { + fileProperties.linkedFileData = origonalFileRef.linkedFileData + } + if (origonalFileRef.hash != null) { + fileProperties.hash = origonalFileRef.hash + } + const fileRef = new File(fileProperties) + return FileStoreHandler.copyFile( + originalProject_id, + origonalFileRef._id, + project._id, + fileRef._id, + function(err, fileStoreUrl) { + if (err != null) { + logger.warn( + { + err, + project_id, + folder_id, + originalProject_id, + origonalFileRef + }, + 'error coping file in s3' + ) + return callback(err) + } + return next( + project_id, + project, + folder_id, + originalProject_id, + origonalFileRef, + userId, + fileRef, + fileStoreUrl, + callback + ) + } + ) + } + }, + withLock( + project_id, + project, + folder_id, + originalProject_id, + origonalFileRef, + userId, + fileRef, + fileStoreUrl, + callback + ) { + if (callback == null) { + callback = function(error, fileRef, folder_id) {} + } + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x => x.id + ) + return ProjectEntityMongoUpdateHandler._putElement( + project, + folder_id, + fileRef, + 'file', + function(err, result, newProject) { + if (err != null) { + logger.warn( + { err, project_id, folder_id }, + 'error putting element as part of copy' + ) + return callback(err) + } + return TpdsUpdateSender.addFile( + { + project_id, + file_id: fileRef._id, + path: __guard__( + result != null ? result.path : undefined, + x1 => x1.fileSystem + ), + rev: fileRef.rev, + project_name: project.name + }, + function(err) { + if (err != null) { + logger.err( + { + err, + project_id, + folder_id, + originalProject_id, + origonalFileRef + }, + 'error sending file to tpds worker' + ) + } + const newFiles = [ + { + file: fileRef, + path: __guard__( + result != null ? result.path : undefined, + x2 => x2.fileSystem + ), + url: fileStoreUrl + } + ] + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + { newFiles, newProject }, + function(error) { + if (error != null) { + return callback(error) + } + return callback(null, fileRef, folder_id) + } + ) + } + ) + } + ) + } + }), + + updateDocLines( + project_id, + doc_id, + lines, + version, + ranges, + lastUpdatedAt, + lastUpdatedBy, + callback + ) { + if (callback == null) { + callback = function(error) {} + } + return ProjectGetter.getProjectWithoutDocLines(project_id, function( + err, + project + ) { + if (err != null) { + return callback(err) + } + if (project == null) { + return callback(new Errors.NotFoundError('project not found')) + } + logger.log({ project_id, doc_id }, 'updating doc lines') + return ProjectLocator.findElement( + { project, element_id: doc_id, type: 'docs' }, + function(err, doc, path) { + let isDeletedDoc = false + if (err != null) { + if (err instanceof Errors.NotFoundError) { + // We need to be able to update the doclines of deleted docs. This is + // so the doc-updater can flush a doc's content to the doc-store after + // the doc is deleted. + isDeletedDoc = true + doc = _.find( + project.deletedDocs, + doc => doc._id.toString() === doc_id.toString() + ) + } else { + return callback(err) + } + } + + if (doc == null) { + // Do not allow an update to a doc which has never exist on this project + logger.warn( + { doc_id, project_id, lines }, + 'doc not found while updating doc lines' + ) + return callback(new Errors.NotFoundError('doc not found')) + } + + logger.log( + { project_id, doc_id }, + 'telling docstore manager to update doc' + ) + return DocstoreManager.updateDoc( + project_id, + doc_id, + lines, + version, + ranges, + function(err, modified, rev) { + if (err != null) { + logger.warn( + { err, doc_id, project_id, lines }, + 'error sending doc to docstore' + ) + return callback(err) + } + logger.log( + { project_id, doc_id, modified }, + 'finished updating doc lines' + ) + // path will only be present if the doc is not deleted + if (modified && !isDeletedDoc) { + // Don't need to block for marking as updated + ProjectUpdateHandler.markAsUpdated( + project_id, + lastUpdatedAt, + lastUpdatedBy + ) + return TpdsUpdateSender.addDoc( + { + project_id, + path: path.fileSystem, + doc_id, + project_name: project.name, + rev + }, + callback + ) + } else { + return callback() + } + } + ) + } + ) + }) + }, + + setRootDoc(project_id, newRootDocID, callback) { + if (callback == null) { + callback = function(error) {} + } + logger.log({ project_id, rootDocId: newRootDocID }, 'setting root doc') + if (project_id == null || newRootDocID == null) { + return callback( + new Errors.InvalidError('missing arguments (project or doc)') + ) + } + ProjectEntityHandler.getDocPathByProjectIdAndDocId( + project_id, + newRootDocID, + function(err, docPath) { + if (err != null) { + return callback(err) + } + if (ProjectEntityUpdateHandler.isPathValidForRootDoc(docPath)) { + return Project.update( + { _id: project_id }, + { rootDoc_id: newRootDocID }, + {}, + callback + ) + } else { + return callback( + new Errors.UnsupportedFileTypeError( + 'invalid file extension for root doc' + ) + ) + } + } + ) + }, + + unsetRootDoc(project_id, callback) { + if (callback == null) { + callback = function(error) {} + } + logger.log({ project_id }, 'removing root doc') + return Project.update( + { _id: project_id }, + { $unset: { rootDoc_id: true } }, + {}, + callback + ) + }, + + _addDocAndSendToTpds(project_id, folder_id, doc, callback) { + if (callback == null) { + callback = function(error, result, project) {} + } + return ProjectEntityMongoUpdateHandler.addDoc( + project_id, + folder_id, + doc, + function(err, result, project) { + if (err != null) { + logger.warn( + { + err, + project_id, + folder_id, + doc_name: doc != null ? doc.name : undefined, + doc_id: doc != null ? doc._id : undefined + }, + 'error adding file with project' + ) + return callback(err) + } + return TpdsUpdateSender.addDoc( + { + project_id, + doc_id: doc != null ? doc._id : undefined, + path: __guard__( + result != null ? result.path : undefined, + x => x.fileSystem + ), + project_name: project.name, + rev: 0 + }, + function(err) { + if (err != null) { + return callback(err) + } + return callback(null, result, project) + } + ) + } + ) + }, + + addDoc(project_id, folder_id, docName, docLines, userId, callback) { + return self.addDocWithRanges( + project_id, + folder_id, + docName, + docLines, + {}, + userId, + callback + ) + }, + + addDocWithRanges: wrapWithLock({ + beforeLock(next) { + return function( + project_id, + folder_id, + docName, + docLines, + ranges, + userId, + callback + ) { + if (callback == null) { + callback = function(error, doc, folder_id) {} + } + if (!SafePath.isCleanFilename(docName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + // Put doc in docstore first, so that if it errors, we don't have a doc_id in the project + // which hasn't been created in docstore. + const doc = new Doc({ name: docName }) + return DocstoreManager.updateDoc( + project_id.toString(), + doc._id.toString(), + docLines, + 0, + ranges, + function(err, modified, rev) { + if (err != null) { + return callback(err) + } + return next( + project_id, + folder_id, + doc, + docName, + docLines, + ranges, + userId, + callback + ) + } + ) + } + }, + withLock( + project_id, + folder_id, + doc, + docName, + docLines, + ranges, + userId, + callback + ) { + if (callback == null) { + callback = function(error, doc, folder_id) {} + } + return ProjectEntityUpdateHandler._addDocAndSendToTpds( + project_id, + folder_id, + doc, + function(err, result, project) { + if (err != null) { + return callback(err) + } + const docPath = __guard__( + result != null ? result.path : undefined, + x => x.fileSystem + ) + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x1 => x1.id + ) + const newDocs = [ + { + doc, + path: docPath, + docLines: docLines.join('\n') + } + ] + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + { newDocs, newProject: project }, + function(error) { + if (error != null) { + return callback(error) + } + return callback(null, doc, folder_id) + } + ) + } + ) + } + }), + + _uploadFile( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + callback + ) { + if (callback == null) { + callback = function(error, fileStoreUrl, fileRef) {} + } + if (!SafePath.isCleanFilename(fileName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + const fileArgs = { + name: fileName, + linkedFileData + } + return FileStoreHandler.uploadFileFromDisk( + project_id, + fileArgs, + fsPath, + function(err, fileStoreUrl, fileRef) { + if (err != null) { + logger.warn( + { err, project_id, folder_id, file_name: fileName, fileRef }, + 'error uploading image to s3' + ) + return callback(err) + } + return callback(null, fileStoreUrl, fileRef) + } + ) + }, + + _addFileAndSendToTpds(project_id, folder_id, fileRef, callback) { + if (callback == null) { + callback = function(error) {} + } + return ProjectEntityMongoUpdateHandler.addFile( + project_id, + folder_id, + fileRef, + function(err, result, project) { + if (err != null) { + logger.warn( + { err, project_id, folder_id, file_name: fileRef.name, fileRef }, + 'error adding file with project' + ) + return callback(err) + } + return TpdsUpdateSender.addFile( + { + project_id, + file_id: fileRef._id, + path: __guard__( + result != null ? result.path : undefined, + x => x.fileSystem + ), + project_name: project.name, + rev: fileRef.rev + }, + function(err) { + if (err != null) { + return callback(err) + } + return callback(null, result, project) + } + ) + } + ) + }, + + addFile: wrapWithLock({ + beforeLock(next) { + return function( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + callback + ) { + if (!SafePath.isCleanFilename(fileName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + return ProjectEntityUpdateHandler._uploadFile( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + function(error, fileStoreUrl, fileRef) { + if (error != null) { + return callback(error) + } + return next( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + callback + ) + } + ) + } + }, + withLock( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + callback + ) { + if (callback == null) { + callback = function(error, fileRef, folder_id) {} + } + return ProjectEntityUpdateHandler._addFileAndSendToTpds( + project_id, + folder_id, + fileRef, + function(err, result, project) { + if (err != null) { + return callback(err) + } + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x => x.id + ) + const newFiles = [ + { + file: fileRef, + path: __guard__( + result != null ? result.path : undefined, + x1 => x1.fileSystem + ), + url: fileStoreUrl + } + ] + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + { newFiles, newProject: project }, + function(error) { + if (error != null) { + return callback(error) + } + return callback(null, fileRef, folder_id) + } + ) + } + ) + } + }), + + replaceFile: wrapWithLock({ + beforeLock(next) { + return function( + project_id, + file_id, + fsPath, + linkedFileData, + userId, + callback + ) { + // create a new file + const fileArgs = { + name: 'dummy-upload-filename', + linkedFileData + } + return FileStoreHandler.uploadFileFromDisk( + project_id, + fileArgs, + fsPath, + function(err, fileStoreUrl, fileRef) { + if (err != null) { + return callback(err) + } + return next( + project_id, + file_id, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + callback + ) + } + ) + } + }, + withLock( + project_id, + file_id, + fsPath, + linkedFileData, + userId, + newFileRef, + fileStoreUrl, + callback + ) { + return ProjectEntityMongoUpdateHandler.replaceFileWithNew( + project_id, + file_id, + newFileRef, + function(err, oldFileRef, project, path, newProject) { + if (err != null) { + return callback(err) + } + const oldFiles = [ + { + file: oldFileRef, + path: path.fileSystem + } + ] + const newFiles = [ + { + file: newFileRef, + path: path.fileSystem, + url: fileStoreUrl + } + ] + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x => x.id + ) + // Increment the rev for an in-place update (with the same path) so the third-party-datastore + // knows this is a new file. + // Ideally we would get this from ProjectEntityMongoUpdateHandler.replaceFileWithNew + // but it returns the original oldFileRef (after incrementing the rev value in mongo), + // so we add 1 to the rev from that. This isn't atomic and relies on the lock + // but it is acceptable for now. + return TpdsUpdateSender.addFile( + { + project_id: project._id, + file_id: newFileRef._id, + path: path.fileSystem, + rev: oldFileRef.rev + 1, + project_name: project.name + }, + function(err) { + if (err != null) { + return callback(err) + } + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + { oldFiles, newFiles, newProject }, + callback + ) + } + ) + } + ) + } + }), + + upsertDoc: wrapWithLock(function( + project_id, + folder_id, + docName, + docLines, + source, + userId, + callback + ) { + if (callback == null) { + callback = function(err, doc, folder_id, isNewDoc) {} + } + if (!SafePath.isCleanFilename(docName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + return ProjectLocator.findElement( + { project_id, element_id: folder_id, type: 'folder' }, + function(error, folder) { + if (error != null) { + return callback(error) + } + if (folder == null) { + return callback(new Error("Couldn't find folder")) + } + let existingDoc = null + for (let doc of Array.from(folder.docs)) { + if (doc.name === docName) { + existingDoc = doc + break + } + } + if (existingDoc != null) { + return DocumentUpdaterHandler.setDocument( + project_id, + existingDoc._id, + userId, + docLines, + source, + err => { + logger.log( + { project_id, doc_id: existingDoc._id }, + 'notifying users that the document has been updated' + ) + return DocumentUpdaterHandler.flushDocToMongo( + project_id, + existingDoc._id, + function(err) { + if (err != null) { + return callback(err) + } + return callback(null, existingDoc, existingDoc == null) + } + ) + } + ) + } else { + return self.addDocWithRanges.withoutLock( + project_id, + folder_id, + docName, + docLines, + {}, + userId, + function(err, doc) { + if (err != null) { + return callback(err) + } + return callback(null, doc, existingDoc == null) + } + ) + } + } + ) + }), + + upsertFile: wrapWithLock({ + beforeLock(next) { + return function( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + callback + ) { + if (!SafePath.isCleanFilename(fileName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + // create a new file + const fileArgs = { + name: fileName, + linkedFileData + } + return FileStoreHandler.uploadFileFromDisk( + project_id, + fileArgs, + fsPath, + function(err, fileStoreUrl, fileRef) { + if (err != null) { + return callback(err) + } + return next( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + callback + ) + } + ) + } + }, + withLock( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + newFileRef, + fileStoreUrl, + callback + ) { + if (callback == null) { + callback = function(err, file, isNewFile, existingFile) {} + } + return ProjectLocator.findElement( + { project_id, element_id: folder_id, type: 'folder' }, + function(error, folder) { + if (error != null) { + return callback(error) + } + if (folder == null) { + return callback(new Error("Couldn't find folder")) + } + let existingFile = null + for (let fileRef of Array.from(folder.fileRefs)) { + if (fileRef.name === fileName) { + existingFile = fileRef + break + } + } + if (existingFile != null) { + // this calls directly into the replaceFile main task (without the beforeLock part) + return self.replaceFile.mainTask( + project_id, + existingFile._id, + fsPath, + linkedFileData, + userId, + newFileRef, + fileStoreUrl, + function(err) { + if (err != null) { + return callback(err) + } + return callback( + null, + newFileRef, + existingFile == null, + existingFile + ) + } + ) + } else { + // this calls directly into the addFile main task (without the beforeLock part) + return self.addFile.mainTask( + project_id, + folder_id, + fileName, + fsPath, + linkedFileData, + userId, + newFileRef, + fileStoreUrl, + function(err) { + if (err != null) { + return callback(err) + } + return callback( + null, + newFileRef, + existingFile == null, + existingFile + ) + } + ) + } + } + ) + } + }), + + upsertDocWithPath: wrapWithLock(function( + project_id, + elementPath, + docLines, + source, + userId, + callback + ) { + if (!SafePath.isCleanPath(elementPath)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + const docName = path.basename(elementPath) + const folderPath = path.dirname(elementPath) + return self.mkdirp.withoutLock(project_id, folderPath, function( + err, + newFolders, + folder + ) { + if (err != null) { + return callback(err) + } + return self.upsertDoc.withoutLock( + project_id, + folder._id, + docName, + docLines, + source, + userId, + function(err, doc, isNewDoc) { + if (err != null) { + return callback(err) + } + return callback(null, doc, isNewDoc, newFolders, folder) + } + ) + }) + }), + + upsertFileWithPath: wrapWithLock({ + beforeLock(next) { + return function( + project_id, + elementPath, + fsPath, + linkedFileData, + userId, + callback + ) { + if (!SafePath.isCleanPath(elementPath)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + const fileName = path.basename(elementPath) + const folderPath = path.dirname(elementPath) + // create a new file + const fileArgs = { + name: fileName, + linkedFileData + } + return FileStoreHandler.uploadFileFromDisk( + project_id, + fileArgs, + fsPath, + function(err, fileStoreUrl, fileRef) { + if (err != null) { + return callback(err) + } + return next( + project_id, + folderPath, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + callback + ) + } + ) + } + }, + withLock( + project_id, + folderPath, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + callback + ) { + return self.mkdirp.withoutLock(project_id, folderPath, function( + err, + newFolders, + folder + ) { + if (err != null) { + return callback(err) + } + // this calls directly into the upsertFile main task (without the beforeLock part) + return self.upsertFile.mainTask( + project_id, + folder._id, + fileName, + fsPath, + linkedFileData, + userId, + fileRef, + fileStoreUrl, + function(err, newFile, isNewFile, existingFile) { + if (err != null) { + return callback(err) + } + return callback( + null, + newFile, + isNewFile, + existingFile, + newFolders, + folder + ) + } + ) + }) + } + }), + + deleteEntity: wrapWithLock(function( + project_id, + entity_id, + entityType, + userId, + callback + ) { + if (callback == null) { + callback = function(error) {} + } + logger.log({ entity_id, entityType, project_id }, 'deleting project entity') + if (entityType == null) { + logger.warn({ err: 'No entityType set', project_id, entity_id }) + return callback(new Error('No entityType set')) + } + entityType = entityType.toLowerCase() + return ProjectEntityMongoUpdateHandler.deleteEntity( + project_id, + entity_id, + entityType, + function(error, entity, path, projectBeforeDeletion, newProject) { + if (error != null) { + return callback(error) + } + return self._cleanUpEntity( + projectBeforeDeletion, + newProject, + entity, + entityType, + path.fileSystem, + userId, + function(error) { + if (error != null) { + return callback(error) + } + return TpdsUpdateSender.deleteEntity( + { + project_id, + path: path.fileSystem, + project_name: projectBeforeDeletion.name + }, + function(error) { + if (error != null) { + return callback(error) + } + return callback(null, entity_id) + } + ) + } + ) + } + ) + }), + + deleteEntityWithPath: wrapWithLock((project_id, path, userId, callback) => + ProjectLocator.findElementByPath({ project_id, path }, function( + err, + element, + type + ) { + if (err != null) { + return callback(err) + } + if (element == null) { + return callback(new Errors.NotFoundError('project not found')) + } + return self.deleteEntity.withoutLock( + project_id, + element._id, + type, + userId, + callback + ) + }) + ), + + mkdirp: wrapWithLock(function(project_id, path, callback) { + if (callback == null) { + callback = function(err, newlyCreatedFolders, lastFolderInPath) {} + } + for (let folder of Array.from(path.split('/'))) { + if (folder.length > 0 && !SafePath.isCleanFilename(folder)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + } + return ProjectEntityMongoUpdateHandler.mkdirp( + project_id, + path, + { exactCaseMatch: false }, + callback + ) + }), + + mkdirpWithExactCase: wrapWithLock(function(project_id, path, callback) { + if (callback == null) { + callback = function(err, newlyCreatedFolders, lastFolderInPath) {} + } + for (let folder of Array.from(path.split('/'))) { + if (folder.length > 0 && !SafePath.isCleanFilename(folder)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + } + return ProjectEntityMongoUpdateHandler.mkdirp( + project_id, + path, + { exactCaseMatch: true }, + callback + ) + }), + + addFolder: wrapWithLock(function( + project_id, + parentFolder_id, + folderName, + callback + ) { + if (!SafePath.isCleanFilename(folderName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + return ProjectEntityMongoUpdateHandler.addFolder( + project_id, + parentFolder_id, + folderName, + callback + ) + }), + + moveEntity: wrapWithLock(function( + project_id, + entity_id, + destFolderId, + entityType, + userId, + callback + ) { + if (callback == null) { + callback = function(error) {} + } + logger.log( + { entityType, entity_id, project_id, destFolderId }, + 'moving entity' + ) + if (entityType == null) { + logger.warn({ err: 'No entityType set', project_id, entity_id }) + return callback(new Error('No entityType set')) + } + entityType = entityType.toLowerCase() + return ProjectEntityMongoUpdateHandler.moveEntity( + project_id, + entity_id, + destFolderId, + entityType, + function(err, project, startPath, endPath, rev, changes) { + if (err != null) { + return callback(err) + } + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x => x.id + ) + TpdsUpdateSender.moveEntity({ + project_id, + project_name: project.name, + startPath, + endPath, + rev + }) + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + changes, + callback + ) + } + ) + }), + + renameEntity: wrapWithLock(function( + project_id, + entity_id, + entityType, + newName, + userId, + callback + ) { + if (!SafePath.isCleanFilename(newName)) { + return callback(new Errors.InvalidNameError('invalid element name')) + } + logger.log({ entity_id, project_id }, `renaming ${entityType}`) + if (entityType == null) { + logger.warn({ err: 'No entityType set', project_id, entity_id }) + return callback(new Error('No entityType set')) + } + entityType = entityType.toLowerCase() + + return ProjectEntityMongoUpdateHandler.renameEntity( + project_id, + entity_id, + entityType, + newName, + function(err, project, startPath, endPath, rev, changes) { + if (err != null) { + return callback(err) + } + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x => x.id + ) + TpdsUpdateSender.moveEntity({ + project_id, + project_name: project.name, + startPath, + endPath, + rev + }) + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + changes, + callback + ) + } + ) + }), + + // This doesn't directly update project structure but we need to take the lock + // to prevent anything else being queued before the resync update + resyncProjectHistory: wrapWithLock((project_id, callback) => + ProjectGetter.getProject( + project_id, + { rootFolder: true, overleaf: true }, + function(error, project) { + if (error != null) { + return callback(error) + } + + const projectHistoryId = __guard__( + __guard__( + project != null ? project.overleaf : undefined, + x1 => x1.history + ), + x => x.id + ) + if (projectHistoryId == null) { + error = new Errors.ProjectHistoryDisabledError( + `project history not enabled for ${project_id}` + ) + return callback(error) + } + + return ProjectEntityHandler.getAllEntitiesFromProject(project, function( + error, + docs, + files + ) { + if (error != null) { + return callback(error) + } + + docs = _.map(docs, doc => ({ + doc: doc.doc._id, + path: doc.path + })) + + files = _.map(files, file => ({ + file: file.file._id, + path: file.path, + url: FileStoreHandler._buildUrl(project_id, file.file._id) + })) + + return DocumentUpdaterHandler.resyncProjectHistory( + project_id, + projectHistoryId, + docs, + files, + callback + ) + }) + } + ) + ), + + isPathValidForRootDoc(docPath) { + let docExtension = path.extname(docPath) + return validRootDocRegExp.test(docExtension) + }, + + _cleanUpEntity( + project, + newProject, + entity, + entityType, + path, + userId, + callback + ) { + if (callback == null) { + callback = function(error) {} + } + return self._updateProjectStructureWithDeletedEntity( + project, + newProject, + entity, + entityType, + path, + userId, + function(error) { + if (error != null) { + return callback(error) + } + if (entityType.indexOf('file') !== -1) { + return self._cleanUpFile(project, entity, path, userId, callback) + } else if (entityType.indexOf('doc') !== -1) { + return self._cleanUpDoc(project, entity, path, userId, callback) + } else if (entityType.indexOf('folder') !== -1) { + return self._cleanUpFolder(project, entity, path, userId, callback) + } else { + return callback() + } + } + ) + }, + + // Note: the _cleanUpEntity code and _updateProjectStructureWithDeletedEntity + // methods both need to recursively iterate over the entities in folder. + // These are currently using separate implementations of the recursion. In + // future, these could be simplified using a common project entity iterator. + _updateProjectStructureWithDeletedEntity( + project, + newProject, + entity, + entityType, + entityPath, + userId, + callback + ) { + // compute the changes to the project structure + let changes + if (callback == null) { + callback = function(error) {} + } + if (entityType.indexOf('file') !== -1) { + changes = { oldFiles: [{ file: entity, path: entityPath }] } + } else if (entityType.indexOf('doc') !== -1) { + changes = { oldDocs: [{ doc: entity, path: entityPath }] } + } else if (entityType.indexOf('folder') !== -1) { + changes = { oldDocs: [], oldFiles: [] } + var _recurseFolder = function(folder, folderPath) { + for (let doc of Array.from(folder.docs)) { + changes.oldDocs.push({ doc, path: path.join(folderPath, doc.name) }) + } + for (let file of Array.from(folder.fileRefs)) { + changes.oldFiles.push({ + file, + path: path.join(folderPath, file.name) + }) + } + return Array.from(folder.folders).map(childFolder => + _recurseFolder(childFolder, path.join(folderPath, childFolder.name)) + ) + } + _recurseFolder(entity, entityPath) + } + // now send the project structure changes to the docupdater + changes.newProject = newProject + const project_id = project._id.toString() + const projectHistoryId = __guard__( + project.overleaf != null ? project.overleaf.history : undefined, + x => x.id + ) + return DocumentUpdaterHandler.updateProjectStructure( + project_id, + projectHistoryId, + userId, + changes, + callback + ) + }, + + _cleanUpDoc(project, doc, path, userId, callback) { + if (callback == null) { + callback = function(error) {} + } + const project_id = project._id.toString() + const doc_id = doc._id.toString() + const unsetRootDocIfRequired = callback => { + if ( + project.rootDoc_id != null && + project.rootDoc_id.toString() === doc_id + ) { + return this.unsetRootDoc(project_id, callback) + } else { + return callback() + } + } + + return unsetRootDocIfRequired(function(error) { + if (error != null) { + return callback(error) + } + return ProjectEntityMongoUpdateHandler._insertDeletedDocReference( + project._id, + doc, + function(error) { + if (error != null) { + return callback(error) + } + return DocumentUpdaterHandler.deleteDoc(project_id, doc_id, function( + error + ) { + if (error != null) { + return callback(error) + } + return DocstoreManager.deleteDoc(project_id, doc_id, callback) + }) + } + ) + }) + }, + + _cleanUpFile(project, file, path, userId, callback) { + if (callback == null) { + callback = function(error) {} + } + return ProjectEntityMongoUpdateHandler._insertDeletedFileReference( + project._id, + file, + callback + ) + }, + + _cleanUpFolder(project, folder, folderPath, userId, callback) { + if (callback == null) { + callback = function(error) {} + } + const jobs = [] + for (let doc of Array.from(folder.docs)) { + ;(function(doc) { + const docPath = path.join(folderPath, doc.name) + return jobs.push(callback => + self._cleanUpDoc(project, doc, docPath, userId, callback) + ) + })(doc) + } + + for (let file of Array.from(folder.fileRefs)) { + ;(function(file) { + const filePath = path.join(folderPath, file.name) + return jobs.push(callback => + self._cleanUpFile(project, file, filePath, userId, callback) + ) + })(file) + } + + for (let childFolder of Array.from(folder.folders)) { + ;(function(childFolder) { + folderPath = path.join(folderPath, childFolder.name) + return jobs.push(callback => + self._cleanUpFolder( + project, + childFolder, + folderPath, + userId, + callback + ) + ) + })(childFolder) + } + + return async.series(jobs, callback) + } +} + +function __guard__(value, transform) { + return typeof value !== 'undefined' && value !== null + ? transform(value) + : undefined +} diff --git a/services/web/app/src/Features/Project/ProjectLocator.js b/services/web/app/src/Features/Project/ProjectLocator.js index c71f695514..00eb24e8fd 100644 --- a/services/web/app/src/Features/Project/ProjectLocator.js +++ b/services/web/app/src/Features/Project/ProjectLocator.js @@ -4,6 +4,7 @@ const Errors = require('../Errors/Errors') const _ = require('underscore') const logger = require('logger-sharelatex') const async = require('async') +const { promisifyAll } = require('../../util/promises') const ProjectLocator = { findElement(options, _callback) { @@ -327,3 +328,9 @@ function getIndexOf(searchEntity, id) { } module.exports = ProjectLocator +module.exports.promises = promisifyAll(ProjectLocator, { + multiResult: { + findElement: ['element', 'path', 'folder'], + findElementByPath: ['element', 'type'] + } +}) diff --git a/services/web/app/src/infrastructure/LockManager.js b/services/web/app/src/infrastructure/LockManager.js index 7c4a05bfa8..65717d0928 100644 --- a/services/web/app/src/infrastructure/LockManager.js +++ b/services/web/app/src/infrastructure/LockManager.js @@ -1,3 +1,4 @@ +const { callbackify, promisify } = require('util') const metrics = require('metrics-sharelatex') const RedisWrapper = require('./RedisWrapper') const rclient = RedisWrapper.client('lock') @@ -182,3 +183,11 @@ const LockManager = { } module.exports = LockManager + +const promisifiedRunWithLock = promisify(LockManager.runWithLock) +LockManager.promises = { + runWithLock(namespace, id, runner) { + const cbRunner = callbackify(runner) + return promisifiedRunWithLock(namespace, id, cbRunner) + } +} diff --git a/services/web/app/src/util/promises.js b/services/web/app/src/util/promises.js index f09e18648e..7c96472d03 100644 --- a/services/web/app/src/util/promises.js +++ b/services/web/app/src/util/promises.js @@ -3,6 +3,8 @@ const pLimit = require('p-limit') module.exports = { promisifyAll, + promisifyMultiResult, + callbackifyMultiResult, expressify, promiseMapWithLimit } @@ -19,22 +21,100 @@ module.exports = { * * This will not magically fix all modules. Special cases should be promisified * manually. + * + * The second argument is a bag of options: + * + * - without: an array of function names that shouldn't be promisified + * + * - multiResult: an object whose keys are function names and values are lists + * of parameter names. This is meant for functions that invoke their callbacks + * with more than one result in separate parameters. The promisifed function + * will return these results as a single object, with each result keyed under + * the corresponding parameter name. */ function promisifyAll(module, opts = {}) { - const { without = [] } = opts + const { without = [], multiResult = {} } = opts const promises = {} for (const propName of Object.getOwnPropertyNames(module)) { if (without.includes(propName)) { continue } const propValue = module[propName] - if (typeof propValue === 'function') { + if (typeof propValue !== 'function') { + continue + } + if (multiResult[propName] != null) { + promises[propName] = promisifyMultiResult( + propValue, + multiResult[propName] + ).bind(module) + } else { promises[propName] = promisify(propValue).bind(module) } } return promises } +/** + * Promisify a function that returns multiple results via additional callback + * parameters. + * + * The promisified function returns the results in a single object whose keys + * are the names given in the array `resultNames`. + * + * Example: + * + * function f(callback) { + * return callback(null, 1, 2, 3) + * } + * + * const g = promisifyMultiResult(f, ['a', 'b', 'c']) + * + * const result = await g() // returns {a: 1, b: 2, c: 3} + */ +function promisifyMultiResult(fn, resultNames) { + function promisified(...args) { + return new Promise((resolve, reject) => { + try { + fn(...args, (err, ...results) => { + if (err != null) { + return reject(err) + } + const promiseResult = {} + for (let i = 0; i < resultNames.length; i++) { + promiseResult[resultNames[i]] = results[i] + } + resolve(promiseResult) + }) + } catch (err) { + reject(err) + } + }) + } + return promisified +} + +/** + * Reverse the effect of `promisifyMultiResult`. + * + * This is meant for providing a temporary backward compatible callback + * interface while we migrate to promises. + */ +function callbackifyMultiResult(fn, resultNames) { + function callbackified(...args) { + const [callback] = args.splice(-1) + fn(...args) + .then(result => { + const cbResults = resultNames.map(resultName => result[resultName]) + callback(null, ...cbResults) + }) + .catch(err => { + callback(err) + }) + } + return callbackified +} + /** * Transform an async function into an Express middleware * diff --git a/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js b/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js new file mode 100644 index 0000000000..52ee396cc6 --- /dev/null +++ b/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerCanaryTests.js @@ -0,0 +1,1034 @@ +const { expect } = require('chai') +const sinon = require('sinon') +const tk = require('timekeeper') +const Errors = require('../../../../app/src/Features/Errors/Errors') +const { ObjectId } = require('mongoose').Types +const SandboxedModule = require('sandboxed-module') +const { Project } = require('../helpers/models/Project') + +const MODULE_PATH = + '../../../../app/src/Features/Project/ProjectEntityMongoUpdateHandlerCanary' + +describe('ProjectEntityMongoUpdateHandler', function() { + beforeEach(function() { + this.doc = { + _id: ObjectId(), + name: 'test-doc.txt', + lines: ['hello', 'world'], + rev: 1234 + } + this.docPath = { + mongo: 'rootFolder.0.docs.0', + fileSystem: '/test-doc.txt' + } + this.file = { + _id: ObjectId(), + name: 'something.jpg', + linkedFileData: { provider: 'url' }, + hash: 'some-hash' + } + this.filePath = { + fileSystem: '/something.png', + mongo: 'rootFolder.0.fileRefs.0' + } + this.subfolder = { _id: ObjectId(), name: 'test-subfolder' } + this.subfolderPath = { + fileSystem: '/test-folder/test-subfolder', + mongo: 'rootFolder.0.folders.0.folders.0' + } + this.folder = { + _id: ObjectId(), + name: 'test-folder', + folders: [this.subfolder] + } + this.folderPath = { + fileSystem: '/test-folder', + mongo: 'rootFolder.0.folders.0' + } + this.rootFolder = { + _id: ObjectId(), + folders: [this.folder], + docs: [this.doc], + fileRefs: [this.file] + } + this.rootFolderPath = { + fileSystem: '/', + mongo: 'rootFolder.0' + } + this.project = { + _id: ObjectId(), + name: 'project name', + rootFolder: [this.rootFolder] + } + + this.logger = { + log: sinon.stub(), + warn: sinon.stub(), + error: sinon.stub(), + err() {} + } + this.Settings = { maxEntitiesPerProject: 100 } + this.CooldownManager = {} + this.LockManager = { + promises: { + runWithLock: sinon.spy((namespace, id, runner) => runner()) + } + } + + this.FolderModel = sinon.stub() + this.ProjectMock = sinon.mock(Project) + this.ProjectEntityHandler = { + promises: { + getAllEntitiesFromProject: sinon.stub() + } + } + this.ProjectLocator = { + promises: { + findElement: sinon.stub().rejects(new Error('not found')), + findElementByPath: sinon.stub().rejects(new Error('not found')) + } + } + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.rootFolder._id, + type: 'folder' + }) + .resolves({ + element: this.rootFolder, + path: this.rootFolderPath + }) + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.folder._id, + type: 'folder' + }) + .resolves({ + element: this.folder, + path: this.folderPath, + folder: this.rootFolder + }) + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.subfolder._id, + type: 'folder' + }) + .resolves({ + element: this.subfolder, + path: this.subfolderPath, + folder: this.folder + }) + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.file._id, + type: 'file' + }) + .resolves({ + element: this.file, + path: this.filePath, + folder: this.rootFolder + }) + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.doc._id, + type: 'doc' + }) + .resolves({ + element: this.doc, + path: this.docPath, + folder: this.rootFolder + }) + this.ProjectLocator.promises.findElementByPath + .withArgs( + sinon.match({ + project: this.project, + path: '/' + }) + ) + .resolves({ element: this.rootFolder, type: 'folder' }) + this.ProjectLocator.promises.findElementByPath + .withArgs( + sinon.match({ + project: this.project, + path: '/test-folder' + }) + ) + .resolves({ element: this.folder, type: 'folder' }) + this.ProjectLocator.promises.findElementByPath + .withArgs( + sinon.match({ + project: this.project, + path: '/test-folder/test-subfolder' + }) + ) + .resolves({ element: this.subfolder, type: 'folder' }) + + this.ProjectGetter = { + promises: { + getProjectWithoutLock: sinon + .stub() + .withArgs(this.project._id) + .resolves(this.project), + getProjectWithOnlyFolders: sinon.stub().resolves(this.project) + } + } + + this.subject = SandboxedModule.require(MODULE_PATH, { + globals: { + console: console + }, + requires: { + 'logger-sharelatex': this.logger, + 'settings-sharelatex': this.Settings, + '../Cooldown/CooldownManager': this.CooldownManager, + '../../models/Folder': { Folder: this.FolderModel }, + '../../infrastructure/LockManager': this.LockManager, + '../../models/Project': { Project }, + './ProjectEntityHandler': this.ProjectEntityHandler, + './ProjectLocator': this.ProjectLocator, + './ProjectGetter': this.ProjectGetter, + // We need to provide Errors here to make instance check work + '../Errors/Errors': Errors + } + }) + }) + + afterEach(function() { + this.ProjectMock.restore() + }) + + beforeEach(function() { + tk.freeze(Date.now()) + }) + + afterEach(function() { + tk.reset() + }) + + describe('addDoc', function() { + beforeEach(async function() { + const doc = { _id: ObjectId(), name: 'other.txt' } + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { 'rootFolder.0.folders.0.docs': doc }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.result = await this.subject.promises.addDoc( + this.project._id, + this.folder._id, + doc + ) + }) + + it('adds the document in Mongo', function() { + this.ProjectMock.verify() + }) + + it('returns path info and the project', function() { + expect(this.result).to.deep.equal({ + result: { + path: { + mongo: 'rootFolder.0.folders.0', + fileSystem: '/test-folder/other.txt' + } + }, + project: this.project + }) + }) + }) + + describe('addFile', function() { + beforeEach(function() { + this.newFile = { _id: ObjectId(), name: 'picture.jpg' } + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { 'rootFolder.0.folders.0.fileRefs': this.newFile }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + }) + + describe('happy path', function() { + beforeEach(async function() { + this.result = await this.subject.promises.addFile( + this.project._id, + this.folder._id, + this.newFile + ) + }) + + it('adds the file in Mongo', function() { + this.ProjectMock.verify() + }) + + it('returns path info and the project', function() { + expect(this.result).to.deep.equal({ + result: { + path: { + mongo: 'rootFolder.0.folders.0', + fileSystem: '/test-folder/picture.jpg' + } + }, + project: this.project + }) + }) + }) + + describe('when entity limit is reached', function() { + beforeEach(function() { + this.savedMaxEntities = this.Settings.maxEntitiesPerProject + this.Settings.maxEntitiesPerProject = 3 + }) + + afterEach(function() { + this.Settings.maxEntitiesPerProject = this.savedMaxEntities + }) + + it('should throw an error', async function() { + await expect( + this.subject.promises.addFile( + this.project._id, + this.folder._id, + this.newFile + ) + ).to.be.rejected + }) + }) + }) + + describe('addFolder', function() { + beforeEach(async function() { + const folderName = 'New folder' + this.FolderModel.withArgs({ name: folderName }).returns({ + _id: ObjectId(), + name: folderName + }) + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { + 'rootFolder.0.folders.0.folders': sinon.match({ + name: folderName + }) + }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + await this.subject.promises.addFolder( + this.project._id, + this.folder._id, + folderName + ) + }) + + it('adds the folder in Mongo', function() { + this.ProjectMock.verify() + }) + }) + + describe('replaceFileWithNew', function() { + beforeEach(async function() { + const newFile = { + _id: ObjectId(), + name: 'some-other-file.png', + linkedFileData: { some: 'data' }, + hash: 'some-hash' + } + // Add a deleted file record + this.ProjectMock.expects('updateOne') + .withArgs( + { _id: this.project._id }, + { + $push: { + deletedFiles: { + _id: this.file._id, + name: this.file.name, + linkedFileData: this.file.linkedFileData, + hash: this.file.hash, + deletedAt: sinon.match.date + } + } + } + ) + .chain('exec') + .resolves() + // Update the file in place + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $set: { + 'rootFolder.0.fileRefs.0._id': newFile._id, + 'rootFolder.0.fileRefs.0.created': sinon.match.date, + 'rootFolder.0.fileRefs.0.linkedFileData': newFile.linkedFileData, + 'rootFolder.0.fileRefs.0.hash': newFile.hash + }, + $inc: { + version: 1, + 'rootFolder.0.fileRefs.0.rev': 1 + } + } + ) + .chain('exec') + .resolves(this.project) + await this.subject.promises.replaceFileWithNew( + this.project._id, + this.file._id, + newFile + ) + }) + + it('updates the database', function() { + this.ProjectMock.verify() + }) + }) + + describe('mkdirp', function() { + describe('when the path is just a slash', function() { + beforeEach(async function() { + this.result = await this.subject.promises.mkdirp(this.project._id, '/') + }) + + it('should return the root folder', function() { + expect(this.result.folder).to.deep.equal(this.rootFolder) + }) + + it('should not report a parent folder', function() { + expect(this.result.folder.parentFolder_id).not.to.exist + }) + + it('should not return new folders', function() { + expect(this.result.newFolders).to.have.length(0) + }) + }) + + describe('when the folder already exists', function() { + beforeEach(async function() { + this.result = await this.subject.promises.mkdirp( + this.project._id, + '/test-folder' + ) + }) + + it('should return the existing folder', function() { + expect(this.result.folder).to.deep.equal(this.folder) + }) + + it('should report the parent folder', function() { + expect(this.result.folder.parentFolder_id).not.equal( + this.rootFolder._id + ) + }) + + it('should not return new folders', function() { + expect(this.result.newFolders).to.have.length(0) + }) + }) + + describe('when the path is a new folder at the top level', function() { + beforeEach(async function() { + this.newFolder = { _id: ObjectId(), name: 'new-folder' } + this.FolderModel.returns(this.newFolder) + this.exactCaseMatch = false + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { 'rootFolder.0.folders': this.newFolder }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.result = await this.subject.promises.mkdirp( + this.project._id, + '/new-folder/', + { exactCaseMatch: this.exactCaseMatch } + ) + }) + it('should update the database', function() { + this.ProjectMock.verify() + }) + + it('should make just one folder', function() { + expect(this.result.newFolders).to.have.length(1) + }) + + it('should return the new folder', function() { + expect(this.result.folder.name).to.equal('new-folder') + }) + + it('should return the parent folder', function() { + expect(this.result.folder.parentFolder_id).to.equal(this.rootFolder._id) + }) + + it('should pass the exactCaseMatch option to ProjectLocator', function() { + expect( + this.ProjectLocator.promises.findElementByPath + ).to.have.been.calledWithMatch({ exactCaseMatch: this.exactCaseMatch }) + }) + }) + + describe('adding a subfolder', function() { + beforeEach(async function() { + this.newFolder = { _id: ObjectId(), name: 'new-folder' } + this.FolderModel.returns(this.newFolder) + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { + 'rootFolder.0.folders.0.folders': sinon.match({ + name: 'new-folder' + }) + }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.result = await this.subject.promises.mkdirp( + this.project._id, + '/test-folder/new-folder' + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + + it('should create one folder', function() { + expect(this.result.newFolders).to.have.length(1) + }) + + it('should return the new folder', function() { + expect(this.result.folder.name).to.equal('new-folder') + }) + + it('should return the parent folder', function() { + expect(this.result.folder.parentFolder_id).to.equal(this.folder._id) + }) + }) + + describe('when mutliple folders are missing', async function() { + beforeEach(function() { + this.folder1 = { _id: ObjectId(), name: 'folder1' } + this.folder1Path = { + fileSystem: '/test-folder/folder1', + mongo: 'rootFolder.0.folders.0.folders.0' + } + this.folder2 = { _id: ObjectId(), name: 'folder2' } + this.folder2Path = { + fileSystem: '/test-folder/folder1/folder2', + mongo: 'rootFolder.0.folders.0.folders.0.folders.0' + } + this.FolderModel.onFirstCall().returns(this.folder1) + this.FolderModel.onSecondCall().returns(this.folder2) + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.folder1._id, + type: 'folder' + }) + .resolves({ + element: this.folder1, + path: this.folder1Path + }) + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.folder2._id, + type: 'folder' + }) + .resolves({ + element: this.folder2, + path: this.folder2Path + }) + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { + 'rootFolder.0.folders.0.folders': sinon.match({ + name: 'folder1' + }) + }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { + 'rootFolder.0.folders.0.folders.0.folders': sinon.match({ + name: 'folder2' + }) + }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + }) + ;[ + { + description: 'without a trailing slash', + path: '/test-folder/folder1/folder2' + }, + { + description: 'with a trailing slash', + path: '/test-folder/folder1/folder2/' + } + ].forEach(({ description, path }) => { + describe(description, function() { + beforeEach(async function() { + this.result = await this.subject.promises.mkdirp( + this.project._id, + path + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + + it('should add multiple folders', function() { + const newFolders = this.result.newFolders + expect(newFolders).to.have.length(2) + expect(newFolders[0].name).to.equal('folder1') + expect(newFolders[1].name).to.equal('folder2') + }) + + it('should return the last folder', function() { + expect(this.result.folder.name).to.equal('folder2') + }) + + it('should return the parent folder', function() { + expect(this.result.folder.parentFolder_id).to.equal( + this.folder1._id + ) + }) + }) + }) + }) + }) + + describe('moveEntity', function() { + describe('moving a doc into a different folder', function() { + beforeEach(async function() { + this.pathAfterMove = { + fileSystem: '/somewhere/else.txt' + } + this.oldDocs = ['old-doc'] + this.oldFiles = ['old-file'] + this.newDocs = ['new-doc'] + this.newFiles = ['new-file'] + + this.ProjectEntityHandler.promises.getAllEntitiesFromProject + .onFirstCall() + .resolves({ docs: this.oldDocs, files: this.oldFiles }) + this.ProjectEntityHandler.promises.getAllEntitiesFromProject + .onSecondCall() + .resolves({ docs: this.newDocs, files: this.newFiles }) + + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { 'rootFolder.0.folders.0.docs': this.doc }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $pull: { 'rootFolder.0.docs': { _id: this.doc._id } }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.result = await this.subject.promises.moveEntity( + this.project._id, + this.doc._id, + this.folder._id, + 'doc' + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + + it('should report what changed', function() { + expect(this.result).to.deep.equal({ + project: this.project, + startPath: '/test-doc.txt', + endPath: '/test-folder/test-doc.txt', + rev: this.doc.rev, + changes: { + oldDocs: this.oldDocs, + newDocs: this.newDocs, + oldFiles: this.oldFiles, + newFiles: this.newFiles, + newProject: this.project + } + }) + }) + }) + + describe('when moving a folder inside itself', function() { + it('throws an error', async function() { + await expect( + this.subject.promises.moveEntity( + this.project._id, + this.folder._id, + this.folder._id, + 'folder' + ) + ).to.be.rejectedWith(Errors.InvalidNameError) + }) + }) + + describe('when moving a folder to a subfolder of itself', function() { + it('throws an error', async function() { + await expect( + this.subject.promises.moveEntity( + this.project._id, + this.folder._id, + this.subfolder._id, + 'folder' + ) + ).to.be.rejectedWith(Errors.InvalidNameError) + }) + }) + }) + + describe('deleteEntity', function() { + beforeEach(async function() { + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $pull: { 'rootFolder.0.docs': { _id: this.doc._id } }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + await this.subject.promises.deleteEntity( + this.project._id, + this.doc._id, + 'doc' + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + }) + + describe('renameEntity', function() { + describe('happy path', function() { + beforeEach(async function() { + this.newName = 'new.tex' + this.oldDocs = ['old-doc'] + this.oldFiles = ['old-file'] + this.newDocs = ['new-doc'] + this.newFiles = ['new-file'] + + this.ProjectEntityHandler.promises.getAllEntitiesFromProject + .onFirstCall() + .resolves({ docs: this.oldDocs, files: this.oldFiles }) + this.ProjectEntityHandler.promises.getAllEntitiesFromProject + .onSecondCall() + .resolves({ docs: this.newDocs, files: this.newFiles }) + + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $set: { 'rootFolder.0.docs.0.name': this.newName }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + this.result = await this.subject.promises.renameEntity( + this.project._id, + this.doc._id, + 'doc', + this.newName + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + + it('returns info', function() { + expect(this.result).to.deep.equal({ + project: this.project, + startPath: '/test-doc.txt', + endPath: '/new.tex', + rev: this.doc.rev, + changes: { + oldDocs: this.oldDocs, + newDocs: this.newDocs, + oldFiles: this.oldFiles, + newFiles: this.newFiles, + newProject: this.project + } + }) + }) + }) + + describe('name already exists', function() { + it('should throw an error', async function() { + await expect( + this.subject.promises.renameEntity( + this.project._id, + this.doc._id, + 'doc', + this.folder.name + ) + ).to.be.rejectedWith(Errors.InvalidNameError) + }) + }) + }) + + describe('_putElement', function() { + describe('updating the project', function() { + describe('when the parent folder is given', function() { + beforeEach(function() { + this.newFile = { _id: ObjectId(), name: 'new file.png' } + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { 'rootFolder.0.folders.0.fileRefs': this.newFile }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + }) + + it('should update the database', async function() { + await this.subject.promises._putElement( + this.project, + this.folder._id, + this.newFile, + 'files' + ) + this.ProjectMock.verify() + }) + + it('should add an s onto the type if not included', async function() { + await this.subject.promises._putElement( + this.project, + this.folder._id, + this.newFile, + 'file' + ) + this.ProjectMock.verify() + }) + }) + + describe('error cases', function() { + it('should throw an error if element is null', async function() { + await expect( + this.subject.promises._putElement( + this.project, + this.folder._id, + null, + 'file' + ) + ).to.be.rejected + }) + + it('should error if the element has no _id', async function() { + const file = { name: 'something' } + await expect( + this.subject.promises._putElement( + this.project, + this.folder._id, + file, + 'file' + ) + ).to.be.rejected + }) + + it('should error if element name contains invalid characters', async function() { + const file = { _id: ObjectId(), name: 'something*bad' } + await expect( + this.subject.promises._putElement( + this.project, + this.folder._id, + file, + 'file' + ) + ).to.be.rejected + }) + + it('should error if element name is too long', async function() { + const file = { + _id: ObjectId(), + name: 'long-'.repeat(1000) + 'something' + } + await expect( + this.subject.promises._putElement( + this.project, + this.folder._id, + file, + 'file' + ) + ).to.be.rejectedWith(Errors.InvalidNameError) + }) + + it('should error if the folder name is too long', async function() { + const file = { + _id: ObjectId(), + name: 'something' + } + this.ProjectLocator.promises.findElement + .withArgs({ + project: this.project, + element_id: this.folder._id, + type: 'folder' + }) + .resolves({ + element: this.folder, + path: { fileSystem: 'subdir/'.repeat(1000) + 'foo' } + }) + await expect( + this.subject.promises._putElement( + this.project, + this.folder._id, + file, + 'file' + ) + ).to.be.rejectedWith(Errors.InvalidNameError) + }) + ;['file', 'doc', 'folder'].forEach(entityType => { + it(`should error if a ${entityType} already exists with the same name`, async function() { + const file = { + _id: ObjectId(), + name: this[entityType].name + } + await expect( + this.subject.promises._putElement( + this.project, + null, + file, + 'file' + ) + ).to.be.rejectedWith(Errors.InvalidNameError) + }) + }) + }) + }) + + describe('when the parent folder is not given', function() { + it('should default to root folder insert', async function() { + this.newFile = { _id: ObjectId(), name: 'new file.png' } + this.ProjectMock.expects('findOneAndUpdate') + .withArgs( + { _id: this.project._id }, + { + $push: { 'rootFolder.0.fileRefs': this.newFile }, + $inc: { version: 1 } + } + ) + .chain('exec') + .resolves(this.project) + await this.subject.promises._putElement( + this.project, + this.rootFolder._id, + this.newFile, + 'file' + ) + }) + }) + }) + + describe('_insertDeletedDocReference', function() { + beforeEach(async function() { + this.ProjectMock.expects('updateOne') + .withArgs( + { _id: this.project._id }, + { + $push: { + deletedDocs: { + _id: this.doc._id, + name: this.doc.name, + deletedAt: sinon.match.date + } + } + } + ) + .chain('exec') + .resolves() + await this.subject.promises._insertDeletedDocReference( + this.project._id, + this.doc + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + }) + + describe('_insertDeletedFileReference', function() { + beforeEach(async function() { + this.ProjectMock.expects('updateOne') + .withArgs( + { _id: this.project._id }, + { + $push: { + deletedFiles: { + _id: this.file._id, + name: this.file.name, + linkedFileData: this.file.linkedFileData, + hash: this.file.hash, + deletedAt: sinon.match.date + } + } + } + ) + .chain('exec') + .resolves() + await this.subject.promises._insertDeletedFileReference( + this.project._id, + this.file + ) + }) + + it('should update the database', function() { + this.ProjectMock.verify() + }) + }) +}) diff --git a/services/web/test/unit/src/util/promisesTests.js b/services/web/test/unit/src/util/promisesTests.js index f9458d901d..7029007892 100644 --- a/services/web/test/unit/src/util/promisesTests.js +++ b/services/web/test/unit/src/util/promisesTests.js @@ -1,5 +1,8 @@ const { expect } = require('chai') -const { promisifyAll } = require('../../../../app/src/util/promises') +const { + promisifyAll, + callbackifyMultiResult +} = require('../../../../app/src/util/promises') describe('promisifyAll', function() { describe('basic functionality', function() { @@ -57,4 +60,64 @@ describe('promisifyAll', function() { expect(sum).to.equal(101) }) }) + + describe('multiResult option', function() { + before(function() { + this.module = { + asyncAdd(a, b, callback) { + callback(null, a + b) + }, + asyncArithmetic(a, b, callback) { + callback(null, a + b, a * b) + } + } + this.promisified = promisifyAll(this.module, { + multiResult: { asyncArithmetic: ['sum', 'product'] } + }) + }) + + it('promisifies multi-result functions', async function() { + const result = await this.promisified.asyncArithmetic(3, 6) + expect(result).to.deep.equal({ sum: 9, product: 18 }) + }) + + it('promisifies other functions normally', async function() { + const sum = await this.promisified.asyncAdd(6, 1) + expect(sum).to.equal(7) + }) + }) +}) + +describe('callbackifyMultiResult', function() { + it('callbackifies a multi-result function', function(done) { + async function asyncArithmetic(a, b) { + return { sum: a + b, product: a * b } + } + const callbackified = callbackifyMultiResult(asyncArithmetic, [ + 'sum', + 'product' + ]) + callbackified(3, 11, (err, sum, product) => { + if (err != null) { + return done(err) + } + expect(sum).to.equal(14) + expect(product).to.equal(33) + done() + }) + }) + + it('propagates errors', function(done) { + async function asyncBomb() { + throw new Error('BOOM!') + } + const callbackified = callbackifyMultiResult(asyncBomb, [ + 'explosives', + 'dynamite' + ]) + callbackified(err => { + expect(err).to.exist + done() + }) + }) })