From 27941dc8af2bb94e3e2149b089cafe51c709de0a Mon Sep 17 00:00:00 2001 From: Eric Mc Sween Date: Thu, 23 Apr 2020 07:51:06 -0400 Subject: [PATCH] Merge pull request #2760 from overleaf/em-faster-uploads Make a single Mongo update when uploading projects GitOrigin-RevId: de102d3e112c9014ca5885f963e35971e4db6cee --- .../Features/FileStore/FileStoreHandler.js | 6 + .../Project/FolderStructureBuilder.js | 30 +- .../ProjectEntityMongoUpdateHandler.js | 32 +- .../Uploads/FileSystemImportManager.js | 202 ++++++--- .../Features/Uploads/ProjectUploadManager.js | 151 +++++-- services/web/package-lock.json | 230 +++++----- services/web/package.json | 1 + .../acceptance/src/ProjectStructureTests.js | 4 +- .../Project/FolderStructureBuilderTests.js | 18 +- .../ProjectEntityMongoUpdateHandlerTests.js | 9 +- .../Uploads/FileSystemImportManagerTests.js | 412 +++++++++--------- .../src/Uploads/ProjectUploadManagerTests.js | 290 +++++++----- 12 files changed, 804 insertions(+), 581 deletions(-) diff --git a/services/web/app/src/Features/FileStore/FileStoreHandler.js b/services/web/app/src/Features/FileStore/FileStoreHandler.js index 45ffa90da7..7b999f91d9 100644 --- a/services/web/app/src/Features/FileStore/FileStoreHandler.js +++ b/services/web/app/src/Features/FileStore/FileStoreHandler.js @@ -7,6 +7,7 @@ const Async = require('async') const FileHashManager = require('./FileHashManager') const { File } = require('../../models/File') const Errors = require('../Errors/Errors') +const { promisifyAll } = require('../../util/promises') const ONE_MIN_IN_MS = 60 * 1000 const FIVE_MINS_IN_MS = ONE_MIN_IN_MS * 5 @@ -227,3 +228,8 @@ const FileStoreHandler = { } module.exports = FileStoreHandler +module.exports.promises = promisifyAll(FileStoreHandler, { + multiResult: { + uploadFileFromDisk: ['url', 'fileRef'] + } +}) diff --git a/services/web/app/src/Features/Project/FolderStructureBuilder.js b/services/web/app/src/Features/Project/FolderStructureBuilder.js index 72513478ba..e5ac70da13 100644 --- a/services/web/app/src/Features/Project/FolderStructureBuilder.js +++ b/services/web/app/src/Features/Project/FolderStructureBuilder.js @@ -4,13 +4,13 @@ const { ObjectId } = require('mongodb') module.exports = { buildFolderStructure } -function buildFolderStructure(docUploads, fileUploads) { +function buildFolderStructure(docEntries, fileEntries) { const builder = new FolderStructureBuilder() - for (const docUpload of docUploads) { - builder.addDocUpload(docUpload) + for (const docEntry of docEntries) { + builder.addDocEntry(docEntry) } - for (const fileUpload of fileUploads) { - builder.addFileUpload(fileUpload) + for (const fileEntry of fileEntries) { + builder.addFileEntry(fileEntry) } return builder.rootFolder } @@ -24,18 +24,18 @@ class FolderStructureBuilder { this.entityPaths.add('/') } - addDocUpload(docUpload) { - this.recordEntityPath(Path.join(docUpload.dirname, docUpload.doc.name)) - const folder = this.mkdirp(docUpload.dirname) - folder.docs.push(docUpload.doc) + addDocEntry(docEntry) { + this.recordEntityPath(docEntry.path) + const folderPath = Path.dirname(docEntry.path) + const folder = this.mkdirp(folderPath) + folder.docs.push(docEntry.doc) } - addFileUpload(fileUpload) { - this.recordEntityPath( - Path.join(fileUpload.dirname, fileUpload.fileRef.name) - ) - const folder = this.mkdirp(fileUpload.dirname) - folder.fileRefs.push(fileUpload.fileRef) + addFileEntry(fileEntry) { + this.recordEntityPath(fileEntry.path) + const folderPath = Path.dirname(fileEntry.path) + const folder = this.mkdirp(folderPath) + folder.fileRefs.push(fileEntry.file) } mkdirp(path) { diff --git a/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js b/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js index cf059d6e8d..784a28bde0 100644 --- a/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js +++ b/services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js @@ -1,8 +1,3 @@ -/* NOTE: this file is an async/await version of - * ProjectEntityMongoUpdateHandler.js. It's temporarily separate from the - * callback-style version so that we can test it in production for some code - * paths only. - */ const { callbackify } = require('util') const { callbackifyMultiResult } = require('../../util/promises') const _ = require('underscore') @@ -635,13 +630,24 @@ async function _checkValidMove( } } -async function createNewFolderStructure(projectId, docUploads, fileUploads) { +/** + * Create an initial file tree out of a list of doc and file entries + * + * Each entry specifies a path to the doc or file. Folders are automatically + * created. + * + * @param {ObjectId} projectId - id of the project + * @param {DocEntry[]} docEntries - list of docs to add + * @param {FileEntry[]} fileEntries - list of files to add + * @return {Promise} the project version after the operation + */ +async function createNewFolderStructure(projectId, docEntries, fileEntries) { try { const rootFolder = FolderStructureBuilder.buildFolderStructure( - docUploads, - fileUploads + docEntries, + fileEntries ) - const result = await Project.updateOne( + const project = await Project.findOneAndUpdate( { _id: projectId, 'rootFolder.0.folders.0': { $exists: false }, @@ -651,14 +657,20 @@ async function createNewFolderStructure(projectId, docUploads, fileUploads) { { $set: { rootFolder: [rootFolder] }, $inc: { version: 1 } + }, + { + new: true, + lean: true, + fields: { version: 1 } } ).exec() - if (result.n !== 1) { + if (project == null) { throw new OError({ message: 'project not found or folder structure already exists', info: { projectId } }) } + return project.version } catch (err) { throw new OError({ message: 'failed to create folder structure', diff --git a/services/web/app/src/Features/Uploads/FileSystemImportManager.js b/services/web/app/src/Features/Uploads/FileSystemImportManager.js index d60bb3422e..26389f034c 100644 --- a/services/web/app/src/Features/Uploads/FileSystemImportManager.js +++ b/services/web/app/src/Features/Uploads/FileSystemImportManager.js @@ -1,37 +1,22 @@ const fs = require('fs') +const Path = require('path') const { callbackify } = require('util') -const FileTypeManager = require('./FileTypeManager') const EditorController = require('../Editor/EditorController') +const Errors = require('../Errors/Errors') +const FileTypeManager = require('./FileTypeManager') +const SafePath = require('../Project/SafePath') const logger = require('logger-sharelatex') module.exports = { - addFolderContents: callbackify(addFolderContents), addEntity: callbackify(addEntity), + importDir: callbackify(importDir), promises: { - addFolderContents, - addEntity + addEntity, + importDir } } -async function addDoc( - userId, - projectId, - folderId, - name, - path, - encoding, - replace -) { - if (!(await _isSafeOnFileSystem(path))) { - logger.log( - { userId, projectId, folderId, name, path }, - 'add doc is from symlink, stopping process' - ) - throw new Error('path is symlink') - } - let content = await fs.promises.readFile(path, encoding) - content = content.replace(/\r\n?/g, '\n') // convert Windows line endings to unix. very old macs also created \r-separated lines - const lines = content.split('\n') +async function addDoc(userId, projectId, folderId, name, lines, replace) { if (replace) { const doc = await EditorController.promises.upsertDoc( projectId, @@ -56,14 +41,6 @@ async function addDoc( } async function addFile(userId, projectId, folderId, name, path, replace) { - if (!(await _isSafeOnFileSystem(path))) { - logger.log( - { userId, projectId, folderId, name, path }, - 'add file is from symlink, stopping insert' - ) - throw new Error('path is symlink') - } - if (replace) { const file = await EditorController.promises.upsertFile( projectId, @@ -90,13 +67,6 @@ async function addFile(userId, projectId, folderId, name, path, replace) { } async function addFolder(userId, projectId, folderId, name, path, replace) { - if (!(await _isSafeOnFileSystem(path))) { - logger.log( - { userId, projectId, folderId, path }, - 'add folder is from symlink, stopping insert' - ) - throw new Error('path is symlink') - } const newFolder = await EditorController.promises.addFolder( projectId, folderId, @@ -137,61 +107,149 @@ async function addFolderContents( } } -async function addEntity(userId, projectId, folderId, name, path, replace) { - if (!(await _isSafeOnFileSystem(path))) { +async function addEntity(userId, projectId, folderId, name, fsPath, replace) { + if (!(await _isSafeOnFileSystem(fsPath))) { logger.log( - { userId, projectId, folderId, path }, + { userId, projectId, folderId, fsPath }, 'add entry is from symlink, stopping insert' ) throw new Error('path is symlink') } - if (await FileTypeManager.promises.isDirectory(path)) { + if (await FileTypeManager.promises.isDirectory(fsPath)) { const newFolder = await addFolder( userId, projectId, folderId, name, - path, + fsPath, replace ) return newFolder } - const { binary, encoding } = await FileTypeManager.promises.getType( - name, - path - ) - if (binary) { - const entity = await addFile( - userId, - projectId, - folderId, - name, - path, - replace - ) - if (entity != null) { - entity.type = 'file' + + // Here, we cheat a little bit and provide the project path relative to the + // folder, not the root of the project. This is because we don't know for sure + // at this point what the final path of the folder will be. The project path + // is still important for importFile() to be able to figure out if the file is + // a binary file or an editable document. + const projectPath = Path.join('/', name) + const importInfo = await importFile(fsPath, projectPath) + switch (importInfo.type) { + case 'file': { + const entity = await addFile( + userId, + projectId, + folderId, + name, + importInfo.fsPath, + replace + ) + if (entity != null) { + entity.type = 'file' + } + return entity } - return entity - } else { - const entity = await addDoc( - userId, - projectId, - folderId, - name, - path, - encoding, - replace - ) - if (entity != null) { - entity.type = 'doc' + case 'doc': { + const entity = await addDoc( + userId, + projectId, + folderId, + name, + importInfo.lines, + replace + ) + if (entity != null) { + entity.type = 'doc' + } + return entity + } + default: { + throw new Error(`unknown import type: ${importInfo.type}`) } - return entity } } async function _isSafeOnFileSystem(path) { + // Use lstat() to ensure we don't follow symlinks. Symlinks from an + // untrusted source are dangerous. const stat = await fs.promises.lstat(path) return stat.isFile() || stat.isDirectory() } + +async function importFile(fsPath, projectPath) { + const stat = await fs.promises.lstat(fsPath) + if (!stat.isFile()) { + throw new Error(`can't import ${fsPath}: not a regular file`) + } + _validateProjectPath(projectPath) + const filename = Path.basename(projectPath) + + const { binary, encoding } = await FileTypeManager.promises.getType( + filename, + fsPath + ) + if (binary) { + return new FileImport(projectPath, fsPath) + } else { + const content = await fs.promises.readFile(fsPath, encoding) + // Handle Unix, DOS and classic Mac newlines + const lines = content.split(/\r\n|\n|\r/) + return new DocImport(projectPath, lines) + } +} + +async function importDir(dirPath) { + const stat = await fs.promises.lstat(dirPath) + if (!stat.isDirectory()) { + throw new Error(`can't import ${dirPath}: not a directory`) + } + const entries = [] + for await (const filePath of _walkDir(dirPath)) { + const projectPath = Path.join('/', Path.relative(dirPath, filePath)) + const importInfo = await importFile(filePath, projectPath) + entries.push(importInfo) + } + return entries +} + +function _validateProjectPath(path) { + if (!SafePath.isAllowedLength(path) || !SafePath.isCleanPath(path)) { + throw new Errors.InvalidNameError(`Invalid path: ${path}`) + } +} + +async function* _walkDir(dirPath) { + const entries = await fs.promises.readdir(dirPath) + for (const entry of entries) { + const entryPath = Path.join(dirPath, entry) + if (await FileTypeManager.promises.shouldIgnore(entryPath)) { + continue + } + + // Use lstat() to ensure we don't follow symlinks. Symlinks from an + // untrusted source are dangerous. + const stat = await fs.promises.lstat(entryPath) + if (stat.isFile()) { + yield entryPath + } else if (stat.isDirectory()) { + yield* _walkDir(entryPath) + } + } +} + +class FileImport { + constructor(projectPath, fsPath) { + this.type = 'file' + this.projectPath = projectPath + this.fsPath = fsPath + } +} + +class DocImport { + constructor(projectPath, lines) { + this.type = 'doc' + this.projectPath = projectPath + this.lines = lines + } +} diff --git a/services/web/app/src/Features/Uploads/ProjectUploadManager.js b/services/web/app/src/Features/Uploads/ProjectUploadManager.js index 4c2fd6e407..7e18d1ef93 100644 --- a/services/web/app/src/Features/Uploads/ProjectUploadManager.js +++ b/services/web/app/src/Features/Uploads/ProjectUploadManager.js @@ -1,13 +1,19 @@ -const path = require('path') +const Path = require('path') const fs = require('fs-extra') const { callbackify } = require('util') const ArchiveManager = require('./ArchiveManager') +const { Doc } = require('../../models/Doc') +const DocstoreManager = require('../Docstore/DocstoreManager') +const DocumentHelper = require('../Documents/DocumentHelper') +const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler') +const FileStoreHandler = require('../FileStore/FileStoreHandler') const FileSystemImportManager = require('./FileSystemImportManager') const ProjectCreationHandler = require('../Project/ProjectCreationHandler') +const ProjectEntityMongoUpdateHandler = require('../Project/ProjectEntityMongoUpdateHandler') const ProjectRootDocManager = require('../Project/ProjectRootDocManager') const ProjectDetailsHandler = require('../Project/ProjectDetailsHandler') const ProjectDeleter = require('../Project/ProjectDeleter') -const DocumentHelper = require('../Documents/DocumentHelper') +const TpdsProjectFlusher = require('../ThirdPartyDataStore/TpdsProjectFlusher') const logger = require('logger-sharelatex') module.exports = { @@ -22,12 +28,12 @@ module.exports = { } async function createProjectFromZipArchive(ownerId, defaultName, zipPath) { - const extractionPath = await _extractZip(zipPath) + const contentsPath = await _extractZip(zipPath) const { path, content } = await ProjectRootDocManager.promises.findRootDocFileFromDirectory( - extractionPath + contentsPath ) const projectName = @@ -38,12 +44,7 @@ async function createProjectFromZipArchive(ownerId, defaultName, zipPath) { uniqueName ) try { - await _insertZipContentsIntoFolder( - ownerId, - project._id, - project.rootFolder[0]._id, - extractionPath - ) + await _initializeProjectWithZipContents(ownerId, project, contentsPath) if (path) { await ProjectRootDocManager.promises.setRootDocFromName(project._id, path) @@ -60,6 +61,7 @@ async function createProjectFromZipArchive(ownerId, defaultName, zipPath) { ) throw err } + await fs.remove(contentsPath) return project } @@ -69,7 +71,7 @@ async function createProjectFromZipArchiveWithName( zipPath, attributes = {} ) { - const extractionPath = await _extractZip(zipPath) + const contentsPath = await _extractZip(zipPath) const uniqueName = await _generateUniqueName(ownerId, proposedName) const project = await ProjectCreationHandler.promises.createBlankProject( ownerId, @@ -78,12 +80,7 @@ async function createProjectFromZipArchiveWithName( ) try { - await _insertZipContentsIntoFolder( - ownerId, - project._id, - project.rootFolder[0]._id, - extractionPath - ) + await _initializeProjectWithZipContents(ownerId, project, contentsPath) await ProjectRootDocManager.promises.setRootDocAutomatically(project._id) } catch (err) { // no need to wait for the cleanup here @@ -97,33 +94,14 @@ async function createProjectFromZipArchiveWithName( ) throw err } - + await fs.remove(contentsPath) return project } -async function _insertZipContentsIntoFolder( - ownerId, - projectId, - folderId, - destination -) { - const topLevelDestination = await ArchiveManager.promises.findTopLevelDirectory( - destination - ) - await FileSystemImportManager.promises.addFolderContents( - ownerId, - projectId, - folderId, - topLevelDestination, - false - ) - await fs.remove(destination) -} - async function _extractZip(zipPath) { - const destination = path.join( - path.dirname(zipPath), - `${path.basename(zipPath, '.zip')}-${Date.now()}` + const destination = Path.join( + Path.dirname(zipPath), + `${Path.basename(zipPath, '.zip')}-${Date.now()}` ) await ArchiveManager.promises.extractZipArchive(zipPath, destination) return destination @@ -137,3 +115,96 @@ async function _generateUniqueName(ownerId, originalName) { ) return uniqueName } + +async function _initializeProjectWithZipContents( + ownerId, + project, + contentsPath +) { + const topLevelDir = await ArchiveManager.promises.findTopLevelDirectory( + contentsPath + ) + const importEntries = await FileSystemImportManager.promises.importDir( + topLevelDir + ) + const { fileEntries, docEntries } = await _createEntriesFromImports( + project._id, + importEntries + ) + const projectVersion = await ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure( + project._id, + docEntries, + fileEntries + ) + await _notifyDocumentUpdater(project, ownerId, { + newFiles: fileEntries, + newDocs: docEntries, + newProject: { version: projectVersion } + }) + await TpdsProjectFlusher.promises.flushProjectToTpds(project._id) +} + +async function _createEntriesFromImports(projectId, importEntries) { + const fileEntries = [] + const docEntries = [] + for (const importEntry of importEntries) { + switch (importEntry.type) { + case 'doc': { + const docEntry = await _createDoc( + projectId, + importEntry.projectPath, + importEntry.lines + ) + docEntries.push(docEntry) + break + } + case 'file': { + const fileEntry = await _createFile( + projectId, + importEntry.projectPath, + importEntry.fsPath + ) + fileEntries.push(fileEntry) + break + } + default: { + throw new Error(`Invalid import type: ${importEntry.type}`) + } + } + } + return { fileEntries, docEntries } +} + +async function _createDoc(projectId, projectPath, docLines) { + const docName = Path.basename(projectPath) + const doc = new Doc({ name: docName }) + await DocstoreManager.promises.updateDoc( + projectId.toString(), + doc._id.toString(), + docLines, + 0, + {} + ) + return { doc, path: projectPath, docLines: docLines.join('\n') } +} + +async function _createFile(projectId, projectPath, fsPath) { + const fileName = Path.basename(projectPath) + const { fileRef, url } = await FileStoreHandler.promises.uploadFileFromDisk( + projectId, + { name: fileName }, + fsPath + ) + return { file: fileRef, path: projectPath, url } +} + +async function _notifyDocumentUpdater(project, userId, changes) { + const projectHistoryId = + project.overleaf && project.overleaf.history && project.overleaf.history.id + await DocumentUpdaterHandler.promises.updateProjectStructure( + project._id, + projectHistoryId, + userId, + changes + ) +} diff --git a/services/web/package-lock.json b/services/web/package-lock.json index 7fcf00937f..cdd95263a9 100644 --- a/services/web/package-lock.json +++ b/services/web/package-lock.json @@ -7,7 +7,7 @@ "@auth0/thumbprint": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/@auth0/thumbprint/-/thumbprint-0.0.6.tgz", - "integrity": "sha1-yrEGLGwEZizmxZLUgVfsQmiuhRg=", + "integrity": "sha512-+YciWHxNUOE78T+xoXI1fMI6G1WdyyAay8ioaMZhvGOJ+lReYzj0b7mpfNr5WtjGrmtWPvPOOxh0TO+5Y2M/Hw==", "dev": true }, "@babel/cli": { @@ -2597,7 +2597,7 @@ "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==" }, "@protobufjs/base64": { "version": "1.1.2", @@ -2612,12 +2612,12 @@ "@protobufjs/eventemitter": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==" }, "@protobufjs/fetch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", "requires": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" @@ -2626,27 +2626,27 @@ "@protobufjs/float": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==" }, "@protobufjs/inquire": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==" }, "@protobufjs/path": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==" }, "@protobufjs/pool": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==" }, "@protobufjs/utf8": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@sentry/browser": { "version": "5.15.4", @@ -3613,7 +3613,7 @@ "ansi-html": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz", - "integrity": "sha1-gTWEAhliqenm/QOflA0S9WynhZ4=", + "integrity": "sha512-JoAxEa1DfP9m2xfB/y2r/aKcwXNlltr4+0QSBC4TrLfcxyvepX2Pv0t/xpgGV5bGsDzCYV8SzjWgyCW0T9yYbA==", "dev": true }, "ansi-regex": { @@ -3908,7 +3908,7 @@ "aria-query": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-3.0.0.tgz", - "integrity": "sha1-ZbP8wcoRVajJrmTW7uKX8V1RM8w=", + "integrity": "sha512-majUxHgLehQTeSA+hClx+DY09OVUqG3GtezWkF1krgLGNdlDu9l9V8DaqNMWbq4Eddc8wsyDA0hpDUtnYxQEXw==", "dev": true, "requires": { "ast-types-flow": "0.0.7", @@ -4294,7 +4294,7 @@ "ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", - "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=", + "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==", "dev": true }, "async": { @@ -4989,7 +4989,7 @@ "batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", - "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", "dev": true }, "bcrypt": { @@ -5330,7 +5330,7 @@ "bintrees": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz", - "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ=" + "integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g==" }, "bitsyntax": { "version": "0.0.4", @@ -5470,7 +5470,7 @@ "bonjour": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz", - "integrity": "sha1-jokKGD2O6aI5OzhExpGkK897yfU=", + "integrity": "sha512-RaVTblr+OnEli0r/ud8InrU7D+G0y6aJhlxaLa6Pwty4+xoxboF1BsUI45tujvRpbj9dQVoglChqonGAsjEBYg==", "dev": true, "requires": { "array-flatten": "^2.1.0", @@ -5644,7 +5644,7 @@ "brorand": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", - "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", + "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==", "dev": true }, "browser-stdout": { @@ -5700,7 +5700,7 @@ "browserify-rsa": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", - "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", + "integrity": "sha512-+YpEyaLDDvvdzIxQ+cCx73r5YEhS3ANGOkiHdyWqW4t3gdeoNEYjSiQwntbU4Uo2/9yRkpYX3SRFeH+7jc2Duw==", "dev": true, "requires": { "bn.js": "^4.1.0", @@ -5710,7 +5710,7 @@ "browserify-sign": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.0.4.tgz", - "integrity": "sha1-qk62jl17ZYuqa/alfmMMvXqT0pg=", + "integrity": "sha512-D2ItxCwNtLcHRrOCuEDZQlIezlFyUV/N5IYz6TY1svu1noyThFuthoEjzT8ChZe3UEctqnwmykcPhet3Eiz58A==", "dev": true, "requires": { "bn.js": "^4.1.1", @@ -5819,7 +5819,7 @@ "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", - "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=", + "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==", "dev": true }, "bufferedstream": { @@ -5850,7 +5850,7 @@ "builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=", + "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", "dev": true }, "bunyan": { @@ -6499,7 +6499,7 @@ "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", "dev": true }, "component-bind": { @@ -6606,7 +6606,7 @@ "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", "dev": true } } @@ -6786,13 +6786,13 @@ "constants-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", - "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=", + "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==", "dev": true }, "contains-path": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", - "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", + "integrity": "sha512-OKZnPGeMQy2RPaUIBPFFd71iNf4791H12MCRuVQDnzGRwCYNYmTDy5pdafo2SLAcEMKzTOQnLWG4QdcjeJUMEg==", "dev": true }, "content-disposition": { @@ -8106,7 +8106,7 @@ "detect-file": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", - "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=", + "integrity": "sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q==", "dev": true }, "detect-node": { @@ -8193,7 +8193,7 @@ "dns-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz", - "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0=", + "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==", "dev": true }, "dns-packet": { @@ -8214,7 +8214,7 @@ "dns-txt": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz", - "integrity": "sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=", + "integrity": "sha512-Ix5PrWjphuSoUXV/Zv5gaFHjnaJtb02F2+Si3Ht9dyJ87+Z/lMmy+dpNHtTGraNK958ndXq2i+GLkWsWHcKaBQ==", "dev": true, "requires": { "buffer-indexof": "^1.0.0" @@ -9727,7 +9727,7 @@ "expand-tilde": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", - "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=", + "integrity": "sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==", "dev": true, "requires": { "homedir-polyfill": "^1.0.1" @@ -10062,7 +10062,7 @@ "faye-websocket": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.10.0.tgz", - "integrity": "sha1-TkkvjQTftviQA1B/btvy1QHnxvQ=", + "integrity": "sha512-Xhj93RXbMSq8urNCUq4p9l0P6hnySJ/7YNRhYNug0bLOuii7pKO7xQFb5mx9xZXWCar88pLPb805PvUkwrLZpQ==", "dev": true, "requires": { "websocket-driver": ">=0.5.1" @@ -10366,7 +10366,7 @@ "findit2": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz", - "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY=" + "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog==" }, "findup-sync": { "version": "0.1.3", @@ -10432,7 +10432,7 @@ "flowstate": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/flowstate/-/flowstate-0.4.1.tgz", - "integrity": "sha1-tfu4t/wte9xbVL5GyYMJ73NvTsA=", + "integrity": "sha512-U67AgveyMwXFIiDgs6Yz/PrUNrZGLJUUMDwJ9Q0fDFTQSzyDg8Jj9YDyZIUnFZKggQZONVueK9+grp/Gxa/scw==", "dev": true, "requires": { "clone": "^1.0.2", @@ -10453,7 +10453,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -10574,7 +10574,7 @@ "from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", - "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", "dev": true, "requires": { "inherits": "^2.0.1", @@ -10584,7 +10584,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -10660,7 +10660,7 @@ "fs-write-stream-atomic": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", - "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=", + "integrity": "sha512-gehEzmPn2nAwr39eay+x3X34Ra+M2QlVUTLhkXPjWdeO8RF9kszk116avgBJM3ZyNHgHXBNx+VmPaFC36k0PzA==", "dev": true, "requires": { "graceful-fs": "^4.1.2", @@ -11504,7 +11504,7 @@ "global-prefix": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz", - "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=", + "integrity": "sha512-5lsx1NUDHtSjfg0eHlmYvZKv8/nVqX4ckFbM+FrGcQ+04KWcWFo9P5MxPZYSzUvyzmdTbI7Eix8Q4IbELDqzKg==", "dev": true, "requires": { "expand-tilde": "^2.0.2", @@ -12136,7 +12136,7 @@ "hash-base": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz", - "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=", + "integrity": "sha512-EeeoJKjTyt868liAlVmcv2ZsUfGHlE3Q+BICOXcZiwN3osr5Q/zFGYmTJpoIzuaSTAwndFy+GqhEwlU4L3j4Ow==", "dev": true, "requires": { "inherits": "^2.0.1", @@ -12233,7 +12233,7 @@ "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", - "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", "dev": true, "requires": { "hash.js": "^1.0.3", @@ -12264,7 +12264,7 @@ "hpack.js": { "version": "2.1.6", "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz", - "integrity": "sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI=", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", "dev": true, "requires": { "inherits": "^2.0.1", @@ -12276,7 +12276,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -12394,7 +12394,7 @@ "http-deceiver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz", - "integrity": "sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc=", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", "dev": true }, "http-errors": { @@ -12418,7 +12418,7 @@ "http-parser-js": { "version": "0.4.10", "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.10.tgz", - "integrity": "sha1-ksnBN0w1CF912zWexWzCV8u5P6Q=", + "integrity": "sha512-ln7+HeZl3lL3PNRX9Y6ub4i8xcgQ0mO2J//ic97dR7tEXB+6IKAjx8JCCmEkwKiMcR2jidU9xNolz1fEyyf/Jg==", "dev": true }, "http-proxy": { @@ -12469,13 +12469,13 @@ "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", - "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=", + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", "dev": true }, "array-unique": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", - "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", "dev": true }, "debug": { @@ -12496,7 +12496,7 @@ "expand-brackets": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", - "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=", + "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", "dev": true, "requires": { "debug": "^2.3.3", @@ -12511,7 +12511,7 @@ "define-property": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", "dev": true, "requires": { "is-descriptor": "^0.1.0" @@ -12520,7 +12520,7 @@ "extend-shallow": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", "dev": true, "requires": { "is-extendable": "^0.1.0" @@ -12529,7 +12529,7 @@ "is-accessor-descriptor": { "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", - "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", + "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", "dev": true, "requires": { "kind-of": "^3.0.2" @@ -12538,7 +12538,7 @@ "kind-of": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", "dev": true, "requires": { "is-buffer": "^1.1.5" @@ -12549,7 +12549,7 @@ "is-data-descriptor": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", - "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", + "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", "dev": true, "requires": { "kind-of": "^3.0.2" @@ -12558,7 +12558,7 @@ "kind-of": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", "dev": true, "requires": { "is-buffer": "^1.1.5" @@ -12604,7 +12604,7 @@ "define-property": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", "dev": true, "requires": { "is-descriptor": "^1.0.0" @@ -12613,7 +12613,7 @@ "extend-shallow": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", "dev": true, "requires": { "is-extendable": "^0.1.0" @@ -12736,7 +12736,7 @@ "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", - "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", + "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==", "dev": true }, "https-proxy-agent": { @@ -12810,7 +12810,7 @@ "iferr": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz", - "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=", + "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==", "dev": true }, "ignore": { @@ -13261,7 +13261,7 @@ "ip-regex": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", - "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=", + "integrity": "sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==", "dev": true }, "ipaddr.js": { @@ -13643,7 +13643,7 @@ "is-wsl": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz", - "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=", + "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==", "dev": true }, "isarray": { @@ -13722,7 +13722,7 @@ "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", - "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + "integrity": "sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w==" }, "joi-mongodb-objectid": { "version": "0.1.0", @@ -13790,7 +13790,7 @@ "json-bigint": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", - "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "integrity": "sha512-u+c/u/F+JNPUekHCFyGVycRPyh9UHD5iUhSyIAn10kxbDTJxijwAbT6XHaONEOXuGGfmWUSroheXgHcml4gLgg==", "requires": { "bignumber.js": "^7.0.0" } @@ -14757,7 +14757,7 @@ "load-json-file": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", - "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "integrity": "sha512-3p6ZOGNbiX4CdvEd1VcE6yi78UrGNpjHO33noGwHCnT/o2fyllJDepsm8+mFFv/DvtwFHht5HIHSyOy5a+ChVQ==", "dev": true, "requires": { "graceful-fs": "^4.1.2", @@ -14769,7 +14769,7 @@ "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", "dev": true } } @@ -14914,7 +14914,7 @@ "lodash.pickby": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz", - "integrity": "sha1-feoh2MGNdwOifHBMFdO4SmfjOv8=" + "integrity": "sha512-AZV+GsS/6ckvPOVQPXSiFFacKvKB4kOQu6ynt9wz0F3LO4R9Ij4K1ddYsIytDpSgLz88JHd9P+oaLeej5/Sl7Q==" }, "lodash.set": { "version": "4.3.2", @@ -15556,7 +15556,7 @@ "lynx": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz", - "integrity": "sha1-Mxjc7xaQi4KG6Bisz9sxzXQkj50=", + "integrity": "sha512-JI52N0NwK2b/Md0TFPdPtUBI46kjyJXF7+q08l2yvQ56q6QA8s7ZjZQQRoxFpS2jDXNf/B0p8ID+OIKcTsZwzw==", "requires": { "mersenne": "~0.0.3", "statsd-parser": "~0.0.4" @@ -15833,7 +15833,7 @@ "mersenne": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz", - "integrity": "sha1-QB/ex+whzbngPNPTAhOY2iGycIU=" + "integrity": "sha512-XoSUL+nF8hMTKGQxUs8r3Btdsf1yuKKBdCCGbh3YXgCXuVKishpZv1CNc385w9s8t4Ynwc5h61BwW/FCVulkbg==" }, "messageformat": { "version": "1.1.1", @@ -16093,7 +16093,7 @@ "minimalistic-crypto-utils": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", - "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=", + "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==", "dev": true }, "minimatch": { @@ -16247,7 +16247,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, "requires": { "wrappy": "1" @@ -16407,10 +16407,16 @@ } } }, + "mock-fs": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-4.11.0.tgz", + "integrity": "sha512-Yp4o3/ZA15wsXqJTT+R+9w2AYIkD1i80Lds47wDbuUhOvQvm+O2EfjFZSz0pMgZZSPHRhGxgcd2+GL4+jZMtdw==", + "dev": true + }, "module-details-from-path": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz", - "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=" + "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==" }, "moment": { "version": "2.24.0", @@ -16572,7 +16578,7 @@ "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", - "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=", + "integrity": "sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==", "dev": true, "requires": { "aproba": "^1.1.1", @@ -16703,7 +16709,7 @@ "multicast-dns-service-types": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz", - "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=", + "integrity": "sha512-cnAsSVxIDsYt0v7HmC0hWZFwwXSh+E6PgCrREDuN/EsjgLwA5XRmlMHhSiDPrt6HxY1gTivEa/Zh7GtODoLevQ==", "dev": true }, "muri": { @@ -16990,7 +16996,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -17045,7 +17051,7 @@ "url": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", - "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "integrity": "sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==", "dev": true, "requires": { "punycode": "1.3.2", @@ -17055,7 +17061,7 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", "dev": true } } @@ -18541,7 +18547,7 @@ "os-browserify": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=", + "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==", "dev": true }, "os-homedir": { @@ -18587,7 +18593,7 @@ "p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", - "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "integrity": "sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==", "dev": true }, "p-each-series": { @@ -18852,7 +18858,7 @@ "parse-json": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", - "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==", "dev": true, "requires": { "error-ex": "^1.2.0" @@ -18871,7 +18877,7 @@ "parse-passwd": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", - "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=", + "integrity": "sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==", "dev": true }, "parse5": { @@ -19096,7 +19102,7 @@ "path-type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", - "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "integrity": "sha512-dUnb5dXUf+kzhC/W/F4e5/SkluXIFf5VUHolW1Eg1irn1hGWjPGdsRcvYJ1nD6lhk8Ir7VM0bHJKsYTx8Jx9OQ==", "dev": true, "requires": { "pify": "^2.0.0" @@ -19105,7 +19111,7 @@ "pify": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", "dev": true } } @@ -20174,7 +20180,7 @@ "promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", - "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=", + "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", "dev": true }, "promisify-any": { @@ -20863,7 +20869,7 @@ "read-pkg": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", - "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "integrity": "sha512-eFIBOPW7FGjzBuk3hdXEuNSiTZS/xEMlH49HxMyzb0hyPfu4EhVjT2DH32K1hSSmVq4sebAWnZuuY5auISUTGA==", "dev": true, "requires": { "load-json-file": "^2.0.0", @@ -20874,7 +20880,7 @@ "read-pkg-up": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", - "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "integrity": "sha512-1orxQfbWGUiTn9XsPlChs6rLie/AV9jwZTGmu2NZw/CUDJQchXJFYE0Fq5j7+n558T1JhDWLdhyd1Zj+wLY//w==", "dev": true, "requires": { "find-up": "^2.0.0", @@ -21818,7 +21824,7 @@ "resolve-cwd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz", - "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=", + "integrity": "sha512-ccu8zQTrzVr954472aUVPLEcB3YpKSYR3cg/3lo1okzobPBM+1INXBbBZlDbnI/hbEocnf8j0QVo43hQKrbchg==", "dev": true, "requires": { "resolve-from": "^3.0.0" @@ -21827,7 +21833,7 @@ "resolve-from": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz", - "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=", + "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==", "dev": true } } @@ -21835,7 +21841,7 @@ "resolve-dir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz", - "integrity": "sha1-eaQGRMNivoLybv/nOcm7U4IEb0M=", + "integrity": "sha512-R7uiTjECzvOsWSfdM0QKFNBVFcK27aHOUwdvK53BcW8zqnGdYp0Fbj82cy54+2A4P2tFM22J5kRfe1R+lM/1yg==", "dev": true, "requires": { "expand-tilde": "^2.0.0", @@ -21883,7 +21889,7 @@ "retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", "dev": true }, "retry-axios": { @@ -22013,7 +22019,7 @@ "run-queue": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", - "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=", + "integrity": "sha512-ntymy489o0/QQplUDnpYAYUsO50K9SBrIVaKCWDOJzYJts0f9WH9RFJkyagebkw5+y1oi00R7ynNW/d12GBumg==", "dev": true, "requires": { "aproba": "^1.1.1" @@ -22070,7 +22076,7 @@ "saml": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/saml/-/saml-0.12.5.tgz", - "integrity": "sha1-pDyQhUifefceg94uxato3sxrjME=", + "integrity": "sha512-723DD6x293D01zvQP4D6Otu207VZXF1t6t15MCvR3SM5vj+DycuoO5mePnD1VexUjbgVeycCiwwK6PQzfSKVnA==", "dev": true, "requires": { "async": "~0.2.9", @@ -22086,19 +22092,19 @@ "async": { "version": "0.2.10", "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", - "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=", + "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==", "dev": true }, "moment": { "version": "2.15.2", "resolved": "https://registry.npmjs.org/moment/-/moment-2.15.2.tgz", - "integrity": "sha1-G/3t9qbjRfMi/pVtXfW9CKjOhNw=", + "integrity": "sha512-dv9NAmbJRSckFY2Dt3EcgoUGg85U4AaUvtJQ56k0QFumwqpOK3Huf0pYutSVgCFfN+DekvF4pW45PP9rf6ts7g==", "dev": true }, "xml-crypto": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-0.10.1.tgz", - "integrity": "sha1-+DL3TM9W8kr8rhFjofyrRNlndKg=", + "integrity": "sha512-w64qUhByslUJ9D9nwfCyRUCXfVWA5WdzHevHT3BwAig2KOsDNYcuvE2soGUGUs0qp9cy+vGG6B/Ap8qCXPLN/g==", "dev": true, "requires": { "xmldom": "=0.1.19", @@ -22108,7 +22114,7 @@ "xmldom": { "version": "0.1.19", "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.19.tgz", - "integrity": "sha1-Yx/Ad3bv2EEYvyUXGzftTQdaCrw=", + "integrity": "sha512-pDyxjQSFQgNHkU+yjvoF+GXVGJU7e9EnOg/KcGMDihBIKjTsOeDYaECwC/O9bsUWKY+Sd9izfE43JXC46EOHKA==", "dev": true } } @@ -22116,13 +22122,13 @@ "xmldom": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.15.tgz", - "integrity": "sha1-swSAYvG91S7cQhQkRZ8G3O6y+U0=", + "integrity": "sha512-ssWmE9kBZudhl4iPLmXqaShPuASNKIQIikBzsloOjZqMyfbuQRn/ggz0k9NDa9YFI3+oFvp8t7TsqwmZLTvpoA==", "dev": true }, "xpath": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.5.tgz", - "integrity": "sha1-RUA29u8PPfWvXUukoRn7dWdLPmw=", + "integrity": "sha512-Y1Oyy8lyIDwWpmKIWBF0RZrQOP1fzE12G0ekSB1yzKPtbAdCI5sBCqBU/CAZUkKk81OXuq9tul/5lyNS+22iKg==", "dev": true } } @@ -22153,7 +22159,7 @@ "xml-crypto": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-0.10.1.tgz", - "integrity": "sha1-+DL3TM9W8kr8rhFjofyrRNlndKg=", + "integrity": "sha512-w64qUhByslUJ9D9nwfCyRUCXfVWA5WdzHevHT3BwAig2KOsDNYcuvE2soGUGUs0qp9cy+vGG6B/Ap8qCXPLN/g==", "dev": true, "requires": { "xmldom": "=0.1.19", @@ -22176,13 +22182,13 @@ "xpath": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.5.tgz", - "integrity": "sha1-RUA29u8PPfWvXUukoRn7dWdLPmw=", + "integrity": "sha512-Y1Oyy8lyIDwWpmKIWBF0RZrQOP1fzE12G0ekSB1yzKPtbAdCI5sBCqBU/CAZUkKk81OXuq9tul/5lyNS+22iKg==", "dev": true }, "xtend": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/xtend/-/xtend-1.0.3.tgz", - "integrity": "sha1-P12Tc1PM7Y4IU5mlY/2yJUHClgo=", + "integrity": "sha512-wv78b3q8kHDveC/C7Yq/UUrJXsAAM1t/j5m28h/ZlqYy0+eqByglhsWR88D2j3VImQzZlNIDsSbZ3QItwgWEGw==", "dev": true } } @@ -22327,7 +22333,7 @@ "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz", - "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", "dev": true }, "selfsigned": { @@ -22444,7 +22450,7 @@ "serve-index": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz", - "integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", "dev": true, "requires": { "accepts": "~1.3.4", @@ -22489,7 +22495,7 @@ "escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "dev": true }, "mime-db": { @@ -23408,7 +23414,7 @@ "statsd-parser": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz", - "integrity": "sha1-y9JDlTzELv/VSLXSI4jtaJ7GOb0=" + "integrity": "sha512-7XO+ur89EalMXXFQaydsczB8sclr5nDsNIoUu0IzJx1pIbHUhO3LtpSzBwetIuU9DyTLMiVaJBMtWS/Nb2KR4g==" }, "statuses": { "version": "1.4.0", @@ -23433,7 +23439,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -23499,7 +23505,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "dev": true, "requires": { "wrappy": "1" @@ -23531,7 +23537,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -24086,7 +24092,7 @@ "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", "dev": true }, "strip-eof": { @@ -24345,7 +24351,7 @@ "tdigest": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz", - "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=", + "integrity": "sha512-CXcDY/NIgIbKZPx5H4JJNpq6JwJhU5Z4+yWj4ZghDc7/9nVajiRlPPyMXRePPPlBfcayUqtoCXjo7/Hm82ecUA==", "requires": { "bintrees": "1.0.1" } @@ -24689,7 +24695,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "dev": true }, "process-nextick-args": { @@ -24788,7 +24794,7 @@ "to-arraybuffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", - "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=", + "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==", "dev": true }, "to-fast-properties": { @@ -25060,7 +25066,7 @@ "tty-browserify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", - "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", + "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==", "dev": true }, "tunnel-agent": { @@ -25508,7 +25514,7 @@ "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -25517,7 +25523,7 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" } } }, @@ -25619,7 +25625,7 @@ "utils-flatten": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/utils-flatten/-/utils-flatten-1.0.0.tgz", - "integrity": "sha1-AfMNMZO+RkxAsxdV5nQNDbDO8kM=", + "integrity": "sha512-s21PUgUZ+XPvH8Wi8aj2FEqzZWeNEdemP7LB4u8u5wTDRO4xB+7czAYd3FY2O2rnu89U//khR0Ce8ka3//6M0w==", "dev": true }, "utils-merge": { @@ -27924,7 +27930,7 @@ "xml-name-validator": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-2.0.1.tgz", - "integrity": "sha1-TYuPHszTQZqjYgYb7O9RXh5VljU=", + "integrity": "sha512-jRKe/iQYMyVJpzPH+3HL97Lgu5HrCfii+qSo+TfjKHtOnvbnvdVfMYrn9Q34YV81M2e5sviJlI6Ko9y+nByzvA==", "dev": true }, "xml2js": { diff --git a/services/web/package.json b/services/web/package.json index 0549c11a24..5840236b54 100644 --- a/services/web/package.json +++ b/services/web/package.json @@ -169,6 +169,7 @@ "less-plugin-autoprefix": "^2.0.0", "mini-css-extract-plugin": "^0.8.0", "mkdirp": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "mock-fs": "^4.11.0", "nodemon": "^1.14.3", "optimize-css-assets-webpack-plugin": "^5.0.3", "postcss-loader": "^3.0.0", diff --git a/services/web/test/acceptance/src/ProjectStructureTests.js b/services/web/test/acceptance/src/ProjectStructureTests.js index 2ef62558a4..7a0b5ac8f0 100644 --- a/services/web/test/acceptance/src/ProjectStructureTests.js +++ b/services/web/test/acceptance/src/ProjectStructureTests.js @@ -424,7 +424,7 @@ describe('ProjectStructureChanges', function() { expect(update.userId).to.equal(owner._id) expect(update.pathname).to.equal('/main.tex') expect(update.docLines).to.equal('Test') - expect(version).to.equal(2) + expect(version).to.equal(1) }) it('should version the files created', function() { @@ -437,7 +437,7 @@ describe('ProjectStructureChanges', function() { expect(update.userId).to.equal(owner._id) expect(update.pathname).to.equal('/1pixel.png') expect(update.url).to.be.a('string') - expect(version).to.equal(2) + expect(version).to.equal(1) }) }) diff --git a/services/web/test/unit/src/Project/FolderStructureBuilderTests.js b/services/web/test/unit/src/Project/FolderStructureBuilderTests.js index c54ba619dd..d44aa4525d 100644 --- a/services/web/test/unit/src/Project/FolderStructureBuilderTests.js +++ b/services/web/test/unit/src/Project/FolderStructureBuilderTests.js @@ -36,18 +36,18 @@ describe('FolderStructureBuilder', function() { describe('when given documents and files', function() { beforeEach(function() { const docUploads = [ - { dirname: '/', doc: { _id: 'doc-1', name: 'main.tex' } }, - { dirname: '/foo', doc: { _id: 'doc-2', name: 'other.tex' } }, - { dirname: '/foo', doc: { _id: 'doc-3', name: 'other.bib' } }, + { path: '/main.tex', doc: { _id: 'doc-1', name: 'main.tex' } }, + { path: '/foo/other.tex', doc: { _id: 'doc-2', name: 'other.tex' } }, + { path: '/foo/other.bib', doc: { _id: 'doc-3', name: 'other.bib' } }, { - dirname: '/foo/foo1/foo2', + path: '/foo/foo1/foo2/another.tex', doc: { _id: 'doc-4', name: 'another.tex' } } ] const fileUploads = [ - { dirname: '/', fileRef: { _id: 'file-1', name: 'aaa.jpg' } }, - { dirname: '/foo', fileRef: { _id: 'file-2', name: 'bbb.jpg' } }, - { dirname: '/bar', fileRef: { _id: 'file-3', name: 'ccc.jpg' } } + { path: '/aaa.jpg', file: { _id: 'file-1', name: 'aaa.jpg' } }, + { path: '/foo/bbb.jpg', file: { _id: 'file-2', name: 'bbb.jpg' } }, + { path: '/bar/ccc.jpg', file: { _id: 'file-3', name: 'ccc.jpg' } } ] this.result = this.FolderStructureBuilder.buildFolderStructure( docUploads, @@ -103,8 +103,8 @@ describe('FolderStructureBuilder', function() { describe('when given duplicate files', function() { it('throws an error', function() { const docUploads = [ - { dirname: '/foo', doc: { _id: 'doc-1', name: 'doc.tex' } }, - { dirname: '/foo', doc: { _id: 'doc-2', name: 'doc.tex' } } + { path: '/foo/doc.tex', doc: { _id: 'doc-1', name: 'doc.tex' } }, + { path: '/foo/doc.tex', doc: { _id: 'doc-2', name: 'doc.tex' } } ] expect(() => this.FolderStructureBuilder.buildFolderStructure(docUploads, []) diff --git a/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js b/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js index 4a67703bc5..b764e88e49 100644 --- a/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js +++ b/services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js @@ -1045,7 +1045,7 @@ describe('ProjectEntityMongoUpdateHandler', function() { this.FolderStructureBuilder.buildFolderStructure .withArgs(this.docUploads, this.fileUploads) .returns(this.mockRootFolder) - this.updateExpectation = this.ProjectMock.expects('updateOne') + this.updateExpectation = this.ProjectMock.expects('findOneAndUpdate') .withArgs( { _id: this.project._id, @@ -1053,14 +1053,15 @@ describe('ProjectEntityMongoUpdateHandler', function() { 'rootFolder.0.docs.0': { $exists: false }, 'rootFolder.0.files.0': { $exists: false } }, - { $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } } + { $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } }, + { new: true, lean: true, fields: { version: 1 } } ) .chain('exec') }) describe('happy path', function() { beforeEach(async function() { - this.updateExpectation.resolves({ n: 1 }) + this.updateExpectation.resolves({ version: 1 }) await this.subject.promises.createNewFolderStructure( this.project._id, this.docUploads, @@ -1075,7 +1076,7 @@ describe('ProjectEntityMongoUpdateHandler', function() { describe("when the update doesn't find a matching document", function() { beforeEach(async function() { - this.updateExpectation.resolves({ n: 0 }) + this.updateExpectation.resolves(null) }) it('throws an error', async function() { diff --git a/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js b/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js index 27b441af63..146e6fd36f 100644 --- a/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js +++ b/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js @@ -1,5 +1,6 @@ const sinon = require('sinon') const { expect } = require('chai') +const mockFs = require('mock-fs') const SandboxedModule = require('sandboxed-module') const { ObjectId } = require('mongodb') @@ -13,50 +14,6 @@ describe('FileSystemImportManager', function() { this.newFolderId = new ObjectId() this.userId = new ObjectId() - this.folderPath = '/path/to/folder' - this.docName = 'test-doc.tex' - this.docPath = `/path/to/folder/${this.docName}` - this.docContent = 'one\ntwo\nthree' - this.docLines = this.docContent.split('\n') - this.fileName = 'test-file.jpg' - this.filePath = `/path/to/folder/${this.fileName}` - this.symlinkName = 'symlink' - this.symlinkPath = `/path/to/${this.symlinkName}` - this.ignoredName = '.DS_Store' - this.ignoredPath = `/path/to/folder/${this.ignoredName}` - this.folderEntries = [this.ignoredName, this.docName, this.fileName] - - this.encoding = 'latin1' - - this.fileStat = { - isFile: sinon.stub().returns(true), - isDirectory: sinon.stub().returns(false) - } - this.dirStat = { - isFile: sinon.stub().returns(false), - isDirectory: sinon.stub().returns(true) - } - this.symlinkStat = { - isFile: sinon.stub().returns(false), - isDirectory: sinon.stub().returns(false) - } - this.fs = { - promises: { - lstat: sinon.stub(), - readFile: sinon.stub(), - readdir: sinon.stub() - } - } - this.fs.promises.lstat.withArgs(this.filePath).resolves(this.fileStat) - this.fs.promises.lstat.withArgs(this.docPath).resolves(this.fileStat) - this.fs.promises.lstat.withArgs(this.symlinkPath).resolves(this.symlinkStat) - this.fs.promises.lstat.withArgs(this.folderPath).resolves(this.dirStat) - this.fs.promises.readFile - .withArgs(this.docPath, this.encoding) - .resolves(this.docContent) - this.fs.promises.readdir - .withArgs(this.folderPath) - .resolves(this.folderEntries) this.EditorController = { promises: { addDoc: sinon.stub().resolves(), @@ -66,25 +23,6 @@ describe('FileSystemImportManager', function() { addFolder: sinon.stub().resolves({ _id: this.newFolderId }) } } - this.FileTypeManager = { - promises: { - isDirectory: sinon.stub().resolves(false), - getType: sinon.stub(), - shouldIgnore: sinon.stub().resolves(false) - } - } - this.FileTypeManager.promises.getType - .withArgs(this.fileName, this.filePath) - .resolves({ binary: true }) - this.FileTypeManager.promises.getType - .withArgs(this.docName, this.docPath) - .resolves({ binary: false, encoding: this.encoding }) - this.FileTypeManager.promises.isDirectory - .withArgs(this.folderPath) - .resolves(true) - this.FileTypeManager.promises.shouldIgnore - .withArgs(this.ignoredName) - .resolves(true) this.logger = { log() {}, err() {} @@ -94,40 +32,165 @@ describe('FileSystemImportManager', function() { console: console }, requires: { - fs: this.fs, '../Editor/EditorController': this.EditorController, - './FileTypeManager': this.FileTypeManager, 'logger-sharelatex': this.logger } }) }) - describe('addFolderContents', function() { - describe('successfully', function() { + describe('importDir', function() { + beforeEach(async function() { + mockFs({ + 'import-test': { + 'main.tex': 'My thesis', + 'link-to-main.tex': mockFs.symlink({ path: 'import-test/main.tex' }), + '.DS_Store': 'Should be ignored', + images: { + 'cat.jpg': Buffer.from([1, 2, 3, 4]) + }, + 'line-endings': { + 'unix.txt': 'one\ntwo\nthree', + 'mac.txt': 'uno\rdos\rtres', + 'windows.txt': 'ein\r\nzwei\r\ndrei', + 'mixed.txt': 'uno\rdue\r\ntre\nquattro' + }, + encodings: { + 'utf16le.txt': Buffer.from('\ufeffétonnant!', 'utf16le'), + 'latin1.txt': Buffer.from('tétanisant!', 'latin1') + } + }, + symlink: mockFs.symlink({ path: 'import-test' }) + }) + this.entries = await this.FileSystemImportManager.promises.importDir( + 'import-test' + ) + this.projectPaths = this.entries.map(x => x.projectPath) + }) + + afterEach(function() { + mockFs.restore() + }) + + it('should import regular docs', function() { + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/main.tex', + lines: ['My thesis'] + }) + }) + + it('should skip symlinks inside the import folder', function() { + expect(this.projectPaths).not.to.include('/link-to-main.tex') + }) + + it('should skip ignored files', function() { + expect(this.projectPaths).not.to.include('/.DS_Store') + }) + + it('should import binary files', function() { + expect(this.entries).to.deep.include({ + type: 'file', + projectPath: '/images/cat.jpg', + fsPath: 'import-test/images/cat.jpg' + }) + }) + + it('should deal with Mac/Windows/Unix line endings', function() { + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/line-endings/unix.txt', + lines: ['one', 'two', 'three'] + }) + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/line-endings/mac.txt', + lines: ['uno', 'dos', 'tres'] + }) + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/line-endings/windows.txt', + lines: ['ein', 'zwei', 'drei'] + }) + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/line-endings/mixed.txt', + lines: ['uno', 'due', 'tre', 'quattro'] + }) + }) + + it('should import documents with latin1 encoding', function() { + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/encodings/latin1.txt', + lines: ['tétanisant!'] + }) + }) + + it('should import documents with utf16-le encoding', function() { + expect(this.entries).to.deep.include({ + type: 'doc', + projectPath: '/encodings/utf16le.txt', + lines: ['\ufeffétonnant!'] + }) + }) + + it('should error when the root folder is a symlink', async function() { + await expect(this.FileSystemImportManager.promises.importDir('symlink')) + .to.be.rejected + }) + }) + + describe('addEntity', function() { + describe('with directory', function() { beforeEach(async function() { - await this.FileSystemImportManager.promises.addFolderContents( + mockFs({ + path: { + to: { + folder: { + 'doc.tex': 'one\ntwo\nthree', + 'image.jpg': Buffer.from([1, 2, 3, 4]) + } + } + } + }) + + await this.FileSystemImportManager.promises.addEntity( this.userId, this.projectId, this.folderId, - this.folderPath, + 'folder', + 'path/to/folder', false ) }) - it('should add each file in the folder which is not ignored', function() { - this.EditorController.promises.addDoc.should.have.been.calledWith( + afterEach(function() { + mockFs.restore() + }) + + it('should add a folder to the project', function() { + this.EditorController.promises.addFolder.should.have.been.calledWith( this.projectId, this.folderId, - this.docName, - this.docLines, + 'folder', + 'upload' + ) + }) + + it("should add the folder's contents", function() { + this.EditorController.promises.addDoc.should.have.been.calledWith( + this.projectId, + this.newFolderId, + 'doc.tex', + ['one', 'two', 'three'], 'upload', this.userId ) this.EditorController.promises.addFile.should.have.been.calledWith( this.projectId, - this.folderId, - this.fileName, - this.filePath, + this.newFolderId, + 'image.jpg', + 'path/to/folder/image.jpg', null, 'upload', this.userId @@ -135,78 +198,23 @@ describe('FileSystemImportManager', function() { }) }) - describe('with symlink', function() { - it('should stop with an error', async function() { - await expect( - this.FileSystemImportManager.promises.addFolderContents( - this.userId, - this.projectId, - this.folderId, - this.symlinkPath, - false - ) - ).to.be.rejectedWith('path is symlink') - this.EditorController.promises.addFolder.should.not.have.been.called - this.EditorController.promises.addDoc.should.not.have.been.called - this.EditorController.promises.addFile.should.not.have.been.called - }) - }) - }) - - describe('addEntity', function() { - describe('with directory', function() { - describe('successfully', function() { - beforeEach(async function() { - await this.FileSystemImportManager.promises.addEntity( - this.userId, - this.projectId, - this.folderId, - this.folderName, - this.folderPath, - false - ) - }) - - it('should add a folder to the project', function() { - this.EditorController.promises.addFolder.should.have.been.calledWith( - this.projectId, - this.folderId, - this.folderName, - 'upload' - ) - }) - - it("should add the folder's contents", function() { - this.EditorController.promises.addDoc.should.have.been.calledWith( - this.projectId, - this.newFolderId, - this.docName, - this.docLines, - 'upload', - this.userId - ) - this.EditorController.promises.addFile.should.have.been.calledWith( - this.projectId, - this.newFolderId, - this.fileName, - this.filePath, - null, - 'upload', - this.userId - ) - }) - }) - }) - describe('with binary file', function() { + beforeEach(function() { + mockFs({ 'uploaded-file': Buffer.from([1, 2, 3, 4]) }) + }) + + afterEach(function() { + mockFs.restore() + }) + describe('with replace set to false', function() { beforeEach(async function() { await this.FileSystemImportManager.promises.addEntity( this.userId, this.projectId, this.folderId, - this.fileName, - this.filePath, + 'image.jpg', + 'uploaded-file', false ) }) @@ -215,8 +223,8 @@ describe('FileSystemImportManager', function() { this.EditorController.promises.addFile.should.have.been.calledWith( this.projectId, this.folderId, - this.fileName, - this.filePath, + 'image.jpg', + 'uploaded-file', null, 'upload', this.userId @@ -230,8 +238,8 @@ describe('FileSystemImportManager', function() { this.userId, this.projectId, this.folderId, - this.fileName, - this.filePath, + 'image.jpg', + 'uploaded-file', true ) }) @@ -240,24 +248,42 @@ describe('FileSystemImportManager', function() { this.EditorController.promises.upsertFile.should.have.been.calledWith( this.projectId, this.folderId, - this.fileName, - this.filePath, + 'image.jpg', + 'uploaded-file', null, 'upload', this.userId ) }) }) + }) + + for (const [lineEndingDescription, lineEnding] of [ + ['Unix', '\n'], + ['Mac', '\r'], + ['Windows', '\r\n'] + ]) { + describe(`with text file (${lineEndingDescription} line endings)`, function() { + beforeEach(function() { + mockFs({ + path: { + to: { 'uploaded-file': `one${lineEnding}two${lineEnding}three` } + } + }) + }) + + afterEach(function() { + mockFs.restore() + }) - describe('with text file', function() { describe('with replace set to false', function() { beforeEach(async function() { await this.FileSystemImportManager.promises.addEntity( this.userId, this.projectId, this.folderId, - this.docName, - this.docPath, + 'doc.tex', + 'path/to/uploaded-file', false ) }) @@ -266,66 +292,8 @@ describe('FileSystemImportManager', function() { this.EditorController.promises.addDoc.should.have.been.calledWith( this.projectId, this.folderId, - this.docName, - this.docLines, - 'upload', - this.userId - ) - }) - }) - - describe('with windows line ending', function() { - beforeEach(async function() { - this.docContent = 'one\r\ntwo\r\nthree' - this.docLines = ['one', 'two', 'three'] - this.fs.promises.readFile - .withArgs(this.docPath, this.encoding) - .resolves(this.docContent) - await this.FileSystemImportManager.promises.addEntity( - this.userId, - this.projectId, - this.folderId, - this.docName, - this.docPath, - false - ) - }) - - it('should strip the \\r characters before adding', function() { - this.EditorController.promises.addDoc.should.have.been.calledWith( - this.projectId, - this.folderId, - this.docName, - this.docLines, - 'upload', - this.userId - ) - }) - }) - - describe('with \r line endings', function() { - beforeEach(async function() { - this.docContent = 'one\rtwo\rthree' - this.docLines = ['one', 'two', 'three'] - this.fs.promises.readFile - .withArgs(this.docPath, this.encoding) - .resolves(this.docContent) - await this.FileSystemImportManager.promises.addEntity( - this.userId, - this.projectId, - this.folderId, - this.docName, - this.docPath, - false - ) - }) - - it('should treat the \\r characters as newlines', function() { - this.EditorController.promises.addDoc.should.have.been.calledWith( - this.projectId, - this.folderId, - this.docName, - this.docLines, + 'doc.tex', + ['one', 'two', 'three'], 'upload', this.userId ) @@ -338,8 +306,8 @@ describe('FileSystemImportManager', function() { this.userId, this.projectId, this.folderId, - this.docName, - this.docPath, + 'doc.tex', + 'path/to/uploaded-file', true ) }) @@ -348,25 +316,35 @@ describe('FileSystemImportManager', function() { this.EditorController.promises.upsertDoc.should.have.been.calledWith( this.projectId, this.folderId, - this.docName, - this.docLines, + 'doc.tex', + ['one', 'two', 'three'], 'upload', this.userId ) }) }) }) - }) + } describe('with symlink', function() { + beforeEach(function() { + mockFs({ + path: { to: { symlink: mockFs.symlink({ path: '/etc/passwd' }) } } + }) + }) + + afterEach(function() { + mockFs.restore() + }) + it('should stop with an error', async function() { await expect( this.FileSystemImportManager.promises.addEntity( this.userId, this.projectId, this.folderId, - this.symlinkName, - this.symlinkPath, + 'main.tex', + 'path/to/symlink', false ) ).to.be.rejectedWith('path is symlink') diff --git a/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js b/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js index bca4338410..af7fccb816 100644 --- a/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js +++ b/services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js @@ -2,6 +2,7 @@ const sinon = require('sinon') const { expect } = require('chai') const timekeeper = require('timekeeper') const SandboxedModule = require('sandboxed-module') +const { ObjectId } = require('mongodb') const MODULE_PATH = '../../../../app/src/Features/Uploads/ProjectUploadManager.js' @@ -10,20 +11,55 @@ describe('ProjectUploadManager', function() { beforeEach(function() { this.now = Date.now() timekeeper.freeze(this.now) - this.project_id = 'project-id-123' - this.folder_id = 'folder-id-123' - this.owner_id = 'owner-id-123' - this.source = '/path/to/zip/file-name.zip' - this.destination = `/path/to/zip/file-name-${this.now}` - this.root_folder_id = this.folder_id - this.owner_id = 'owner-id-123' - this.name = 'Project name' - this.othername = 'Other name' + this.rootFolderId = new ObjectId() + this.ownerId = new ObjectId() + this.zipPath = '/path/to/zip/file-name.zip' + this.extractedZipPath = `/path/to/zip/file-name-${this.now}` + this.mainContent = 'Contents of main.tex' + this.projectName = 'My project*' + this.fixedProjectName = 'My project' + this.uniqueProjectName = 'My project (1)' this.project = { - _id: this.project_id, - rootFolder: [{ _id: this.root_folder_id }] + _id: new ObjectId(), + rootFolder: [{ _id: this.rootFolderId }], + overleaf: { history: { id: 12345 } } } + this.doc = { + _id: new ObjectId(), + name: 'main.tex' + } + this.docFsPath = '/path/to/doc' + this.docLines = ['My thesis', 'by A. U. Thor'] + this.file = { + _id: new ObjectId(), + name: 'image.png' + } + this.fileFsPath = '/path/to/file' + this.topLevelDestination = '/path/to/zip/file-extracted/nested' + this.newProjectVersion = 123 + this.importEntries = [ + { + type: 'doc', + projectPath: '/main.tex', + lines: this.docLines + }, + { + type: 'file', + projectPath: `/${this.file.name}`, + fsPath: this.fileFsPath + } + ] + this.docEntries = [ + { + doc: this.doc, + path: `/${this.doc.name}`, + docLines: this.docLines.join('\n') + } + ] + this.fileEntries = [ + { file: this.file, path: `/${this.file.name}`, url: this.fileStoreUrl } + ] this.fs = { remove: sinon.stub().resolves() @@ -31,12 +67,42 @@ describe('ProjectUploadManager', function() { this.ArchiveManager = { promises: { extractZipArchive: sinon.stub().resolves(), - findTopLevelDirectory: sinon.stub().resolves(this.topLevelDestination) + findTopLevelDirectory: sinon + .stub() + .withArgs(this.extractedZipPath) + .resolves(this.topLevelDestination) + } + } + this.Doc = sinon.stub().returns(this.doc) + this.DocstoreManager = { + promises: { + updateDoc: sinon.stub().resolves() + } + } + this.DocumentHelper = { + getTitleFromTexContent: sinon + .stub() + .withArgs(this.mainContent) + .returns(this.projectName) + } + this.DocumentUpdaterHandler = { + promises: { + updateProjectStructure: sinon.stub().resolves() + } + } + this.FileStoreHandler = { + promises: { + uploadFileFromDisk: sinon + .stub() + .resolves({ fileRef: this.file, url: this.fileStoreUrl }) } } this.FileSystemImportManager = { promises: { - addFolderContents: sinon.stub().resolves() + importDir: sinon + .stub() + .withArgs(this.topLevelDestination) + .resolves(this.importEntries) } } this.ProjectCreationHandler = { @@ -44,19 +110,27 @@ describe('ProjectUploadManager', function() { createBlankProject: sinon.stub().resolves(this.project) } } + this.ProjectEntityMongoUpdateHandler = { + promises: { + createNewFolderStructure: sinon.stub().resolves(this.newProjectVersion) + } + } this.ProjectRootDocManager = { promises: { setRootDocAutomatically: sinon.stub().resolves(), findRootDocFileFromDirectory: sinon .stub() - .resolves({ path: 'main.tex', content: this.othername }), + .resolves({ path: 'main.tex', content: this.mainContent }), setRootDocFromName: sinon.stub().resolves() } } this.ProjectDetailsHandler = { - fixProjectName: sinon.stub().returnsArg(0), + fixProjectName: sinon + .stub() + .withArgs(this.projectName) + .returns(this.fixedProjectName), promises: { - generateUniqueName: sinon.stub().resolves(this.othername) + generateUniqueName: sinon.stub().resolves(this.uniqueProjectName) } } this.ProjectDeleter = { @@ -64,8 +138,10 @@ describe('ProjectUploadManager', function() { deleteProject: sinon.stub().resolves() } } - this.DocumentHelper = { - getTitleFromTexContent: sinon.stub().returns(this.othername) + this.TpdsProjectFlusher = { + promises: { + flushProjectToTpds: sinon.stub().resolves() + } } this.ProjectUploadManager = SandboxedModule.require(MODULE_PATH, { @@ -74,13 +150,21 @@ describe('ProjectUploadManager', function() { }, requires: { 'fs-extra': this.fs, - './FileSystemImportManager': this.FileSystemImportManager, './ArchiveManager': this.ArchiveManager, + '../../models/Doc': { Doc: this.Doc }, + '../Docstore/DocstoreManager': this.DocstoreManager, + '../Documents/DocumentHelper': this.DocumentHelper, + '../DocumentUpdater/DocumentUpdaterHandler': this + .DocumentUpdaterHandler, + '../FileStore/FileStoreHandler': this.FileStoreHandler, + './FileSystemImportManager': this.FileSystemImportManager, '../Project/ProjectCreationHandler': this.ProjectCreationHandler, + '../Project/ProjectEntityMongoUpdateHandler': this + .ProjectEntityMongoUpdateHandler, '../Project/ProjectRootDocManager': this.ProjectRootDocManager, '../Project/ProjectDetailsHandler': this.ProjectDetailsHandler, '../Project/ProjectDeleter': this.ProjectDeleter, - '../Documents/DocumentHelper': this.DocumentHelper + '../ThirdPartyDataStore/TpdsProjectFlusher': this.TpdsProjectFlusher } }) }) @@ -93,61 +177,62 @@ describe('ProjectUploadManager', function() { describe('when the title can be read from the root document', function() { beforeEach(async function() { await this.ProjectUploadManager.promises.createProjectFromZipArchive( - this.owner_id, - this.name, - this.source + this.ownerId, + this.projectName, + this.zipPath ) }) it('should extract the archive', function() { this.ArchiveManager.promises.extractZipArchive.should.have.been.calledWith( - this.source, - this.destination + this.zipPath, + this.extractedZipPath ) }) - it('should find the top level directory', function() { - this.ArchiveManager.promises.findTopLevelDirectory.should.have.been.calledWith( - this.destination - ) - }) - - it('should insert the extracted archive into the folder', function() { - this.FileSystemImportManager.promises.addFolderContents.should.have.been.calledWith( - this.owner_id, - this.project_id, - this.folder_id, - this.topLevelDestination, - false - ) - }) - - it('should create a project owned by the owner_id', function() { + it('should create a project', function() { this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith( - this.owner_id + this.ownerId, + this.uniqueProjectName ) }) - it('should create a project with the correct name', function() { - this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith( - sinon.match.any, - this.othername + it('should initialize the file tree', function() { + this.ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure.should.have.been.calledWith( + this.project._id, + this.docEntries, + this.fileEntries ) }) - it('should read the title from the tex contents', function() { - this.DocumentHelper.getTitleFromTexContent.should.have.been.called + it('should notify document updater', function() { + this.DocumentUpdaterHandler.promises.updateProjectStructure.should.have.been.calledWith( + this.project._id, + this.project.overleaf.history.id, + this.ownerId, + { + newDocs: this.docEntries, + newFiles: this.fileEntries, + newProject: { version: this.newProjectVersion } + } + ) + }) + + it('should flush the project to TPDS', function() { + this.TpdsProjectFlusher.promises.flushProjectToTpds.should.have.been.calledWith( + this.project._id + ) }) it('should set the root document', function() { this.ProjectRootDocManager.promises.setRootDocFromName.should.have.been.calledWith( - this.project_id, + this.project._id, 'main.tex' ) }) - it('should ensure the name is valid', function() { - this.ProjectDetailsHandler.fixProjectName.should.have.been.called + it('should remove the destination directory afterwards', function() { + this.fs.remove.should.have.been.calledWith(this.extractedZipPath) }) }) @@ -157,9 +242,9 @@ describe('ProjectUploadManager', function() { {} ) await this.ProjectUploadManager.promises.createProjectFromZipArchive( - this.owner_id, - this.name, - this.source + this.ownerId, + this.projectName, + this.zipPath ) }) @@ -173,73 +258,78 @@ describe('ProjectUploadManager', function() { describe('createProjectFromZipArchiveWithName', function() { beforeEach(async function() { await this.ProjectUploadManager.promises.createProjectFromZipArchiveWithName( - this.owner_id, - this.name, - this.source - ) - }) - - it('should create a project owned by the owner_id', function() { - this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith( - this.owner_id - ) - }) - - it('should create a project with the correct name', function() { - this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith( - sinon.match.any, - this.othername - ) - }) - - it('should automatically set the root doc', function() { - this.ProjectRootDocManager.promises.setRootDocAutomatically.should.have.been.calledWith( - this.project_id + this.ownerId, + this.projectName, + this.zipPath ) }) it('should extract the archive', function() { this.ArchiveManager.promises.extractZipArchive.should.have.been.calledWith( - this.source, - this.destination + this.zipPath, + this.extractedZipPath ) }) - it('should find the top level directory', function() { - this.ArchiveManager.promises.findTopLevelDirectory.should.have.been.calledWith( - this.destination + it('should create a project owned by the owner_id', function() { + this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith( + this.ownerId, + this.uniqueProjectName ) }) - it('should insert the extracted archive into the folder', function() { - this.FileSystemImportManager.promises.addFolderContents.should.have.been.calledWith( - this.owner_id, - this.project_id, - this.folder_id, - this.topLevelDestination, - false + it('should automatically set the root doc', function() { + this.ProjectRootDocManager.promises.setRootDocAutomatically.should.have.been.calledWith( + this.project._id + ) + }) + + it('should initialize the file tree', function() { + this.ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure.should.have.been.calledWith( + this.project._id, + this.docEntries, + this.fileEntries + ) + }) + + it('should notify document updater', function() { + this.DocumentUpdaterHandler.promises.updateProjectStructure.should.have.been.calledWith( + this.project._id, + this.project.overleaf.history.id, + this.ownerId, + { + newDocs: this.docEntries, + newFiles: this.fileEntries, + newProject: { version: this.newProjectVersion } + } + ) + }) + + it('should flush the project to TPDS', function() { + this.TpdsProjectFlusher.promises.flushProjectToTpds.should.have.been.calledWith( + this.project._id ) }) it('should remove the destination directory afterwards', function() { - this.fs.remove.should.have.been.calledWith(this.destination) + this.fs.remove.should.have.been.calledWith(this.extractedZipPath) }) - describe('when inserting the zip file contents into the root folder fails', function() { + describe('when initializing the folder structure fails', function() { beforeEach(async function() { - this.FileSystemImportManager.promises.addFolderContents.rejects() + this.ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure.rejects() await expect( this.ProjectUploadManager.promises.createProjectFromZipArchiveWithName( - this.owner_id, - this.name, - this.source + this.ownerId, + this.projectName, + this.zipPath ) ).to.be.rejected }) it('should cleanup the blank project created', async function() { this.ProjectDeleter.promises.deleteProject.should.have.been.calledWith( - this.project_id + this.project._id ) }) }) @@ -249,16 +339,16 @@ describe('ProjectUploadManager', function() { this.ProjectRootDocManager.promises.setRootDocAutomatically.rejects() await expect( this.ProjectUploadManager.promises.createProjectFromZipArchiveWithName( - this.owner_id, - this.name, - this.source + this.ownerId, + this.projectName, + this.zipPath ) ).to.be.rejected }) it('should cleanup the blank project created', function() { this.ProjectDeleter.promises.deleteProject.should.have.been.calledWith( - this.project_id + this.project._id ) }) })