Merge pull request #2760 from overleaf/em-faster-uploads

Make a single Mongo update when uploading projects

GitOrigin-RevId: de102d3e112c9014ca5885f963e35971e4db6cee

services/web/app/src/Features/FileStore/FileStoreHandler.js

@@ -7,6 +7,7 @@ const Async = require('async')
 const FileHashManager = require('./FileHashManager')
 const { File } = require('../../models/File')
 const Errors = require('../Errors/Errors')
+const { promisifyAll } = require('../../util/promises')
 
 const ONE_MIN_IN_MS = 60 * 1000
 const FIVE_MINS_IN_MS = ONE_MIN_IN_MS * 5
@@ -227,3 +228,8 @@ const FileStoreHandler = {
 }
 
 module.exports = FileStoreHandler
+module.exports.promises = promisifyAll(FileStoreHandler, {
+  multiResult: {
+    uploadFileFromDisk: ['url', 'fileRef']
+  }
+})

services/web/app/src/Features/Project/FolderStructureBuilder.js

@@ -4,13 +4,13 @@ const { ObjectId } = require('mongodb')
 
 module.exports = { buildFolderStructure }
 
-function buildFolderStructure(docUploads, fileUploads) {
+function buildFolderStructure(docEntries, fileEntries) {
   const builder = new FolderStructureBuilder()
-  for (const docUpload of docUploads) {
+  for (const docEntry of docEntries) {
-    builder.addDocUpload(docUpload)
+    builder.addDocEntry(docEntry)
   }
-  for (const fileUpload of fileUploads) {
+  for (const fileEntry of fileEntries) {
-    builder.addFileUpload(fileUpload)
+    builder.addFileEntry(fileEntry)
   }
   return builder.rootFolder
 }
@@ -24,18 +24,18 @@ class FolderStructureBuilder {
     this.entityPaths.add('/')
   }
 
-  addDocUpload(docUpload) {
+  addDocEntry(docEntry) {
-    this.recordEntityPath(Path.join(docUpload.dirname, docUpload.doc.name))
+    this.recordEntityPath(docEntry.path)
-    const folder = this.mkdirp(docUpload.dirname)
+    const folderPath = Path.dirname(docEntry.path)
-    folder.docs.push(docUpload.doc)
+    const folder = this.mkdirp(folderPath)
+    folder.docs.push(docEntry.doc)
   }
 
-  addFileUpload(fileUpload) {
+  addFileEntry(fileEntry) {
-    this.recordEntityPath(
+    this.recordEntityPath(fileEntry.path)
-      Path.join(fileUpload.dirname, fileUpload.fileRef.name)
+    const folderPath = Path.dirname(fileEntry.path)
-    )
+    const folder = this.mkdirp(folderPath)
-    const folder = this.mkdirp(fileUpload.dirname)
+    folder.fileRefs.push(fileEntry.file)
-    folder.fileRefs.push(fileUpload.fileRef)
   }
 
   mkdirp(path) {

services/web/app/src/Features/Project/ProjectEntityMongoUpdateHandler.js

@@ -1,8 +1,3 @@
-/* NOTE: this file is an async/await version of
- * ProjectEntityMongoUpdateHandler.js. It's temporarily separate from the
- * callback-style version so that we can test it in production for some code
- * paths only.
- */
 const { callbackify } = require('util')
 const { callbackifyMultiResult } = require('../../util/promises')
 const _ = require('underscore')
@@ -635,13 +630,24 @@ async function _checkValidMove(
   }
 }
 
-async function createNewFolderStructure(projectId, docUploads, fileUploads) {
+/**
+ * Create an initial file tree out of a list of doc and file entries
+ *
+ * Each entry specifies a path to the doc or file. Folders are automatically
+ * created.
+ *
+ * @param {ObjectId} projectId - id of the project
+ * @param {DocEntry[]} docEntries - list of docs to add
+ * @param {FileEntry[]} fileEntries - list of files to add
+ * @return {Promise<string>} the project version after the operation
+ */
+async function createNewFolderStructure(projectId, docEntries, fileEntries) {
   try {
     const rootFolder = FolderStructureBuilder.buildFolderStructure(
-      docUploads,
+      docEntries,
-      fileUploads
+      fileEntries
     )
-    const result = await Project.updateOne(
+    const project = await Project.findOneAndUpdate(
       {
         _id: projectId,
         'rootFolder.0.folders.0': { $exists: false },
@@ -651,14 +657,20 @@ async function createNewFolderStructure(projectId, docUploads, fileUploads) {
       {
         $set: { rootFolder: [rootFolder] },
         $inc: { version: 1 }
+      },
+      {
+        new: true,
+        lean: true,
+        fields: { version: 1 }
       }
     ).exec()
-    if (result.n !== 1) {
+    if (project == null) {
       throw new OError({
         message: 'project not found or folder structure already exists',
         info: { projectId }
       })
     }
+    return project.version
   } catch (err) {
     throw new OError({
       message: 'failed to create folder structure',

services/web/app/src/Features/Uploads/FileSystemImportManager.js

@@ -1,37 +1,22 @@
 const fs = require('fs')
+const Path = require('path')
 const { callbackify } = require('util')
-const FileTypeManager = require('./FileTypeManager')
 const EditorController = require('../Editor/EditorController')
+const Errors = require('../Errors/Errors')
+const FileTypeManager = require('./FileTypeManager')
+const SafePath = require('../Project/SafePath')
 const logger = require('logger-sharelatex')
 
 module.exports = {
-  addFolderContents: callbackify(addFolderContents),
   addEntity: callbackify(addEntity),
+  importDir: callbackify(importDir),
   promises: {
-    addFolderContents,
+    addEntity,
-    addEntity
+    importDir
   }
 }
 
-async function addDoc(
+async function addDoc(userId, projectId, folderId, name, lines, replace) {
-  userId,
-  projectId,
-  folderId,
-  name,
-  path,
-  encoding,
-  replace
-) {
-  if (!(await _isSafeOnFileSystem(path))) {
-    logger.log(
-      { userId, projectId, folderId, name, path },
-      'add doc is from symlink, stopping process'
-    )
-    throw new Error('path is symlink')
-  }
-  let content = await fs.promises.readFile(path, encoding)
-  content = content.replace(/\r\n?/g, '\n') // convert Windows line endings to unix. very old macs also created \r-separated lines
-  const lines = content.split('\n')
   if (replace) {
     const doc = await EditorController.promises.upsertDoc(
       projectId,
@@ -56,14 +41,6 @@ async function addDoc(
 }
 
 async function addFile(userId, projectId, folderId, name, path, replace) {
-  if (!(await _isSafeOnFileSystem(path))) {
-    logger.log(
-      { userId, projectId, folderId, name, path },
-      'add file is from symlink, stopping insert'
-    )
-    throw new Error('path is symlink')
-  }
-
   if (replace) {
     const file = await EditorController.promises.upsertFile(
       projectId,
@@ -90,13 +67,6 @@ async function addFile(userId, projectId, folderId, name, path, replace) {
 }
 
 async function addFolder(userId, projectId, folderId, name, path, replace) {
-  if (!(await _isSafeOnFileSystem(path))) {
-    logger.log(
-      { userId, projectId, folderId, path },
-      'add folder is from symlink, stopping insert'
-    )
-    throw new Error('path is symlink')
-  }
   const newFolder = await EditorController.promises.addFolder(
     projectId,
     folderId,
@@ -137,61 +107,149 @@ async function addFolderContents(
   }
 }
 
-async function addEntity(userId, projectId, folderId, name, path, replace) {
+async function addEntity(userId, projectId, folderId, name, fsPath, replace) {
-  if (!(await _isSafeOnFileSystem(path))) {
+  if (!(await _isSafeOnFileSystem(fsPath))) {
     logger.log(
-      { userId, projectId, folderId, path },
+      { userId, projectId, folderId, fsPath },
       'add entry is from symlink, stopping insert'
     )
     throw new Error('path is symlink')
   }
 
-  if (await FileTypeManager.promises.isDirectory(path)) {
+  if (await FileTypeManager.promises.isDirectory(fsPath)) {
     const newFolder = await addFolder(
       userId,
       projectId,
       folderId,
       name,
-      path,
+      fsPath,
       replace
     )
     return newFolder
   }
-  const { binary, encoding } = await FileTypeManager.promises.getType(
+
-    name,
+  // Here, we cheat a little bit and provide the project path relative to the
-    path
+  // folder, not the root of the project. This is because we don't know for sure
-  )
+  // at this point what the final path of the folder will be. The project path
-  if (binary) {
+  // is still important for importFile() to be able to figure out if the file is
-    const entity = await addFile(
+  // a binary file or an editable document.
-      userId,
+  const projectPath = Path.join('/', name)
-      projectId,
+  const importInfo = await importFile(fsPath, projectPath)
-      folderId,
+  switch (importInfo.type) {
-      name,
+    case 'file': {
-      path,
+      const entity = await addFile(
-      replace
+        userId,
-    )
+        projectId,
-    if (entity != null) {
+        folderId,
-      entity.type = 'file'
+        name,
+        importInfo.fsPath,
+        replace
+      )
+      if (entity != null) {
+        entity.type = 'file'
+      }
+      return entity
     }
-    return entity
+    case 'doc': {
-  } else {
+      const entity = await addDoc(
-    const entity = await addDoc(
+        userId,
-      userId,
+        projectId,
-      projectId,
+        folderId,
-      folderId,
+        name,
-      name,
+        importInfo.lines,
-      path,
+        replace
-      encoding,
+      )
-      replace
+      if (entity != null) {
-    )
+        entity.type = 'doc'
-    if (entity != null) {
+      }
-      entity.type = 'doc'
+      return entity
+    }
+    default: {
+      throw new Error(`unknown import type: ${importInfo.type}`)
     }
-    return entity
   }
 }
 
 async function _isSafeOnFileSystem(path) {
+  // Use lstat() to ensure we don't follow symlinks. Symlinks from an
+  // untrusted source are dangerous.
   const stat = await fs.promises.lstat(path)
   return stat.isFile() || stat.isDirectory()
 }
+
+async function importFile(fsPath, projectPath) {
+  const stat = await fs.promises.lstat(fsPath)
+  if (!stat.isFile()) {
+    throw new Error(`can't import ${fsPath}: not a regular file`)
+  }
+  _validateProjectPath(projectPath)
+  const filename = Path.basename(projectPath)
+
+  const { binary, encoding } = await FileTypeManager.promises.getType(
+    filename,
+    fsPath
+  )
+  if (binary) {
+    return new FileImport(projectPath, fsPath)
+  } else {
+    const content = await fs.promises.readFile(fsPath, encoding)
+    // Handle Unix, DOS and classic Mac newlines
+    const lines = content.split(/\r\n|\n|\r/)
+    return new DocImport(projectPath, lines)
+  }
+}
+
+async function importDir(dirPath) {
+  const stat = await fs.promises.lstat(dirPath)
+  if (!stat.isDirectory()) {
+    throw new Error(`can't import ${dirPath}: not a directory`)
+  }
+  const entries = []
+  for await (const filePath of _walkDir(dirPath)) {
+    const projectPath = Path.join('/', Path.relative(dirPath, filePath))
+    const importInfo = await importFile(filePath, projectPath)
+    entries.push(importInfo)
+  }
+  return entries
+}
+
+function _validateProjectPath(path) {
+  if (!SafePath.isAllowedLength(path) || !SafePath.isCleanPath(path)) {
+    throw new Errors.InvalidNameError(`Invalid path: ${path}`)
+  }
+}
+
+async function* _walkDir(dirPath) {
+  const entries = await fs.promises.readdir(dirPath)
+  for (const entry of entries) {
+    const entryPath = Path.join(dirPath, entry)
+    if (await FileTypeManager.promises.shouldIgnore(entryPath)) {
+      continue
+    }
+
+    // Use lstat() to ensure we don't follow symlinks. Symlinks from an
+    // untrusted source are dangerous.
+    const stat = await fs.promises.lstat(entryPath)
+    if (stat.isFile()) {
+      yield entryPath
+    } else if (stat.isDirectory()) {
+      yield* _walkDir(entryPath)
+    }
+  }
+}
+
+class FileImport {
+  constructor(projectPath, fsPath) {
+    this.type = 'file'
+    this.projectPath = projectPath
+    this.fsPath = fsPath
+  }
+}
+
+class DocImport {
+  constructor(projectPath, lines) {
+    this.type = 'doc'
+    this.projectPath = projectPath
+    this.lines = lines
+  }
+}

services/web/app/src/Features/Uploads/ProjectUploadManager.js

@@ -1,13 +1,19 @@
-const path = require('path')
+const Path = require('path')
 const fs = require('fs-extra')
 const { callbackify } = require('util')
 const ArchiveManager = require('./ArchiveManager')
+const { Doc } = require('../../models/Doc')
+const DocstoreManager = require('../Docstore/DocstoreManager')
+const DocumentHelper = require('../Documents/DocumentHelper')
+const DocumentUpdaterHandler = require('../DocumentUpdater/DocumentUpdaterHandler')
+const FileStoreHandler = require('../FileStore/FileStoreHandler')
 const FileSystemImportManager = require('./FileSystemImportManager')
 const ProjectCreationHandler = require('../Project/ProjectCreationHandler')
+const ProjectEntityMongoUpdateHandler = require('../Project/ProjectEntityMongoUpdateHandler')
 const ProjectRootDocManager = require('../Project/ProjectRootDocManager')
 const ProjectDetailsHandler = require('../Project/ProjectDetailsHandler')
 const ProjectDeleter = require('../Project/ProjectDeleter')
-const DocumentHelper = require('../Documents/DocumentHelper')
+const TpdsProjectFlusher = require('../ThirdPartyDataStore/TpdsProjectFlusher')
 const logger = require('logger-sharelatex')
 
 module.exports = {
@@ -22,12 +28,12 @@ module.exports = {
 }
 
 async function createProjectFromZipArchive(ownerId, defaultName, zipPath) {
-  const extractionPath = await _extractZip(zipPath)
+  const contentsPath = await _extractZip(zipPath)
   const {
     path,
     content
   } = await ProjectRootDocManager.promises.findRootDocFileFromDirectory(
-    extractionPath
+    contentsPath
   )
 
   const projectName =
@@ -38,12 +44,7 @@ async function createProjectFromZipArchive(ownerId, defaultName, zipPath) {
     uniqueName
   )
   try {
-    await _insertZipContentsIntoFolder(
+    await _initializeProjectWithZipContents(ownerId, project, contentsPath)
-      ownerId,
-      project._id,
-      project.rootFolder[0]._id,
-      extractionPath
-    )
 
     if (path) {
       await ProjectRootDocManager.promises.setRootDocFromName(project._id, path)
@@ -60,6 +61,7 @@ async function createProjectFromZipArchive(ownerId, defaultName, zipPath) {
       )
     throw err
   }
+  await fs.remove(contentsPath)
   return project
 }
 
@@ -69,7 +71,7 @@ async function createProjectFromZipArchiveWithName(
   zipPath,
   attributes = {}
 ) {
-  const extractionPath = await _extractZip(zipPath)
+  const contentsPath = await _extractZip(zipPath)
   const uniqueName = await _generateUniqueName(ownerId, proposedName)
   const project = await ProjectCreationHandler.promises.createBlankProject(
     ownerId,
@@ -78,12 +80,7 @@ async function createProjectFromZipArchiveWithName(
   )
 
   try {
-    await _insertZipContentsIntoFolder(
+    await _initializeProjectWithZipContents(ownerId, project, contentsPath)
-      ownerId,
-      project._id,
-      project.rootFolder[0]._id,
-      extractionPath
-    )
     await ProjectRootDocManager.promises.setRootDocAutomatically(project._id)
   } catch (err) {
     // no need to wait for the cleanup here
@@ -97,33 +94,14 @@ async function createProjectFromZipArchiveWithName(
       )
     throw err
   }
-
+  await fs.remove(contentsPath)
   return project
 }
 
-async function _insertZipContentsIntoFolder(
-  ownerId,
-  projectId,
-  folderId,
-  destination
-) {
-  const topLevelDestination = await ArchiveManager.promises.findTopLevelDirectory(
-    destination
-  )
-  await FileSystemImportManager.promises.addFolderContents(
-    ownerId,
-    projectId,
-    folderId,
-    topLevelDestination,
-    false
-  )
-  await fs.remove(destination)
-}
-
 async function _extractZip(zipPath) {
-  const destination = path.join(
+  const destination = Path.join(
-    path.dirname(zipPath),
+    Path.dirname(zipPath),
-    `${path.basename(zipPath, '.zip')}-${Date.now()}`
+    `${Path.basename(zipPath, '.zip')}-${Date.now()}`
   )
   await ArchiveManager.promises.extractZipArchive(zipPath, destination)
   return destination
@@ -137,3 +115,96 @@ async function _generateUniqueName(ownerId, originalName) {
   )
   return uniqueName
 }
+
+async function _initializeProjectWithZipContents(
+  ownerId,
+  project,
+  contentsPath
+) {
+  const topLevelDir = await ArchiveManager.promises.findTopLevelDirectory(
+    contentsPath
+  )
+  const importEntries = await FileSystemImportManager.promises.importDir(
+    topLevelDir
+  )
+  const { fileEntries, docEntries } = await _createEntriesFromImports(
+    project._id,
+    importEntries
+  )
+  const projectVersion = await ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure(
+    project._id,
+    docEntries,
+    fileEntries
+  )
+  await _notifyDocumentUpdater(project, ownerId, {
+    newFiles: fileEntries,
+    newDocs: docEntries,
+    newProject: { version: projectVersion }
+  })
+  await TpdsProjectFlusher.promises.flushProjectToTpds(project._id)
+}
+
+async function _createEntriesFromImports(projectId, importEntries) {
+  const fileEntries = []
+  const docEntries = []
+  for (const importEntry of importEntries) {
+    switch (importEntry.type) {
+      case 'doc': {
+        const docEntry = await _createDoc(
+          projectId,
+          importEntry.projectPath,
+          importEntry.lines
+        )
+        docEntries.push(docEntry)
+        break
+      }
+      case 'file': {
+        const fileEntry = await _createFile(
+          projectId,
+          importEntry.projectPath,
+          importEntry.fsPath
+        )
+        fileEntries.push(fileEntry)
+        break
+      }
+      default: {
+        throw new Error(`Invalid import type: ${importEntry.type}`)
+      }
+    }
+  }
+  return { fileEntries, docEntries }
+}
+
+async function _createDoc(projectId, projectPath, docLines) {
+  const docName = Path.basename(projectPath)
+  const doc = new Doc({ name: docName })
+  await DocstoreManager.promises.updateDoc(
+    projectId.toString(),
+    doc._id.toString(),
+    docLines,
+    0,
+    {}
+  )
+  return { doc, path: projectPath, docLines: docLines.join('\n') }
+}
+
+async function _createFile(projectId, projectPath, fsPath) {
+  const fileName = Path.basename(projectPath)
+  const { fileRef, url } = await FileStoreHandler.promises.uploadFileFromDisk(
+    projectId,
+    { name: fileName },
+    fsPath
+  )
+  return { file: fileRef, path: projectPath, url }
+}
+
+async function _notifyDocumentUpdater(project, userId, changes) {
+  const projectHistoryId =
+    project.overleaf && project.overleaf.history && project.overleaf.history.id
+  await DocumentUpdaterHandler.promises.updateProjectStructure(
+    project._id,
+    projectHistoryId,
+    userId,
+    changes
+  )
+}

services/web/package-lock.json

@@ -7,7 +7,7 @@
     "@auth0/thumbprint": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/@auth0/thumbprint/-/thumbprint-0.0.6.tgz",
-      "integrity": "sha1-yrEGLGwEZizmxZLUgVfsQmiuhRg=",
+      "integrity": "sha512-+YciWHxNUOE78T+xoXI1fMI6G1WdyyAay8ioaMZhvGOJ+lReYzj0b7mpfNr5WtjGrmtWPvPOOxh0TO+5Y2M/Hw==",
       "dev": true
     },
     "@babel/cli": {
@@ -2597,7 +2597,7 @@
     "@protobufjs/aspromise": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
-      "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78="
+      "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="
     },
     "@protobufjs/base64": {
       "version": "1.1.2",
@@ -2612,12 +2612,12 @@
     "@protobufjs/eventemitter": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
-      "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A="
+      "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="
     },
     "@protobufjs/fetch": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
-      "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=",
+      "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==",
       "requires": {
         "@protobufjs/aspromise": "^1.1.1",
         "@protobufjs/inquire": "^1.1.0"
@@ -2626,27 +2626,27 @@
     "@protobufjs/float": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
-      "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E="
+      "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="
     },
     "@protobufjs/inquire": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
-      "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik="
+      "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="
     },
     "@protobufjs/path": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
-      "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0="
+      "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="
     },
     "@protobufjs/pool": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
-      "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q="
+      "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="
     },
     "@protobufjs/utf8": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
-      "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
+      "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="
     },
     "@sentry/browser": {
       "version": "5.15.4",
@@ -3613,7 +3613,7 @@
     "ansi-html": {
       "version": "0.0.7",
       "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz",
-      "integrity": "sha1-gTWEAhliqenm/QOflA0S9WynhZ4=",
+      "integrity": "sha512-JoAxEa1DfP9m2xfB/y2r/aKcwXNlltr4+0QSBC4TrLfcxyvepX2Pv0t/xpgGV5bGsDzCYV8SzjWgyCW0T9yYbA==",
       "dev": true
     },
     "ansi-regex": {
@@ -3908,7 +3908,7 @@
     "aria-query": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-3.0.0.tgz",
-      "integrity": "sha1-ZbP8wcoRVajJrmTW7uKX8V1RM8w=",
+      "integrity": "sha512-majUxHgLehQTeSA+hClx+DY09OVUqG3GtezWkF1krgLGNdlDu9l9V8DaqNMWbq4Eddc8wsyDA0hpDUtnYxQEXw==",
       "dev": true,
       "requires": {
         "ast-types-flow": "0.0.7",
@@ -4294,7 +4294,7 @@
     "ast-types-flow": {
       "version": "0.0.7",
       "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz",
-      "integrity": "sha1-9wtzXGvKGlycItmCw+Oef+ujva0=",
+      "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==",
       "dev": true
     },
     "async": {
@@ -4989,7 +4989,7 @@
     "batch": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz",
-      "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY=",
+      "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==",
       "dev": true
     },
     "bcrypt": {
@@ -5330,7 +5330,7 @@
     "bintrees": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz",
-      "integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ="
+      "integrity": "sha512-tbaUB1QpTIj4cKY8c1rvNAvEQXA+ekzHmbe4jzNfW3QWsF9GnnP/BRWyl6/qqS53heoYJ93naaFcm/jooONH8g=="
     },
     "bitsyntax": {
       "version": "0.0.4",
@@ -5470,7 +5470,7 @@
     "bonjour": {
       "version": "3.5.0",
       "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz",
-      "integrity": "sha1-jokKGD2O6aI5OzhExpGkK897yfU=",
+      "integrity": "sha512-RaVTblr+OnEli0r/ud8InrU7D+G0y6aJhlxaLa6Pwty4+xoxboF1BsUI45tujvRpbj9dQVoglChqonGAsjEBYg==",
       "dev": true,
       "requires": {
         "array-flatten": "^2.1.0",
@@ -5644,7 +5644,7 @@
     "brorand": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
-      "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=",
+      "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==",
       "dev": true
     },
     "browser-stdout": {
@@ -5700,7 +5700,7 @@
     "browserify-rsa": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
-      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
+      "integrity": "sha512-+YpEyaLDDvvdzIxQ+cCx73r5YEhS3ANGOkiHdyWqW4t3gdeoNEYjSiQwntbU4Uo2/9yRkpYX3SRFeH+7jc2Duw==",
       "dev": true,
       "requires": {
         "bn.js": "^4.1.0",
@@ -5710,7 +5710,7 @@
     "browserify-sign": {
       "version": "4.0.4",
       "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.0.4.tgz",
-      "integrity": "sha1-qk62jl17ZYuqa/alfmMMvXqT0pg=",
+      "integrity": "sha512-D2ItxCwNtLcHRrOCuEDZQlIezlFyUV/N5IYz6TY1svu1noyThFuthoEjzT8ChZe3UEctqnwmykcPhet3Eiz58A==",
       "dev": true,
       "requires": {
         "bn.js": "^4.1.1",
@@ -5819,7 +5819,7 @@
     "buffer-xor": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
-      "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=",
+      "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==",
       "dev": true
     },
     "bufferedstream": {
@@ -5850,7 +5850,7 @@
     "builtin-status-codes": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
-      "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=",
+      "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==",
       "dev": true
     },
     "bunyan": {
@@ -6499,7 +6499,7 @@
     "commondir": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
-      "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
+      "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==",
       "dev": true
     },
     "component-bind": {
@@ -6606,7 +6606,7 @@
         "vary": {
           "version": "1.1.2",
           "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
-          "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=",
+          "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==",
           "dev": true
         }
       }
@@ -6786,13 +6786,13 @@
     "constants-browserify": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
-      "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=",
+      "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==",
       "dev": true
     },
     "contains-path": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz",
-      "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=",
+      "integrity": "sha512-OKZnPGeMQy2RPaUIBPFFd71iNf4791H12MCRuVQDnzGRwCYNYmTDy5pdafo2SLAcEMKzTOQnLWG4QdcjeJUMEg==",
       "dev": true
     },
     "content-disposition": {
@@ -8106,7 +8106,7 @@
     "detect-file": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz",
-      "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=",
+      "integrity": "sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q==",
       "dev": true
     },
     "detect-node": {
@@ -8193,7 +8193,7 @@
     "dns-equal": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz",
-      "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0=",
+      "integrity": "sha512-z+paD6YUQsk+AbGCEM4PrOXSss5gd66QfcVBFTKR/HpFL9jCqikS94HYwKww6fQyO7IxrIIyUu+g0Ka9tUS2Cg==",
       "dev": true
     },
     "dns-packet": {
@@ -8214,7 +8214,7 @@
     "dns-txt": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz",
-      "integrity": "sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=",
+      "integrity": "sha512-Ix5PrWjphuSoUXV/Zv5gaFHjnaJtb02F2+Si3Ht9dyJ87+Z/lMmy+dpNHtTGraNK958ndXq2i+GLkWsWHcKaBQ==",
       "dev": true,
       "requires": {
         "buffer-indexof": "^1.0.0"
@@ -9727,7 +9727,7 @@
     "expand-tilde": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz",
-      "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=",
+      "integrity": "sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==",
       "dev": true,
       "requires": {
         "homedir-polyfill": "^1.0.1"
@@ -10062,7 +10062,7 @@
     "faye-websocket": {
       "version": "0.10.0",
       "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.10.0.tgz",
-      "integrity": "sha1-TkkvjQTftviQA1B/btvy1QHnxvQ=",
+      "integrity": "sha512-Xhj93RXbMSq8urNCUq4p9l0P6hnySJ/7YNRhYNug0bLOuii7pKO7xQFb5mx9xZXWCar88pLPb805PvUkwrLZpQ==",
       "dev": true,
       "requires": {
         "websocket-driver": ">=0.5.1"
@@ -10366,7 +10366,7 @@
     "findit2": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/findit2/-/findit2-2.2.3.tgz",
-      "integrity": "sha1-WKRmaX34piBc39vzlVNri9d3pfY="
+      "integrity": "sha512-lg/Moejf4qXovVutL0Lz4IsaPoNYMuxt4PA0nGqFxnJ1CTTGGlEO2wKgoDpwknhvZ8k4Q2F+eesgkLbG2Mxfog=="
     },
     "findup-sync": {
       "version": "0.1.3",
@@ -10432,7 +10432,7 @@
     "flowstate": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/flowstate/-/flowstate-0.4.1.tgz",
-      "integrity": "sha1-tfu4t/wte9xbVL5GyYMJ73NvTsA=",
+      "integrity": "sha512-U67AgveyMwXFIiDgs6Yz/PrUNrZGLJUUMDwJ9Q0fDFTQSzyDg8Jj9YDyZIUnFZKggQZONVueK9+grp/Gxa/scw==",
       "dev": true,
       "requires": {
         "clone": "^1.0.2",
@@ -10453,7 +10453,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -10574,7 +10574,7 @@
     "from2": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
-      "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=",
+      "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==",
       "dev": true,
       "requires": {
         "inherits": "^2.0.1",
@@ -10584,7 +10584,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -10660,7 +10660,7 @@
     "fs-write-stream-atomic": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz",
-      "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=",
+      "integrity": "sha512-gehEzmPn2nAwr39eay+x3X34Ra+M2QlVUTLhkXPjWdeO8RF9kszk116avgBJM3ZyNHgHXBNx+VmPaFC36k0PzA==",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.2",
@@ -11504,7 +11504,7 @@
     "global-prefix": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz",
-      "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=",
+      "integrity": "sha512-5lsx1NUDHtSjfg0eHlmYvZKv8/nVqX4ckFbM+FrGcQ+04KWcWFo9P5MxPZYSzUvyzmdTbI7Eix8Q4IbELDqzKg==",
       "dev": true,
       "requires": {
         "expand-tilde": "^2.0.2",
@@ -12136,7 +12136,7 @@
     "hash-base": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz",
-      "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=",
+      "integrity": "sha512-EeeoJKjTyt868liAlVmcv2ZsUfGHlE3Q+BICOXcZiwN3osr5Q/zFGYmTJpoIzuaSTAwndFy+GqhEwlU4L3j4Ow==",
       "dev": true,
       "requires": {
         "inherits": "^2.0.1",
@@ -12233,7 +12233,7 @@
     "hmac-drbg": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
-      "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=",
+      "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==",
       "dev": true,
       "requires": {
         "hash.js": "^1.0.3",
@@ -12264,7 +12264,7 @@
     "hpack.js": {
       "version": "2.1.6",
       "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz",
-      "integrity": "sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI=",
+      "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==",
       "dev": true,
       "requires": {
         "inherits": "^2.0.1",
@@ -12276,7 +12276,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -12394,7 +12394,7 @@
     "http-deceiver": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz",
-      "integrity": "sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc=",
+      "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==",
       "dev": true
     },
     "http-errors": {
@@ -12418,7 +12418,7 @@
     "http-parser-js": {
       "version": "0.4.10",
       "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.10.tgz",
-      "integrity": "sha1-ksnBN0w1CF912zWexWzCV8u5P6Q=",
+      "integrity": "sha512-ln7+HeZl3lL3PNRX9Y6ub4i8xcgQ0mO2J//ic97dR7tEXB+6IKAjx8JCCmEkwKiMcR2jidU9xNolz1fEyyf/Jg==",
       "dev": true
     },
     "http-proxy": {
@@ -12469,13 +12469,13 @@
         "arr-diff": {
           "version": "4.0.0",
           "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-          "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=",
+          "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==",
           "dev": true
         },
         "array-unique": {
           "version": "0.3.2",
           "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
+          "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==",
           "dev": true
         },
         "debug": {
@@ -12496,7 +12496,7 @@
         "expand-brackets": {
           "version": "2.1.4",
           "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-          "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
+          "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==",
           "dev": true,
           "requires": {
             "debug": "^2.3.3",
@@ -12511,7 +12511,7 @@
             "define-property": {
               "version": "0.2.5",
               "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-              "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
+              "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==",
               "dev": true,
               "requires": {
                 "is-descriptor": "^0.1.0"
@@ -12520,7 +12520,7 @@
             "extend-shallow": {
               "version": "2.0.1",
               "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+              "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
               "dev": true,
               "requires": {
                 "is-extendable": "^0.1.0"
@@ -12529,7 +12529,7 @@
             "is-accessor-descriptor": {
               "version": "0.1.6",
               "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+              "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==",
               "dev": true,
               "requires": {
                 "kind-of": "^3.0.2"
@@ -12538,7 +12538,7 @@
                 "kind-of": {
                   "version": "3.2.2",
                   "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
                   "dev": true,
                   "requires": {
                     "is-buffer": "^1.1.5"
@@ -12549,7 +12549,7 @@
             "is-data-descriptor": {
               "version": "0.1.4",
               "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+              "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==",
               "dev": true,
               "requires": {
                 "kind-of": "^3.0.2"
@@ -12558,7 +12558,7 @@
                 "kind-of": {
                   "version": "3.2.2",
                   "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
                   "dev": true,
                   "requires": {
                     "is-buffer": "^1.1.5"
@@ -12604,7 +12604,7 @@
             "define-property": {
               "version": "1.0.0",
               "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
+              "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==",
               "dev": true,
               "requires": {
                 "is-descriptor": "^1.0.0"
@@ -12613,7 +12613,7 @@
             "extend-shallow": {
               "version": "2.0.1",
               "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
+              "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
               "dev": true,
               "requires": {
                 "is-extendable": "^0.1.0"
@@ -12736,7 +12736,7 @@
     "https-browserify": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
-      "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=",
+      "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==",
       "dev": true
     },
     "https-proxy-agent": {
@@ -12810,7 +12810,7 @@
     "iferr": {
       "version": "0.1.5",
       "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz",
-      "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
+      "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==",
       "dev": true
     },
     "ignore": {
@@ -13261,7 +13261,7 @@
     "ip-regex": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
-      "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=",
+      "integrity": "sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==",
       "dev": true
     },
     "ipaddr.js": {
@@ -13643,7 +13643,7 @@
     "is-wsl": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz",
-      "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=",
+      "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==",
       "dev": true
     },
     "isarray": {
@@ -13722,7 +13722,7 @@
     "jmespath": {
       "version": "0.15.0",
       "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
-      "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
+      "integrity": "sha512-+kHj8HXArPfpPEKGLZ+kB5ONRTCiGQXo8RQYL0hH8t6pWXUBBK5KkkQmTNOwKK4LEsd0yTsgtjJVm4UBSZea4w=="
     },
     "joi-mongodb-objectid": {
       "version": "0.1.0",
@@ -13790,7 +13790,7 @@
     "json-bigint": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz",
-      "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=",
+      "integrity": "sha512-u+c/u/F+JNPUekHCFyGVycRPyh9UHD5iUhSyIAn10kxbDTJxijwAbT6XHaONEOXuGGfmWUSroheXgHcml4gLgg==",
       "requires": {
         "bignumber.js": "^7.0.0"
       }
@@ -14757,7 +14757,7 @@
     "load-json-file": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
-      "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=",
+      "integrity": "sha512-3p6ZOGNbiX4CdvEd1VcE6yi78UrGNpjHO33noGwHCnT/o2fyllJDepsm8+mFFv/DvtwFHht5HIHSyOy5a+ChVQ==",
       "dev": true,
       "requires": {
         "graceful-fs": "^4.1.2",
@@ -14769,7 +14769,7 @@
         "pify": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
           "dev": true
         }
       }
@@ -14914,7 +14914,7 @@
     "lodash.pickby": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz",
-      "integrity": "sha1-feoh2MGNdwOifHBMFdO4SmfjOv8="
+      "integrity": "sha512-AZV+GsS/6ckvPOVQPXSiFFacKvKB4kOQu6ynt9wz0F3LO4R9Ij4K1ddYsIytDpSgLz88JHd9P+oaLeej5/Sl7Q=="
     },
     "lodash.set": {
       "version": "4.3.2",
@@ -15556,7 +15556,7 @@
     "lynx": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/lynx/-/lynx-0.1.1.tgz",
-      "integrity": "sha1-Mxjc7xaQi4KG6Bisz9sxzXQkj50=",
+      "integrity": "sha512-JI52N0NwK2b/Md0TFPdPtUBI46kjyJXF7+q08l2yvQ56q6QA8s7ZjZQQRoxFpS2jDXNf/B0p8ID+OIKcTsZwzw==",
       "requires": {
         "mersenne": "~0.0.3",
         "statsd-parser": "~0.0.4"
@@ -15833,7 +15833,7 @@
     "mersenne": {
       "version": "0.0.4",
       "resolved": "https://registry.npmjs.org/mersenne/-/mersenne-0.0.4.tgz",
-      "integrity": "sha1-QB/ex+whzbngPNPTAhOY2iGycIU="
+      "integrity": "sha512-XoSUL+nF8hMTKGQxUs8r3Btdsf1yuKKBdCCGbh3YXgCXuVKishpZv1CNc385w9s8t4Ynwc5h61BwW/FCVulkbg=="
     },
     "messageformat": {
       "version": "1.1.1",
@@ -16093,7 +16093,7 @@
     "minimalistic-crypto-utils": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz",
-      "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=",
+      "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==",
       "dev": true
     },
     "minimatch": {
@@ -16247,7 +16247,7 @@
         "once": {
           "version": "1.4.0",
           "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-          "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+          "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
           "dev": true,
           "requires": {
             "wrappy": "1"
@@ -16407,10 +16407,16 @@
         }
       }
     },
+    "mock-fs": {
+      "version": "4.11.0",
+      "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-4.11.0.tgz",
+      "integrity": "sha512-Yp4o3/ZA15wsXqJTT+R+9w2AYIkD1i80Lds47wDbuUhOvQvm+O2EfjFZSz0pMgZZSPHRhGxgcd2+GL4+jZMtdw==",
+      "dev": true
+    },
     "module-details-from-path": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz",
-      "integrity": "sha1-EUyUlnPiqKNenTV4hSeqN7Z52is="
+      "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A=="
     },
     "moment": {
       "version": "2.24.0",
@@ -16572,7 +16578,7 @@
     "move-concurrently": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz",
-      "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=",
+      "integrity": "sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==",
       "dev": true,
       "requires": {
         "aproba": "^1.1.1",
@@ -16703,7 +16709,7 @@
     "multicast-dns-service-types": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz",
-      "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE=",
+      "integrity": "sha512-cnAsSVxIDsYt0v7HmC0hWZFwwXSh+E6PgCrREDuN/EsjgLwA5XRmlMHhSiDPrt6HxY1gTivEa/Zh7GtODoLevQ==",
       "dev": true
     },
     "muri": {
@@ -16990,7 +16996,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -17045,7 +17051,7 @@
         "url": {
           "version": "0.11.0",
           "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
-          "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
+          "integrity": "sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==",
           "dev": true,
           "requires": {
             "punycode": "1.3.2",
@@ -17055,7 +17061,7 @@
             "punycode": {
               "version": "1.3.2",
               "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-              "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=",
+              "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==",
               "dev": true
             }
           }
@@ -18541,7 +18547,7 @@
     "os-browserify": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz",
-      "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=",
+      "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==",
       "dev": true
     },
     "os-homedir": {
@@ -18587,7 +18593,7 @@
     "p-defer": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz",
-      "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=",
+      "integrity": "sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==",
       "dev": true
     },
     "p-each-series": {
@@ -18852,7 +18858,7 @@
     "parse-json": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
-      "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=",
+      "integrity": "sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==",
       "dev": true,
       "requires": {
         "error-ex": "^1.2.0"
@@ -18871,7 +18877,7 @@
     "parse-passwd": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz",
-      "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=",
+      "integrity": "sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==",
       "dev": true
     },
     "parse5": {
@@ -19096,7 +19102,7 @@
     "path-type": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz",
-      "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=",
+      "integrity": "sha512-dUnb5dXUf+kzhC/W/F4e5/SkluXIFf5VUHolW1Eg1irn1hGWjPGdsRcvYJ1nD6lhk8Ir7VM0bHJKsYTx8Jx9OQ==",
       "dev": true,
       "requires": {
         "pify": "^2.0.0"
@@ -19105,7 +19111,7 @@
         "pify": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
+          "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==",
           "dev": true
         }
       }
@@ -20174,7 +20180,7 @@
     "promise-inflight": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
-      "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=",
+      "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==",
       "dev": true
     },
     "promisify-any": {
@@ -20863,7 +20869,7 @@
     "read-pkg": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz",
-      "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=",
+      "integrity": "sha512-eFIBOPW7FGjzBuk3hdXEuNSiTZS/xEMlH49HxMyzb0hyPfu4EhVjT2DH32K1hSSmVq4sebAWnZuuY5auISUTGA==",
       "dev": true,
       "requires": {
         "load-json-file": "^2.0.0",
@@ -20874,7 +20880,7 @@
     "read-pkg-up": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz",
-      "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=",
+      "integrity": "sha512-1orxQfbWGUiTn9XsPlChs6rLie/AV9jwZTGmu2NZw/CUDJQchXJFYE0Fq5j7+n558T1JhDWLdhyd1Zj+wLY//w==",
       "dev": true,
       "requires": {
         "find-up": "^2.0.0",
@@ -21818,7 +21824,7 @@
     "resolve-cwd": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz",
-      "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=",
+      "integrity": "sha512-ccu8zQTrzVr954472aUVPLEcB3YpKSYR3cg/3lo1okzobPBM+1INXBbBZlDbnI/hbEocnf8j0QVo43hQKrbchg==",
       "dev": true,
       "requires": {
         "resolve-from": "^3.0.0"
@@ -21827,7 +21833,7 @@
         "resolve-from": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
-          "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=",
+          "integrity": "sha512-GnlH6vxLymXJNMBo7XP1fJIzBFbdYt49CuTwmB/6N53t+kMPRMFKz783LlQ4tv28XoQfMWinAJX6WCGf2IlaIw==",
           "dev": true
         }
       }
@@ -21835,7 +21841,7 @@
     "resolve-dir": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz",
-      "integrity": "sha1-eaQGRMNivoLybv/nOcm7U4IEb0M=",
+      "integrity": "sha512-R7uiTjECzvOsWSfdM0QKFNBVFcK27aHOUwdvK53BcW8zqnGdYp0Fbj82cy54+2A4P2tFM22J5kRfe1R+lM/1yg==",
       "dev": true,
       "requires": {
         "expand-tilde": "^2.0.0",
@@ -21883,7 +21889,7 @@
     "retry": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=",
+      "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==",
       "dev": true
     },
     "retry-axios": {
@@ -22013,7 +22019,7 @@
     "run-queue": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",
-      "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=",
+      "integrity": "sha512-ntymy489o0/QQplUDnpYAYUsO50K9SBrIVaKCWDOJzYJts0f9WH9RFJkyagebkw5+y1oi00R7ynNW/d12GBumg==",
       "dev": true,
       "requires": {
         "aproba": "^1.1.1"
@@ -22070,7 +22076,7 @@
     "saml": {
       "version": "0.12.5",
       "resolved": "https://registry.npmjs.org/saml/-/saml-0.12.5.tgz",
-      "integrity": "sha1-pDyQhUifefceg94uxato3sxrjME=",
+      "integrity": "sha512-723DD6x293D01zvQP4D6Otu207VZXF1t6t15MCvR3SM5vj+DycuoO5mePnD1VexUjbgVeycCiwwK6PQzfSKVnA==",
       "dev": true,
       "requires": {
         "async": "~0.2.9",
@@ -22086,19 +22092,19 @@
         "async": {
           "version": "0.2.10",
           "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz",
-          "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=",
+          "integrity": "sha512-eAkdoKxU6/LkKDBzLpT+t6Ff5EtfSF4wx1WfJiPEEV7WNLnDaRXk0oVysiEPm262roaachGexwUv94WhSgN5TQ==",
           "dev": true
         },
         "moment": {
           "version": "2.15.2",
           "resolved": "https://registry.npmjs.org/moment/-/moment-2.15.2.tgz",
-          "integrity": "sha1-G/3t9qbjRfMi/pVtXfW9CKjOhNw=",
+          "integrity": "sha512-dv9NAmbJRSckFY2Dt3EcgoUGg85U4AaUvtJQ56k0QFumwqpOK3Huf0pYutSVgCFfN+DekvF4pW45PP9rf6ts7g==",
           "dev": true
         },
         "xml-crypto": {
           "version": "0.10.1",
           "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-0.10.1.tgz",
-          "integrity": "sha1-+DL3TM9W8kr8rhFjofyrRNlndKg=",
+          "integrity": "sha512-w64qUhByslUJ9D9nwfCyRUCXfVWA5WdzHevHT3BwAig2KOsDNYcuvE2soGUGUs0qp9cy+vGG6B/Ap8qCXPLN/g==",
           "dev": true,
           "requires": {
             "xmldom": "=0.1.19",
@@ -22108,7 +22114,7 @@
             "xmldom": {
               "version": "0.1.19",
               "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.19.tgz",
-              "integrity": "sha1-Yx/Ad3bv2EEYvyUXGzftTQdaCrw=",
+              "integrity": "sha512-pDyxjQSFQgNHkU+yjvoF+GXVGJU7e9EnOg/KcGMDihBIKjTsOeDYaECwC/O9bsUWKY+Sd9izfE43JXC46EOHKA==",
               "dev": true
             }
           }
@@ -22116,13 +22122,13 @@
         "xmldom": {
           "version": "0.1.15",
           "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.1.15.tgz",
-          "integrity": "sha1-swSAYvG91S7cQhQkRZ8G3O6y+U0=",
+          "integrity": "sha512-ssWmE9kBZudhl4iPLmXqaShPuASNKIQIikBzsloOjZqMyfbuQRn/ggz0k9NDa9YFI3+oFvp8t7TsqwmZLTvpoA==",
           "dev": true
         },
         "xpath": {
           "version": "0.0.5",
           "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.5.tgz",
-          "integrity": "sha1-RUA29u8PPfWvXUukoRn7dWdLPmw=",
+          "integrity": "sha512-Y1Oyy8lyIDwWpmKIWBF0RZrQOP1fzE12G0ekSB1yzKPtbAdCI5sBCqBU/CAZUkKk81OXuq9tul/5lyNS+22iKg==",
           "dev": true
         }
       }
@@ -22153,7 +22159,7 @@
         "xml-crypto": {
           "version": "0.10.1",
           "resolved": "https://registry.npmjs.org/xml-crypto/-/xml-crypto-0.10.1.tgz",
-          "integrity": "sha1-+DL3TM9W8kr8rhFjofyrRNlndKg=",
+          "integrity": "sha512-w64qUhByslUJ9D9nwfCyRUCXfVWA5WdzHevHT3BwAig2KOsDNYcuvE2soGUGUs0qp9cy+vGG6B/Ap8qCXPLN/g==",
           "dev": true,
           "requires": {
             "xmldom": "=0.1.19",
@@ -22176,13 +22182,13 @@
         "xpath": {
           "version": "0.0.5",
           "resolved": "https://registry.npmjs.org/xpath/-/xpath-0.0.5.tgz",
-          "integrity": "sha1-RUA29u8PPfWvXUukoRn7dWdLPmw=",
+          "integrity": "sha512-Y1Oyy8lyIDwWpmKIWBF0RZrQOP1fzE12G0ekSB1yzKPtbAdCI5sBCqBU/CAZUkKk81OXuq9tul/5lyNS+22iKg==",
           "dev": true
         },
         "xtend": {
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/xtend/-/xtend-1.0.3.tgz",
-          "integrity": "sha1-P12Tc1PM7Y4IU5mlY/2yJUHClgo=",
+          "integrity": "sha512-wv78b3q8kHDveC/C7Yq/UUrJXsAAM1t/j5m28h/ZlqYy0+eqByglhsWR88D2j3VImQzZlNIDsSbZ3QItwgWEGw==",
           "dev": true
         }
       }
@@ -22327,7 +22333,7 @@
     "select-hose": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz",
-      "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo=",
+      "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==",
       "dev": true
     },
     "selfsigned": {
@@ -22444,7 +22450,7 @@
     "serve-index": {
       "version": "1.9.1",
       "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz",
-      "integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=",
+      "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==",
       "dev": true,
       "requires": {
         "accepts": "~1.3.4",
@@ -22489,7 +22495,7 @@
         "escape-html": {
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
-          "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=",
+          "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
           "dev": true
         },
         "mime-db": {
@@ -23408,7 +23414,7 @@
     "statsd-parser": {
       "version": "0.0.4",
       "resolved": "https://registry.npmjs.org/statsd-parser/-/statsd-parser-0.0.4.tgz",
-      "integrity": "sha1-y9JDlTzELv/VSLXSI4jtaJ7GOb0="
+      "integrity": "sha512-7XO+ur89EalMXXFQaydsczB8sclr5nDsNIoUu0IzJx1pIbHUhO3LtpSzBwetIuU9DyTLMiVaJBMtWS/Nb2KR4g=="
     },
     "statuses": {
       "version": "1.4.0",
@@ -23433,7 +23439,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -23499,7 +23505,7 @@
         "once": {
           "version": "1.4.0",
           "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-          "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+          "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
           "dev": true,
           "requires": {
             "wrappy": "1"
@@ -23531,7 +23537,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -24086,7 +24092,7 @@
     "strip-bom": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-      "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=",
+      "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==",
       "dev": true
     },
     "strip-eof": {
@@ -24345,7 +24351,7 @@
     "tdigest": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz",
-      "integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=",
+      "integrity": "sha512-CXcDY/NIgIbKZPx5H4JJNpq6JwJhU5Z4+yWj4ZghDc7/9nVajiRlPPyMXRePPPlBfcayUqtoCXjo7/Hm82ecUA==",
       "requires": {
         "bintrees": "1.0.1"
       }
@@ -24689,7 +24695,7 @@
         "isarray": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-          "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
+          "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
           "dev": true
         },
         "process-nextick-args": {
@@ -24788,7 +24794,7 @@
     "to-arraybuffer": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz",
-      "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=",
+      "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==",
       "dev": true
     },
     "to-fast-properties": {
@@ -25060,7 +25066,7 @@
     "tty-browserify": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
-      "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=",
+      "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==",
       "dev": true
     },
     "tunnel-agent": {
@@ -25508,7 +25514,7 @@
     "url": {
       "version": "0.10.3",
       "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
-      "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
+      "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==",
       "requires": {
         "punycode": "1.3.2",
         "querystring": "0.2.0"
@@ -25517,7 +25523,7 @@
         "punycode": {
           "version": "1.3.2",
           "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-          "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
+          "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw=="
         }
       }
     },
@@ -25619,7 +25625,7 @@
     "utils-flatten": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/utils-flatten/-/utils-flatten-1.0.0.tgz",
-      "integrity": "sha1-AfMNMZO+RkxAsxdV5nQNDbDO8kM=",
+      "integrity": "sha512-s21PUgUZ+XPvH8Wi8aj2FEqzZWeNEdemP7LB4u8u5wTDRO4xB+7czAYd3FY2O2rnu89U//khR0Ce8ka3//6M0w==",
       "dev": true
     },
     "utils-merge": {
@@ -27924,7 +27930,7 @@
     "xml-name-validator": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-2.0.1.tgz",
-      "integrity": "sha1-TYuPHszTQZqjYgYb7O9RXh5VljU=",
+      "integrity": "sha512-jRKe/iQYMyVJpzPH+3HL97Lgu5HrCfii+qSo+TfjKHtOnvbnvdVfMYrn9Q34YV81M2e5sviJlI6Ko9y+nByzvA==",
       "dev": true
     },
     "xml2js": {

services/web/package.json

@@ -169,6 +169,7 @@
     "less-plugin-autoprefix": "^2.0.0",
     "mini-css-extract-plugin": "^0.8.0",
     "mkdirp": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+    "mock-fs": "^4.11.0",
     "nodemon": "^1.14.3",
     "optimize-css-assets-webpack-plugin": "^5.0.3",
     "postcss-loader": "^3.0.0",

services/web/test/acceptance/src/ProjectStructureTests.js

@@ -424,7 +424,7 @@ describe('ProjectStructureChanges', function() {
       expect(update.userId).to.equal(owner._id)
       expect(update.pathname).to.equal('/main.tex')
       expect(update.docLines).to.equal('Test')
-      expect(version).to.equal(2)
+      expect(version).to.equal(1)
     })
 
     it('should version the files created', function() {
@@ -437,7 +437,7 @@ describe('ProjectStructureChanges', function() {
       expect(update.userId).to.equal(owner._id)
       expect(update.pathname).to.equal('/1pixel.png')
       expect(update.url).to.be.a('string')
-      expect(version).to.equal(2)
+      expect(version).to.equal(1)
     })
   })
 

services/web/test/unit/src/Project/FolderStructureBuilderTests.js

@@ -36,18 +36,18 @@ describe('FolderStructureBuilder', function() {
     describe('when given documents and files', function() {
       beforeEach(function() {
         const docUploads = [
-          { dirname: '/', doc: { _id: 'doc-1', name: 'main.tex' } },
+          { path: '/main.tex', doc: { _id: 'doc-1', name: 'main.tex' } },
-          { dirname: '/foo', doc: { _id: 'doc-2', name: 'other.tex' } },
+          { path: '/foo/other.tex', doc: { _id: 'doc-2', name: 'other.tex' } },
-          { dirname: '/foo', doc: { _id: 'doc-3', name: 'other.bib' } },
+          { path: '/foo/other.bib', doc: { _id: 'doc-3', name: 'other.bib' } },
           {
-            dirname: '/foo/foo1/foo2',
+            path: '/foo/foo1/foo2/another.tex',
             doc: { _id: 'doc-4', name: 'another.tex' }
           }
         ]
         const fileUploads = [
-          { dirname: '/', fileRef: { _id: 'file-1', name: 'aaa.jpg' } },
+          { path: '/aaa.jpg', file: { _id: 'file-1', name: 'aaa.jpg' } },
-          { dirname: '/foo', fileRef: { _id: 'file-2', name: 'bbb.jpg' } },
+          { path: '/foo/bbb.jpg', file: { _id: 'file-2', name: 'bbb.jpg' } },
-          { dirname: '/bar', fileRef: { _id: 'file-3', name: 'ccc.jpg' } }
+          { path: '/bar/ccc.jpg', file: { _id: 'file-3', name: 'ccc.jpg' } }
         ]
         this.result = this.FolderStructureBuilder.buildFolderStructure(
           docUploads,
@@ -103,8 +103,8 @@ describe('FolderStructureBuilder', function() {
     describe('when given duplicate files', function() {
       it('throws an error', function() {
         const docUploads = [
-          { dirname: '/foo', doc: { _id: 'doc-1', name: 'doc.tex' } },
+          { path: '/foo/doc.tex', doc: { _id: 'doc-1', name: 'doc.tex' } },
-          { dirname: '/foo', doc: { _id: 'doc-2', name: 'doc.tex' } }
+          { path: '/foo/doc.tex', doc: { _id: 'doc-2', name: 'doc.tex' } }
         ]
         expect(() =>
           this.FolderStructureBuilder.buildFolderStructure(docUploads, [])

services/web/test/unit/src/Project/ProjectEntityMongoUpdateHandlerTests.js

@@ -1045,7 +1045,7 @@ describe('ProjectEntityMongoUpdateHandler', function() {
       this.FolderStructureBuilder.buildFolderStructure
         .withArgs(this.docUploads, this.fileUploads)
         .returns(this.mockRootFolder)
-      this.updateExpectation = this.ProjectMock.expects('updateOne')
+      this.updateExpectation = this.ProjectMock.expects('findOneAndUpdate')
         .withArgs(
           {
             _id: this.project._id,
@@ -1053,14 +1053,15 @@ describe('ProjectEntityMongoUpdateHandler', function() {
             'rootFolder.0.docs.0': { $exists: false },
             'rootFolder.0.files.0': { $exists: false }
           },
-          { $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } }
+          { $set: { rootFolder: [this.mockRootFolder] }, $inc: { version: 1 } },
+          { new: true, lean: true, fields: { version: 1 } }
         )
         .chain('exec')
     })
 
     describe('happy path', function() {
       beforeEach(async function() {
-        this.updateExpectation.resolves({ n: 1 })
+        this.updateExpectation.resolves({ version: 1 })
         await this.subject.promises.createNewFolderStructure(
           this.project._id,
           this.docUploads,
@@ -1075,7 +1076,7 @@ describe('ProjectEntityMongoUpdateHandler', function() {
 
     describe("when the update doesn't find a matching document", function() {
       beforeEach(async function() {
-        this.updateExpectation.resolves({ n: 0 })
+        this.updateExpectation.resolves(null)
       })
 
       it('throws an error', async function() {

services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js

@@ -1,5 +1,6 @@
 const sinon = require('sinon')
 const { expect } = require('chai')
+const mockFs = require('mock-fs')
 const SandboxedModule = require('sandboxed-module')
 const { ObjectId } = require('mongodb')
 
@@ -13,50 +14,6 @@ describe('FileSystemImportManager', function() {
     this.newFolderId = new ObjectId()
     this.userId = new ObjectId()
 
-    this.folderPath = '/path/to/folder'
-    this.docName = 'test-doc.tex'
-    this.docPath = `/path/to/folder/${this.docName}`
-    this.docContent = 'one\ntwo\nthree'
-    this.docLines = this.docContent.split('\n')
-    this.fileName = 'test-file.jpg'
-    this.filePath = `/path/to/folder/${this.fileName}`
-    this.symlinkName = 'symlink'
-    this.symlinkPath = `/path/to/${this.symlinkName}`
-    this.ignoredName = '.DS_Store'
-    this.ignoredPath = `/path/to/folder/${this.ignoredName}`
-    this.folderEntries = [this.ignoredName, this.docName, this.fileName]
-
-    this.encoding = 'latin1'
-
-    this.fileStat = {
-      isFile: sinon.stub().returns(true),
-      isDirectory: sinon.stub().returns(false)
-    }
-    this.dirStat = {
-      isFile: sinon.stub().returns(false),
-      isDirectory: sinon.stub().returns(true)
-    }
-    this.symlinkStat = {
-      isFile: sinon.stub().returns(false),
-      isDirectory: sinon.stub().returns(false)
-    }
-    this.fs = {
-      promises: {
-        lstat: sinon.stub(),
-        readFile: sinon.stub(),
-        readdir: sinon.stub()
-      }
-    }
-    this.fs.promises.lstat.withArgs(this.filePath).resolves(this.fileStat)
-    this.fs.promises.lstat.withArgs(this.docPath).resolves(this.fileStat)
-    this.fs.promises.lstat.withArgs(this.symlinkPath).resolves(this.symlinkStat)
-    this.fs.promises.lstat.withArgs(this.folderPath).resolves(this.dirStat)
-    this.fs.promises.readFile
-      .withArgs(this.docPath, this.encoding)
-      .resolves(this.docContent)
-    this.fs.promises.readdir
-      .withArgs(this.folderPath)
-      .resolves(this.folderEntries)
     this.EditorController = {
       promises: {
         addDoc: sinon.stub().resolves(),
@@ -66,25 +23,6 @@ describe('FileSystemImportManager', function() {
         addFolder: sinon.stub().resolves({ _id: this.newFolderId })
       }
     }
-    this.FileTypeManager = {
-      promises: {
-        isDirectory: sinon.stub().resolves(false),
-        getType: sinon.stub(),
-        shouldIgnore: sinon.stub().resolves(false)
-      }
-    }
-    this.FileTypeManager.promises.getType
-      .withArgs(this.fileName, this.filePath)
-      .resolves({ binary: true })
-    this.FileTypeManager.promises.getType
-      .withArgs(this.docName, this.docPath)
-      .resolves({ binary: false, encoding: this.encoding })
-    this.FileTypeManager.promises.isDirectory
-      .withArgs(this.folderPath)
-      .resolves(true)
-    this.FileTypeManager.promises.shouldIgnore
-      .withArgs(this.ignoredName)
-      .resolves(true)
     this.logger = {
       log() {},
       err() {}
@@ -94,40 +32,165 @@ describe('FileSystemImportManager', function() {
         console: console
       },
       requires: {
-        fs: this.fs,
         '../Editor/EditorController': this.EditorController,
-        './FileTypeManager': this.FileTypeManager,
         'logger-sharelatex': this.logger
       }
     })
   })
 
-  describe('addFolderContents', function() {
+  describe('importDir', function() {
-    describe('successfully', function() {
+    beforeEach(async function() {
+      mockFs({
+        'import-test': {
+          'main.tex': 'My thesis',
+          'link-to-main.tex': mockFs.symlink({ path: 'import-test/main.tex' }),
+          '.DS_Store': 'Should be ignored',
+          images: {
+            'cat.jpg': Buffer.from([1, 2, 3, 4])
+          },
+          'line-endings': {
+            'unix.txt': 'one\ntwo\nthree',
+            'mac.txt': 'uno\rdos\rtres',
+            'windows.txt': 'ein\r\nzwei\r\ndrei',
+            'mixed.txt': 'uno\rdue\r\ntre\nquattro'
+          },
+          encodings: {
+            'utf16le.txt': Buffer.from('\ufeffétonnant!', 'utf16le'),
+            'latin1.txt': Buffer.from('tétanisant!', 'latin1')
+          }
+        },
+        symlink: mockFs.symlink({ path: 'import-test' })
+      })
+      this.entries = await this.FileSystemImportManager.promises.importDir(
+        'import-test'
+      )
+      this.projectPaths = this.entries.map(x => x.projectPath)
+    })
+
+    afterEach(function() {
+      mockFs.restore()
+    })
+
+    it('should import regular docs', function() {
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/main.tex',
+        lines: ['My thesis']
+      })
+    })
+
+    it('should skip symlinks inside the import folder', function() {
+      expect(this.projectPaths).not.to.include('/link-to-main.tex')
+    })
+
+    it('should skip ignored files', function() {
+      expect(this.projectPaths).not.to.include('/.DS_Store')
+    })
+
+    it('should import binary files', function() {
+      expect(this.entries).to.deep.include({
+        type: 'file',
+        projectPath: '/images/cat.jpg',
+        fsPath: 'import-test/images/cat.jpg'
+      })
+    })
+
+    it('should deal with Mac/Windows/Unix line endings', function() {
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/line-endings/unix.txt',
+        lines: ['one', 'two', 'three']
+      })
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/line-endings/mac.txt',
+        lines: ['uno', 'dos', 'tres']
+      })
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/line-endings/windows.txt',
+        lines: ['ein', 'zwei', 'drei']
+      })
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/line-endings/mixed.txt',
+        lines: ['uno', 'due', 'tre', 'quattro']
+      })
+    })
+
+    it('should import documents with latin1 encoding', function() {
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/encodings/latin1.txt',
+        lines: ['tétanisant!']
+      })
+    })
+
+    it('should import documents with utf16-le encoding', function() {
+      expect(this.entries).to.deep.include({
+        type: 'doc',
+        projectPath: '/encodings/utf16le.txt',
+        lines: ['\ufeffétonnant!']
+      })
+    })
+
+    it('should error when the root folder is a symlink', async function() {
+      await expect(this.FileSystemImportManager.promises.importDir('symlink'))
+        .to.be.rejected
+    })
+  })
+
+  describe('addEntity', function() {
+    describe('with directory', function() {
       beforeEach(async function() {
-        await this.FileSystemImportManager.promises.addFolderContents(
+        mockFs({
+          path: {
+            to: {
+              folder: {
+                'doc.tex': 'one\ntwo\nthree',
+                'image.jpg': Buffer.from([1, 2, 3, 4])
+              }
+            }
+          }
+        })
+
+        await this.FileSystemImportManager.promises.addEntity(
           this.userId,
           this.projectId,
           this.folderId,
-          this.folderPath,
+          'folder',
+          'path/to/folder',
           false
         )
       })
 
-      it('should add each file in the folder which is not ignored', function() {
+      afterEach(function() {
-        this.EditorController.promises.addDoc.should.have.been.calledWith(
+        mockFs.restore()
+      })
+
+      it('should add a folder to the project', function() {
+        this.EditorController.promises.addFolder.should.have.been.calledWith(
           this.projectId,
           this.folderId,
-          this.docName,
+          'folder',
-          this.docLines,
+          'upload'
+        )
+      })
+
+      it("should add the folder's contents", function() {
+        this.EditorController.promises.addDoc.should.have.been.calledWith(
+          this.projectId,
+          this.newFolderId,
+          'doc.tex',
+          ['one', 'two', 'three'],
           'upload',
           this.userId
         )
         this.EditorController.promises.addFile.should.have.been.calledWith(
           this.projectId,
-          this.folderId,
+          this.newFolderId,
-          this.fileName,
+          'image.jpg',
-          this.filePath,
+          'path/to/folder/image.jpg',
           null,
           'upload',
           this.userId
@@ -135,78 +198,23 @@ describe('FileSystemImportManager', function() {
       })
     })
 
-    describe('with symlink', function() {
-      it('should stop with an error', async function() {
-        await expect(
-          this.FileSystemImportManager.promises.addFolderContents(
-            this.userId,
-            this.projectId,
-            this.folderId,
-            this.symlinkPath,
-            false
-          )
-        ).to.be.rejectedWith('path is symlink')
-        this.EditorController.promises.addFolder.should.not.have.been.called
-        this.EditorController.promises.addDoc.should.not.have.been.called
-        this.EditorController.promises.addFile.should.not.have.been.called
-      })
-    })
-  })
-
-  describe('addEntity', function() {
-    describe('with directory', function() {
-      describe('successfully', function() {
-        beforeEach(async function() {
-          await this.FileSystemImportManager.promises.addEntity(
-            this.userId,
-            this.projectId,
-            this.folderId,
-            this.folderName,
-            this.folderPath,
-            false
-          )
-        })
-
-        it('should add a folder to the project', function() {
-          this.EditorController.promises.addFolder.should.have.been.calledWith(
-            this.projectId,
-            this.folderId,
-            this.folderName,
-            'upload'
-          )
-        })
-
-        it("should add the folder's contents", function() {
-          this.EditorController.promises.addDoc.should.have.been.calledWith(
-            this.projectId,
-            this.newFolderId,
-            this.docName,
-            this.docLines,
-            'upload',
-            this.userId
-          )
-          this.EditorController.promises.addFile.should.have.been.calledWith(
-            this.projectId,
-            this.newFolderId,
-            this.fileName,
-            this.filePath,
-            null,
-            'upload',
-            this.userId
-          )
-        })
-      })
-    })
-
     describe('with binary file', function() {
+      beforeEach(function() {
+        mockFs({ 'uploaded-file': Buffer.from([1, 2, 3, 4]) })
+      })
+
+      afterEach(function() {
+        mockFs.restore()
+      })
+
       describe('with replace set to false', function() {
         beforeEach(async function() {
           await this.FileSystemImportManager.promises.addEntity(
             this.userId,
             this.projectId,
             this.folderId,
-            this.fileName,
+            'image.jpg',
-            this.filePath,
+            'uploaded-file',
             false
           )
         })
@@ -215,8 +223,8 @@ describe('FileSystemImportManager', function() {
           this.EditorController.promises.addFile.should.have.been.calledWith(
             this.projectId,
             this.folderId,
-            this.fileName,
+            'image.jpg',
-            this.filePath,
+            'uploaded-file',
             null,
             'upload',
             this.userId
@@ -230,8 +238,8 @@ describe('FileSystemImportManager', function() {
             this.userId,
             this.projectId,
             this.folderId,
-            this.fileName,
+            'image.jpg',
-            this.filePath,
+            'uploaded-file',
             true
           )
         })
@@ -240,24 +248,42 @@ describe('FileSystemImportManager', function() {
           this.EditorController.promises.upsertFile.should.have.been.calledWith(
             this.projectId,
             this.folderId,
-            this.fileName,
+            'image.jpg',
-            this.filePath,
+            'uploaded-file',
             null,
             'upload',
             this.userId
           )
         })
       })
+    })
+
+    for (const [lineEndingDescription, lineEnding] of [
+      ['Unix', '\n'],
+      ['Mac', '\r'],
+      ['Windows', '\r\n']
+    ]) {
+      describe(`with text file (${lineEndingDescription} line endings)`, function() {
+        beforeEach(function() {
+          mockFs({
+            path: {
+              to: { 'uploaded-file': `one${lineEnding}two${lineEnding}three` }
+            }
+          })
+        })
+
+        afterEach(function() {
+          mockFs.restore()
+        })
 
-      describe('with text file', function() {
         describe('with replace set to false', function() {
           beforeEach(async function() {
             await this.FileSystemImportManager.promises.addEntity(
               this.userId,
               this.projectId,
               this.folderId,
-              this.docName,
+              'doc.tex',
-              this.docPath,
+              'path/to/uploaded-file',
               false
             )
           })
@@ -266,66 +292,8 @@ describe('FileSystemImportManager', function() {
             this.EditorController.promises.addDoc.should.have.been.calledWith(
               this.projectId,
               this.folderId,
-              this.docName,
+              'doc.tex',
-              this.docLines,
+              ['one', 'two', 'three'],
-              'upload',
-              this.userId
-            )
-          })
-        })
-
-        describe('with windows line ending', function() {
-          beforeEach(async function() {
-            this.docContent = 'one\r\ntwo\r\nthree'
-            this.docLines = ['one', 'two', 'three']
-            this.fs.promises.readFile
-              .withArgs(this.docPath, this.encoding)
-              .resolves(this.docContent)
-            await this.FileSystemImportManager.promises.addEntity(
-              this.userId,
-              this.projectId,
-              this.folderId,
-              this.docName,
-              this.docPath,
-              false
-            )
-          })
-
-          it('should strip the \\r characters before adding', function() {
-            this.EditorController.promises.addDoc.should.have.been.calledWith(
-              this.projectId,
-              this.folderId,
-              this.docName,
-              this.docLines,
-              'upload',
-              this.userId
-            )
-          })
-        })
-
-        describe('with \r line endings', function() {
-          beforeEach(async function() {
-            this.docContent = 'one\rtwo\rthree'
-            this.docLines = ['one', 'two', 'three']
-            this.fs.promises.readFile
-              .withArgs(this.docPath, this.encoding)
-              .resolves(this.docContent)
-            await this.FileSystemImportManager.promises.addEntity(
-              this.userId,
-              this.projectId,
-              this.folderId,
-              this.docName,
-              this.docPath,
-              false
-            )
-          })
-
-          it('should treat the \\r characters as newlines', function() {
-            this.EditorController.promises.addDoc.should.have.been.calledWith(
-              this.projectId,
-              this.folderId,
-              this.docName,
-              this.docLines,
               'upload',
               this.userId
             )
@@ -338,8 +306,8 @@ describe('FileSystemImportManager', function() {
               this.userId,
               this.projectId,
               this.folderId,
-              this.docName,
+              'doc.tex',
-              this.docPath,
+              'path/to/uploaded-file',
               true
             )
           })
@@ -348,25 +316,35 @@ describe('FileSystemImportManager', function() {
             this.EditorController.promises.upsertDoc.should.have.been.calledWith(
               this.projectId,
               this.folderId,
-              this.docName,
+              'doc.tex',
-              this.docLines,
+              ['one', 'two', 'three'],
               'upload',
               this.userId
             )
           })
         })
       })
-    })
+    }
 
     describe('with symlink', function() {
+      beforeEach(function() {
+        mockFs({
+          path: { to: { symlink: mockFs.symlink({ path: '/etc/passwd' }) } }
+        })
+      })
+
+      afterEach(function() {
+        mockFs.restore()
+      })
+
       it('should stop with an error', async function() {
         await expect(
           this.FileSystemImportManager.promises.addEntity(
             this.userId,
             this.projectId,
             this.folderId,
-            this.symlinkName,
+            'main.tex',
-            this.symlinkPath,
+            'path/to/symlink',
             false
           )
         ).to.be.rejectedWith('path is symlink')

services/web/test/unit/src/Uploads/ProjectUploadManagerTests.js

@@ -2,6 +2,7 @@ const sinon = require('sinon')
 const { expect } = require('chai')
 const timekeeper = require('timekeeper')
 const SandboxedModule = require('sandboxed-module')
+const { ObjectId } = require('mongodb')
 
 const MODULE_PATH =
   '../../../../app/src/Features/Uploads/ProjectUploadManager.js'
@@ -10,20 +11,55 @@ describe('ProjectUploadManager', function() {
   beforeEach(function() {
     this.now = Date.now()
     timekeeper.freeze(this.now)
-    this.project_id = 'project-id-123'
+    this.rootFolderId = new ObjectId()
-    this.folder_id = 'folder-id-123'
+    this.ownerId = new ObjectId()
-    this.owner_id = 'owner-id-123'
+    this.zipPath = '/path/to/zip/file-name.zip'
-    this.source = '/path/to/zip/file-name.zip'
+    this.extractedZipPath = `/path/to/zip/file-name-${this.now}`
-    this.destination = `/path/to/zip/file-name-${this.now}`
+    this.mainContent = 'Contents of main.tex'
-    this.root_folder_id = this.folder_id
+    this.projectName = 'My project*'
-    this.owner_id = 'owner-id-123'
+    this.fixedProjectName = 'My project'
-    this.name = 'Project name'
+    this.uniqueProjectName = 'My project (1)'
-    this.othername = 'Other name'
     this.project = {
-      _id: this.project_id,
+      _id: new ObjectId(),
-      rootFolder: [{ _id: this.root_folder_id }]
+      rootFolder: [{ _id: this.rootFolderId }],
+      overleaf: { history: { id: 12345 } }
     }
+    this.doc = {
+      _id: new ObjectId(),
+      name: 'main.tex'
+    }
+    this.docFsPath = '/path/to/doc'
+    this.docLines = ['My thesis', 'by A. U. Thor']
+    this.file = {
+      _id: new ObjectId(),
+      name: 'image.png'
+    }
+    this.fileFsPath = '/path/to/file'
+
     this.topLevelDestination = '/path/to/zip/file-extracted/nested'
+    this.newProjectVersion = 123
+    this.importEntries = [
+      {
+        type: 'doc',
+        projectPath: '/main.tex',
+        lines: this.docLines
+      },
+      {
+        type: 'file',
+        projectPath: `/${this.file.name}`,
+        fsPath: this.fileFsPath
+      }
+    ]
+    this.docEntries = [
+      {
+        doc: this.doc,
+        path: `/${this.doc.name}`,
+        docLines: this.docLines.join('\n')
+      }
+    ]
+    this.fileEntries = [
+      { file: this.file, path: `/${this.file.name}`, url: this.fileStoreUrl }
+    ]
 
     this.fs = {
       remove: sinon.stub().resolves()
@@ -31,12 +67,42 @@ describe('ProjectUploadManager', function() {
     this.ArchiveManager = {
       promises: {
         extractZipArchive: sinon.stub().resolves(),
-        findTopLevelDirectory: sinon.stub().resolves(this.topLevelDestination)
+        findTopLevelDirectory: sinon
+          .stub()
+          .withArgs(this.extractedZipPath)
+          .resolves(this.topLevelDestination)
+      }
+    }
+    this.Doc = sinon.stub().returns(this.doc)
+    this.DocstoreManager = {
+      promises: {
+        updateDoc: sinon.stub().resolves()
+      }
+    }
+    this.DocumentHelper = {
+      getTitleFromTexContent: sinon
+        .stub()
+        .withArgs(this.mainContent)
+        .returns(this.projectName)
+    }
+    this.DocumentUpdaterHandler = {
+      promises: {
+        updateProjectStructure: sinon.stub().resolves()
+      }
+    }
+    this.FileStoreHandler = {
+      promises: {
+        uploadFileFromDisk: sinon
+          .stub()
+          .resolves({ fileRef: this.file, url: this.fileStoreUrl })
       }
     }
     this.FileSystemImportManager = {
       promises: {
-        addFolderContents: sinon.stub().resolves()
+        importDir: sinon
+          .stub()
+          .withArgs(this.topLevelDestination)
+          .resolves(this.importEntries)
       }
     }
     this.ProjectCreationHandler = {
@@ -44,19 +110,27 @@ describe('ProjectUploadManager', function() {
         createBlankProject: sinon.stub().resolves(this.project)
       }
     }
+    this.ProjectEntityMongoUpdateHandler = {
+      promises: {
+        createNewFolderStructure: sinon.stub().resolves(this.newProjectVersion)
+      }
+    }
     this.ProjectRootDocManager = {
       promises: {
         setRootDocAutomatically: sinon.stub().resolves(),
         findRootDocFileFromDirectory: sinon
           .stub()
-          .resolves({ path: 'main.tex', content: this.othername }),
+          .resolves({ path: 'main.tex', content: this.mainContent }),
         setRootDocFromName: sinon.stub().resolves()
       }
     }
     this.ProjectDetailsHandler = {
-      fixProjectName: sinon.stub().returnsArg(0),
+      fixProjectName: sinon
+        .stub()
+        .withArgs(this.projectName)
+        .returns(this.fixedProjectName),
       promises: {
-        generateUniqueName: sinon.stub().resolves(this.othername)
+        generateUniqueName: sinon.stub().resolves(this.uniqueProjectName)
       }
     }
     this.ProjectDeleter = {
@@ -64,8 +138,10 @@ describe('ProjectUploadManager', function() {
         deleteProject: sinon.stub().resolves()
       }
     }
-    this.DocumentHelper = {
+    this.TpdsProjectFlusher = {
-      getTitleFromTexContent: sinon.stub().returns(this.othername)
+      promises: {
+        flushProjectToTpds: sinon.stub().resolves()
+      }
     }
 
     this.ProjectUploadManager = SandboxedModule.require(MODULE_PATH, {
@@ -74,13 +150,21 @@ describe('ProjectUploadManager', function() {
       },
       requires: {
         'fs-extra': this.fs,
-        './FileSystemImportManager': this.FileSystemImportManager,
         './ArchiveManager': this.ArchiveManager,
+        '../../models/Doc': { Doc: this.Doc },
+        '../Docstore/DocstoreManager': this.DocstoreManager,
+        '../Documents/DocumentHelper': this.DocumentHelper,
+        '../DocumentUpdater/DocumentUpdaterHandler': this
+          .DocumentUpdaterHandler,
+        '../FileStore/FileStoreHandler': this.FileStoreHandler,
+        './FileSystemImportManager': this.FileSystemImportManager,
         '../Project/ProjectCreationHandler': this.ProjectCreationHandler,
+        '../Project/ProjectEntityMongoUpdateHandler': this
+          .ProjectEntityMongoUpdateHandler,
         '../Project/ProjectRootDocManager': this.ProjectRootDocManager,
         '../Project/ProjectDetailsHandler': this.ProjectDetailsHandler,
         '../Project/ProjectDeleter': this.ProjectDeleter,
-        '../Documents/DocumentHelper': this.DocumentHelper
+        '../ThirdPartyDataStore/TpdsProjectFlusher': this.TpdsProjectFlusher
       }
     })
   })
@@ -93,61 +177,62 @@ describe('ProjectUploadManager', function() {
     describe('when the title can be read from the root document', function() {
       beforeEach(async function() {
         await this.ProjectUploadManager.promises.createProjectFromZipArchive(
-          this.owner_id,
+          this.ownerId,
-          this.name,
+          this.projectName,
-          this.source
+          this.zipPath
         )
       })
 
       it('should extract the archive', function() {
         this.ArchiveManager.promises.extractZipArchive.should.have.been.calledWith(
-          this.source,
+          this.zipPath,
-          this.destination
+          this.extractedZipPath
         )
       })
 
-      it('should find the top level directory', function() {
+      it('should create a project', function() {
-        this.ArchiveManager.promises.findTopLevelDirectory.should.have.been.calledWith(
-          this.destination
-        )
-      })
-
-      it('should insert the extracted archive into the folder', function() {
-        this.FileSystemImportManager.promises.addFolderContents.should.have.been.calledWith(
-          this.owner_id,
-          this.project_id,
-          this.folder_id,
-          this.topLevelDestination,
-          false
-        )
-      })
-
-      it('should create a project owned by the owner_id', function() {
         this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith(
-          this.owner_id
+          this.ownerId,
+          this.uniqueProjectName
         )
       })
 
-      it('should create a project with the correct name', function() {
+      it('should initialize the file tree', function() {
-        this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith(
+        this.ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure.should.have.been.calledWith(
-          sinon.match.any,
+          this.project._id,
-          this.othername
+          this.docEntries,
+          this.fileEntries
         )
       })
 
-      it('should read the title from the tex contents', function() {
+      it('should notify document updater', function() {
-        this.DocumentHelper.getTitleFromTexContent.should.have.been.called
+        this.DocumentUpdaterHandler.promises.updateProjectStructure.should.have.been.calledWith(
+          this.project._id,
+          this.project.overleaf.history.id,
+          this.ownerId,
+          {
+            newDocs: this.docEntries,
+            newFiles: this.fileEntries,
+            newProject: { version: this.newProjectVersion }
+          }
+        )
+      })
+
+      it('should flush the project to TPDS', function() {
+        this.TpdsProjectFlusher.promises.flushProjectToTpds.should.have.been.calledWith(
+          this.project._id
+        )
       })
 
       it('should set the root document', function() {
         this.ProjectRootDocManager.promises.setRootDocFromName.should.have.been.calledWith(
-          this.project_id,
+          this.project._id,
           'main.tex'
         )
       })
 
-      it('should ensure the name is valid', function() {
+      it('should remove the destination directory afterwards', function() {
-        this.ProjectDetailsHandler.fixProjectName.should.have.been.called
+        this.fs.remove.should.have.been.calledWith(this.extractedZipPath)
       })
     })
 
@@ -157,9 +242,9 @@ describe('ProjectUploadManager', function() {
           {}
         )
         await this.ProjectUploadManager.promises.createProjectFromZipArchive(
-          this.owner_id,
+          this.ownerId,
-          this.name,
+          this.projectName,
-          this.source
+          this.zipPath
         )
       })
 
@@ -173,73 +258,78 @@ describe('ProjectUploadManager', function() {
   describe('createProjectFromZipArchiveWithName', function() {
     beforeEach(async function() {
       await this.ProjectUploadManager.promises.createProjectFromZipArchiveWithName(
-        this.owner_id,
+        this.ownerId,
-        this.name,
+        this.projectName,
-        this.source
+        this.zipPath
-      )
-    })
-
-    it('should create a project owned by the owner_id', function() {
-      this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith(
-        this.owner_id
-      )
-    })
-
-    it('should create a project with the correct name', function() {
-      this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith(
-        sinon.match.any,
-        this.othername
-      )
-    })
-
-    it('should automatically set the root doc', function() {
-      this.ProjectRootDocManager.promises.setRootDocAutomatically.should.have.been.calledWith(
-        this.project_id
       )
     })
 
     it('should extract the archive', function() {
       this.ArchiveManager.promises.extractZipArchive.should.have.been.calledWith(
-        this.source,
+        this.zipPath,
-        this.destination
+        this.extractedZipPath
       )
     })
 
-    it('should find the top level directory', function() {
+    it('should create a project owned by the owner_id', function() {
-      this.ArchiveManager.promises.findTopLevelDirectory.should.have.been.calledWith(
+      this.ProjectCreationHandler.promises.createBlankProject.should.have.been.calledWith(
-        this.destination
+        this.ownerId,
+        this.uniqueProjectName
       )
     })
 
-    it('should insert the extracted archive into the folder', function() {
+    it('should automatically set the root doc', function() {
-      this.FileSystemImportManager.promises.addFolderContents.should.have.been.calledWith(
+      this.ProjectRootDocManager.promises.setRootDocAutomatically.should.have.been.calledWith(
-        this.owner_id,
+        this.project._id
-        this.project_id,
+      )
-        this.folder_id,
+    })
-        this.topLevelDestination,
+
-        false
+    it('should initialize the file tree', function() {
+      this.ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure.should.have.been.calledWith(
+        this.project._id,
+        this.docEntries,
+        this.fileEntries
+      )
+    })
+
+    it('should notify document updater', function() {
+      this.DocumentUpdaterHandler.promises.updateProjectStructure.should.have.been.calledWith(
+        this.project._id,
+        this.project.overleaf.history.id,
+        this.ownerId,
+        {
+          newDocs: this.docEntries,
+          newFiles: this.fileEntries,
+          newProject: { version: this.newProjectVersion }
+        }
+      )
+    })
+
+    it('should flush the project to TPDS', function() {
+      this.TpdsProjectFlusher.promises.flushProjectToTpds.should.have.been.calledWith(
+        this.project._id
       )
     })
 
     it('should remove the destination directory afterwards', function() {
-      this.fs.remove.should.have.been.calledWith(this.destination)
+      this.fs.remove.should.have.been.calledWith(this.extractedZipPath)
     })
 
-    describe('when inserting the zip file contents into the root folder fails', function() {
+    describe('when initializing the folder structure fails', function() {
       beforeEach(async function() {
-        this.FileSystemImportManager.promises.addFolderContents.rejects()
+        this.ProjectEntityMongoUpdateHandler.promises.createNewFolderStructure.rejects()
         await expect(
           this.ProjectUploadManager.promises.createProjectFromZipArchiveWithName(
-            this.owner_id,
+            this.ownerId,
-            this.name,
+            this.projectName,
-            this.source
+            this.zipPath
           )
         ).to.be.rejected
       })
 
       it('should cleanup the blank project created', async function() {
         this.ProjectDeleter.promises.deleteProject.should.have.been.calledWith(
-          this.project_id
+          this.project._id
         )
       })
     })
@@ -249,16 +339,16 @@ describe('ProjectUploadManager', function() {
         this.ProjectRootDocManager.promises.setRootDocAutomatically.rejects()
         await expect(
           this.ProjectUploadManager.promises.createProjectFromZipArchiveWithName(
-            this.owner_id,
+            this.ownerId,
-            this.name,
+            this.projectName,
-            this.source
+            this.zipPath
           )
         ).to.be.rejected
       })
 
       it('should cleanup the blank project created', function() {
         this.ProjectDeleter.promises.deleteProject.should.have.been.calledWith(
-          this.project_id
+          this.project._id
         )
       })
     })