From 7ec7237f17cb1925aa72dead58bf95eeb8290aad Mon Sep 17 00:00:00 2001 From: Shane Kilkelly Date: Wed, 8 Apr 2020 09:35:10 +0100 Subject: [PATCH] Merge pull request #2716 from overleaf/em-promisify Promisify FileSystemImportManager GitOrigin-RevId: 8f89492872c94a596afbfa644e5f2b985eb65a28 --- .../Uploads/FileSystemImportManager.js | 494 ++++------ .../src/Features/Uploads/FileTypeManager.js | 4 + .../Uploads/FileSystemImportManagerTests.js | 853 +++++++----------- 3 files changed, 533 insertions(+), 818 deletions(-) diff --git a/services/web/app/src/Features/Uploads/FileSystemImportManager.js b/services/web/app/src/Features/Uploads/FileSystemImportManager.js index 49636a7c6c..d60bb3422e 100644 --- a/services/web/app/src/Features/Uploads/FileSystemImportManager.js +++ b/services/web/app/src/Features/Uploads/FileSystemImportManager.js @@ -1,311 +1,197 @@ -/* eslint-disable - camelcase, - handle-callback-err, - max-len, - no-unused-vars, - standard/no-callback-literal, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS102: Remove unnecessary code created because of implicit returns - * DS207: Consider shorter variations of null checks - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ -let FileSystemImportManager -const async = require('async') const fs = require('fs') -const _ = require('underscore') +const { callbackify } = require('util') const FileTypeManager = require('./FileTypeManager') const EditorController = require('../Editor/EditorController') const logger = require('logger-sharelatex') -module.exports = FileSystemImportManager = { - addDoc( - user_id, - project_id, - folder_id, - name, - path, - encoding, - replace, - callback - ) { - if (callback == null) { - callback = function(error, doc) {} - } - return FileSystemImportManager._isSafeOnFileSystem(path, function( - err, - isSafe - ) { - if (!isSafe) { - logger.log( - { user_id, project_id, folder_id, name, path }, - 'add doc is from symlink, stopping process' - ) - return callback(new Error('path is symlink')) - } - return fs.readFile(path, encoding, function(error, content) { - if (error != null) { - return callback(error) - } - content = content.replace(/\r\n?/g, '\n') // convert Windows line endings to unix. very old macs also created \r-separated lines - const lines = content.split('\n') - if (replace) { - return EditorController.upsertDoc( - project_id, - folder_id, - name, - lines, - 'upload', - user_id, - callback - ) - } else { - return EditorController.addDoc( - project_id, - folder_id, - name, - lines, - 'upload', - user_id, - callback - ) - } - }) - }) - }, - - addFile(user_id, project_id, folder_id, name, path, replace, callback) { - if (callback == null) { - callback = function(error, file) {} - } - return FileSystemImportManager._isSafeOnFileSystem(path, function( - err, - isSafe - ) { - if (!isSafe) { - logger.log( - { user_id, project_id, folder_id, name, path }, - 'add file is from symlink, stopping insert' - ) - return callback(new Error('path is symlink')) - } - - if (replace) { - return EditorController.upsertFile( - project_id, - folder_id, - name, - path, - null, - 'upload', - user_id, - callback - ) - } else { - return EditorController.addFile( - project_id, - folder_id, - name, - path, - null, - 'upload', - user_id, - callback - ) - } - }) - }, - - addFolder(user_id, project_id, folder_id, name, path, replace, callback) { - if (callback == null) { - callback = function(error) {} - } - return FileSystemImportManager._isSafeOnFileSystem(path, function( - err, - isSafe - ) { - if (!isSafe) { - logger.log( - { user_id, project_id, folder_id, path }, - 'add folder is from symlink, stopping insert' - ) - return callback(new Error('path is symlink')) - } - return EditorController.addFolder( - project_id, - folder_id, - name, - 'upload', - (error, new_folder) => { - if (error != null) { - return callback(error) - } - return FileSystemImportManager.addFolderContents( - user_id, - project_id, - new_folder._id, - path, - replace, - function(error) { - if (error != null) { - return callback(error) - } - return callback(null, new_folder) - } - ) - } - ) - }) - }, - - addFolderContents( - user_id, - project_id, - parent_folder_id, - folderPath, - replace, - callback - ) { - if (callback == null) { - callback = function(error) {} - } - return FileSystemImportManager._isSafeOnFileSystem(folderPath, function( - err, - isSafe - ) { - if (!isSafe) { - logger.log( - { user_id, project_id, parent_folder_id, folderPath }, - 'add folder contents is from symlink, stopping insert' - ) - return callback(new Error('path is symlink')) - } - return fs.readdir(folderPath, (error, entries) => { - if (entries == null) { - entries = [] - } - if (error != null) { - return callback(error) - } - return async.eachSeries( - entries, - (entry, callback) => { - return FileTypeManager.shouldIgnore(entry, (error, ignore) => { - if (error != null) { - return callback(error) - } - if (!ignore) { - return FileSystemImportManager.addEntity( - user_id, - project_id, - parent_folder_id, - entry, - `${folderPath}/${entry}`, - replace, - callback - ) - } else { - return callback() - } - }) - }, - callback - ) - }) - }) - }, - - addEntity(user_id, project_id, folder_id, name, path, replace, callback) { - if (callback == null) { - callback = function(error, entity) {} - } - return FileSystemImportManager._isSafeOnFileSystem(path, function( - err, - isSafe - ) { - if (!isSafe) { - logger.log( - { user_id, project_id, folder_id, path }, - 'add entry is from symlink, stopping insert' - ) - return callback(new Error('path is symlink')) - } - - return FileTypeManager.isDirectory(path, (error, isDirectory) => { - if (error != null) { - return callback(error) - } - if (isDirectory) { - return FileSystemImportManager.addFolder( - user_id, - project_id, - folder_id, - name, - path, - replace, - callback - ) - } else { - return FileTypeManager.getType( - name, - path, - (error, { binary, encoding }) => { - if (error != null) { - return callback(error) - } - if (binary) { - return FileSystemImportManager.addFile( - user_id, - project_id, - folder_id, - name, - path, - replace, - function(err, entity) { - if (entity != null) { - entity.type = 'file' - } - return callback(err, entity) - } - ) - } else { - return FileSystemImportManager.addDoc( - user_id, - project_id, - folder_id, - name, - path, - encoding, - replace, - function(err, entity) { - if (entity != null) { - entity.type = 'doc' - } - return callback(err, entity) - } - ) - } - } - ) - } - }) - }) - }, - - _isSafeOnFileSystem(path, callback) { - if (callback == null) { - callback = function(err, isSafe) {} - } - return fs.lstat(path, function(err, stat) { - if (err != null) { - logger.warn({ err }, 'error with path symlink check') - return callback(err) - } - const isSafe = stat.isFile() || stat.isDirectory() - return callback(err, isSafe) - }) +module.exports = { + addFolderContents: callbackify(addFolderContents), + addEntity: callbackify(addEntity), + promises: { + addFolderContents, + addEntity } } + +async function addDoc( + userId, + projectId, + folderId, + name, + path, + encoding, + replace +) { + if (!(await _isSafeOnFileSystem(path))) { + logger.log( + { userId, projectId, folderId, name, path }, + 'add doc is from symlink, stopping process' + ) + throw new Error('path is symlink') + } + let content = await fs.promises.readFile(path, encoding) + content = content.replace(/\r\n?/g, '\n') // convert Windows line endings to unix. very old macs also created \r-separated lines + const lines = content.split('\n') + if (replace) { + const doc = await EditorController.promises.upsertDoc( + projectId, + folderId, + name, + lines, + 'upload', + userId + ) + return doc + } else { + const doc = await EditorController.promises.addDoc( + projectId, + folderId, + name, + lines, + 'upload', + userId + ) + return doc + } +} + +async function addFile(userId, projectId, folderId, name, path, replace) { + if (!(await _isSafeOnFileSystem(path))) { + logger.log( + { userId, projectId, folderId, name, path }, + 'add file is from symlink, stopping insert' + ) + throw new Error('path is symlink') + } + + if (replace) { + const file = await EditorController.promises.upsertFile( + projectId, + folderId, + name, + path, + null, + 'upload', + userId + ) + return file + } else { + const file = await EditorController.promises.addFile( + projectId, + folderId, + name, + path, + null, + 'upload', + userId + ) + return file + } +} + +async function addFolder(userId, projectId, folderId, name, path, replace) { + if (!(await _isSafeOnFileSystem(path))) { + logger.log( + { userId, projectId, folderId, path }, + 'add folder is from symlink, stopping insert' + ) + throw new Error('path is symlink') + } + const newFolder = await EditorController.promises.addFolder( + projectId, + folderId, + name, + 'upload' + ) + await addFolderContents(userId, projectId, newFolder._id, path, replace) + return newFolder +} + +async function addFolderContents( + userId, + projectId, + parentFolderId, + folderPath, + replace +) { + if (!(await _isSafeOnFileSystem(folderPath))) { + logger.log( + { userId, projectId, parentFolderId, folderPath }, + 'add folder contents is from symlink, stopping insert' + ) + throw new Error('path is symlink') + } + const entries = (await fs.promises.readdir(folderPath)) || [] + for (const entry of entries) { + if (await FileTypeManager.promises.shouldIgnore(entry)) { + continue + } + await addEntity( + userId, + projectId, + parentFolderId, + entry, + `${folderPath}/${entry}`, + replace + ) + } +} + +async function addEntity(userId, projectId, folderId, name, path, replace) { + if (!(await _isSafeOnFileSystem(path))) { + logger.log( + { userId, projectId, folderId, path }, + 'add entry is from symlink, stopping insert' + ) + throw new Error('path is symlink') + } + + if (await FileTypeManager.promises.isDirectory(path)) { + const newFolder = await addFolder( + userId, + projectId, + folderId, + name, + path, + replace + ) + return newFolder + } + const { binary, encoding } = await FileTypeManager.promises.getType( + name, + path + ) + if (binary) { + const entity = await addFile( + userId, + projectId, + folderId, + name, + path, + replace + ) + if (entity != null) { + entity.type = 'file' + } + return entity + } else { + const entity = await addDoc( + userId, + projectId, + folderId, + name, + path, + encoding, + replace + ) + if (entity != null) { + entity.type = 'doc' + } + return entity + } +} + +async function _isSafeOnFileSystem(path) { + const stat = await fs.promises.lstat(path) + return stat.isFile() || stat.isDirectory() +} diff --git a/services/web/app/src/Features/Uploads/FileTypeManager.js b/services/web/app/src/Features/Uploads/FileTypeManager.js index 8589bda2e4..e0ca272644 100644 --- a/services/web/app/src/Features/Uploads/FileTypeManager.js +++ b/services/web/app/src/Features/Uploads/FileTypeManager.js @@ -1,6 +1,7 @@ const fs = require('fs') const Path = require('path') const isUtf8 = require('utf-8-validate') +const { promisifyAll } = require('../../util/promises') const FileTypeManager = { TEXT_EXTENSIONS: [ @@ -159,3 +160,6 @@ function _detectEncoding(bytes) { } module.exports = FileTypeManager +module.exports.promises = promisifyAll(FileTypeManager, { + without: ['getStrictTypeFromContent'] +}) diff --git a/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js b/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js index f8e9d7bea1..27b441af63 100644 --- a/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js +++ b/services/web/test/unit/src/Uploads/FileSystemImportManagerTests.js @@ -1,553 +1,378 @@ -/* eslint-disable - max-len, - no-return-assign, - no-unused-vars, -*/ -// TODO: This file was created by bulk-decaffeinate. -// Fix any style issues and re-enable lint. -/* - * decaffeinate suggestions: - * DS101: Remove unnecessary use of Array.from - * DS102: Remove unnecessary code created because of implicit returns - * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md - */ const sinon = require('sinon') -const chai = require('chai') -const should = chai.should() -const modulePath = - '../../../../app/src/Features/Uploads/FileSystemImportManager.js' +const { expect } = require('chai') const SandboxedModule = require('sandboxed-module') +const { ObjectId } = require('mongodb') + +const MODULE_PATH = + '../../../../app/src/Features/Uploads/FileSystemImportManager.js' describe('FileSystemImportManager', function() { beforeEach(function() { - this.project_id = 'project-id-123' - this.folder_id = 'folder-id-123' - this.name = 'test-file.tex' - this.path_on_disk = `/path/to/file/${this.name}` - this.replace = 'replace-boolean-flag-mock' - this.user_id = 'mock-user-123' - this.callback = sinon.stub() + this.projectId = new ObjectId() + this.folderId = new ObjectId() + this.newFolderId = new ObjectId() + this.userId = new ObjectId() + + this.folderPath = '/path/to/folder' + this.docName = 'test-doc.tex' + this.docPath = `/path/to/folder/${this.docName}` + this.docContent = 'one\ntwo\nthree' + this.docLines = this.docContent.split('\n') + this.fileName = 'test-file.jpg' + this.filePath = `/path/to/folder/${this.fileName}` + this.symlinkName = 'symlink' + this.symlinkPath = `/path/to/${this.symlinkName}` + this.ignoredName = '.DS_Store' + this.ignoredPath = `/path/to/folder/${this.ignoredName}` + this.folderEntries = [this.ignoredName, this.docName, this.fileName] + this.encoding = 'latin1' - this.DocumentHelper = { - convertTexEncodingsToUtf8: sinon.stub().returnsArg(0) + + this.fileStat = { + isFile: sinon.stub().returns(true), + isDirectory: sinon.stub().returns(false) } - return (this.FileSystemImportManager = SandboxedModule.require(modulePath, { + this.dirStat = { + isFile: sinon.stub().returns(false), + isDirectory: sinon.stub().returns(true) + } + this.symlinkStat = { + isFile: sinon.stub().returns(false), + isDirectory: sinon.stub().returns(false) + } + this.fs = { + promises: { + lstat: sinon.stub(), + readFile: sinon.stub(), + readdir: sinon.stub() + } + } + this.fs.promises.lstat.withArgs(this.filePath).resolves(this.fileStat) + this.fs.promises.lstat.withArgs(this.docPath).resolves(this.fileStat) + this.fs.promises.lstat.withArgs(this.symlinkPath).resolves(this.symlinkStat) + this.fs.promises.lstat.withArgs(this.folderPath).resolves(this.dirStat) + this.fs.promises.readFile + .withArgs(this.docPath, this.encoding) + .resolves(this.docContent) + this.fs.promises.readdir + .withArgs(this.folderPath) + .resolves(this.folderEntries) + this.EditorController = { + promises: { + addDoc: sinon.stub().resolves(), + addFile: sinon.stub().resolves(), + upsertDoc: sinon.stub().resolves(), + upsertFile: sinon.stub().resolves(), + addFolder: sinon.stub().resolves({ _id: this.newFolderId }) + } + } + this.FileTypeManager = { + promises: { + isDirectory: sinon.stub().resolves(false), + getType: sinon.stub(), + shouldIgnore: sinon.stub().resolves(false) + } + } + this.FileTypeManager.promises.getType + .withArgs(this.fileName, this.filePath) + .resolves({ binary: true }) + this.FileTypeManager.promises.getType + .withArgs(this.docName, this.docPath) + .resolves({ binary: false, encoding: this.encoding }) + this.FileTypeManager.promises.isDirectory + .withArgs(this.folderPath) + .resolves(true) + this.FileTypeManager.promises.shouldIgnore + .withArgs(this.ignoredName) + .resolves(true) + this.logger = { + log() {}, + err() {} + } + this.FileSystemImportManager = SandboxedModule.require(MODULE_PATH, { globals: { console: console }, requires: { - fs: (this.fs = {}), - '../Editor/EditorController': (this.EditorController = {}), - './FileTypeManager': (this.FileTypeManager = {}), - '../Project/ProjectLocator': (this.ProjectLocator = {}), - '../Documents/DocumentHelper': this.DocumentHelper, - 'logger-sharelatex': { - log() {}, - err() {} - } + fs: this.fs, + '../Editor/EditorController': this.EditorController, + './FileTypeManager': this.FileTypeManager, + 'logger-sharelatex': this.logger } - })) - }) - - describe('addDoc', function() { - beforeEach(function() { - this.docContent = 'one\ntwo\nthree' - this.docLines = this.docContent.split('\n') - this.fs.readFile = sinon.stub().callsArgWith(2, null, this.docContent) - return (this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true)) - }) - - describe('when path is symlink', function() { - beforeEach(function() { - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, false) - this.EditorController.addDoc = sinon.stub() - return this.FileSystemImportManager.addDoc( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.encoding, - false, - this.callback - ) - }) - - it('should not read the file from disk', function() { - return this.fs.readFile.called.should.equal(false) - }) - - it('should not insert the doc', function() { - return this.EditorController.addDoc.called.should.equal(false) - }) - }) - - describe('with replace set to false', function() { - beforeEach(function() { - this.EditorController.addDoc = sinon.stub().callsArg(6) - return this.FileSystemImportManager.addDoc( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.encoding, - false, - this.callback - ) - }) - - it('should read the file from disk', function() { - return this.fs.readFile.calledWith(this.path_on_disk).should.equal(true) - }) - - it('should insert the doc', function() { - return this.EditorController.addDoc - .calledWith( - this.project_id, - this.folder_id, - this.name, - this.docLines, - 'upload', - this.user_id - ) - .should.equal(true) - }) - }) - - describe('with windows line ending', function() { - beforeEach(function() { - this.docContent = 'one\r\ntwo\r\nthree' - this.docLines = ['one', 'two', 'three'] - this.fs.readFile = sinon.stub().callsArgWith(2, null, this.docContent) - this.EditorController.addDoc = sinon.stub().callsArg(6) - return this.FileSystemImportManager.addDoc( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.encoding, - false, - this.callback - ) - }) - - it('should strip the \\r characters before adding', function() { - return this.EditorController.addDoc - .calledWith( - this.project_id, - this.folder_id, - this.name, - this.docLines, - 'upload', - this.user_id - ) - .should.equal(true) - }) - }) - - describe('with \r line endings', function() { - beforeEach(function() { - this.docContent = 'one\rtwo\rthree' - this.docLines = ['one', 'two', 'three'] - this.fs.readFile = sinon.stub().callsArgWith(2, null, this.docContent) - this.EditorController.addDoc = sinon.stub().callsArg(6) - return this.FileSystemImportManager.addDoc( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.encoding, - false, - this.callback - ) - }) - - it('should treat the \\r characters as newlines', function() { - return this.EditorController.addDoc - .calledWith( - this.project_id, - this.folder_id, - this.name, - this.docLines, - 'upload', - this.user_id - ) - .should.equal(true) - }) - }) - - describe('with replace set to true', function() { - beforeEach(function() { - this.EditorController.upsertDoc = sinon.stub().yields() - return this.FileSystemImportManager.addDoc( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.encoding, - true, - this.callback - ) - }) - - it('should upsert the doc', function() { - return this.EditorController.upsertDoc - .calledWith( - this.project_id, - this.folder_id, - this.name, - this.docLines, - 'upload', - this.user_id - ) - .should.equal(true) - }) - - it('should read the file with the correct encoding', function() { - return sinon.assert.calledWith( - this.fs.readFile, - this.path_on_disk, - this.encoding - ) - }) - }) - }) - - describe('addFile with replace set to false', function() { - beforeEach(function() { - this.EditorController.addFile = sinon.stub().yields() - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - return this.FileSystemImportManager.addFile( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - false, - this.callback - ) - }) - - it('should add the file', function() { - return this.EditorController.addFile - .calledWith( - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - null, - 'upload', - this.user_id - ) - .should.equal(true) - }) - }) - - describe('addFile with symlink', function() { - beforeEach(function() { - this.EditorController.addFile = sinon.stub() - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, false) - this.EditorController.replaceFile = sinon.stub() - return this.FileSystemImportManager.addFile( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - false, - this.callback - ) - }) - - it('should node add the file', function() { - this.EditorController.addFile.called.should.equal(false) - return this.EditorController.replaceFile.called.should.equal(false) - }) - }) - - describe('addFile with replace set to true', function() { - beforeEach(function() { - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - this.EditorController.upsertFile = sinon.stub().yields() - return this.FileSystemImportManager.addFile( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - true, - this.callback - ) - }) - - it('should add the file', function() { - return this.EditorController.upsertFile - .calledWith( - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - null, - 'upload', - this.user_id - ) - .should.equal(true) - }) - }) - - describe('addFolder', function() { - beforeEach(function() { - this.new_folder_id = 'new-folder-id' - this.EditorController.addFolder = sinon - .stub() - .callsArgWith(4, null, { _id: this.new_folder_id }) - return (this.FileSystemImportManager.addFolderContents = sinon - .stub() - .callsArg(5)) - }) - - describe('successfully', function() { - beforeEach(function() { - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - return this.FileSystemImportManager.addFolder( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace, - this.callback - ) - }) - - it('should add a folder to the project', function() { - return this.EditorController.addFolder - .calledWith(this.project_id, this.folder_id, this.name, 'upload') - .should.equal(true) - }) - - it('should add the folders contents', function() { - return this.FileSystemImportManager.addFolderContents - .calledWith( - this.user_id, - this.project_id, - this.new_folder_id, - this.path_on_disk, - this.replace - ) - .should.equal(true) - }) - }) - - describe('with symlink', function() { - beforeEach(function() { - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, false) - return this.FileSystemImportManager.addFolder( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace, - this.callback - ) - }) - - it('should not add a folder to the project', function() { - this.EditorController.addFolder.called.should.equal(false) - return this.FileSystemImportManager.addFolderContents.called.should.equal( - false - ) - }) }) }) describe('addFolderContents', function() { - beforeEach(function() { - this.folderEntries = ['path1', 'path2', 'path3'] - this.ignoredEntries = ['.DS_Store'] - this.fs.readdir = sinon - .stub() - .callsArgWith(1, null, this.folderEntries.concat(this.ignoredEntries)) - this.FileSystemImportManager.addEntity = sinon.stub().callsArg(6) - this.FileTypeManager.shouldIgnore = (path, callback) => { - return callback( - null, - this.ignoredEntries.indexOf(require('path').basename(path)) !== -1 + describe('successfully', function() { + beforeEach(async function() { + await this.FileSystemImportManager.promises.addFolderContents( + this.userId, + this.projectId, + this.folderId, + this.folderPath, + false ) - } - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - return this.FileSystemImportManager.addFolderContents( - this.user_id, - this.project_id, - this.folder_id, - this.path_on_disk, - this.replace, - this.callback - ) + }) + + it('should add each file in the folder which is not ignored', function() { + this.EditorController.promises.addDoc.should.have.been.calledWith( + this.projectId, + this.folderId, + this.docName, + this.docLines, + 'upload', + this.userId + ) + this.EditorController.promises.addFile.should.have.been.calledWith( + this.projectId, + this.folderId, + this.fileName, + this.filePath, + null, + 'upload', + this.userId + ) + }) }) - it('should call addEntity for each file in the folder which is not ignored', function() { - return Array.from(this.folderEntries).map(name => - this.FileSystemImportManager.addEntity - .calledWith( - this.user_id, - this.project_id, - this.folder_id, - name, - `${this.path_on_disk}/${name}`, - this.replace + describe('with symlink', function() { + it('should stop with an error', async function() { + await expect( + this.FileSystemImportManager.promises.addFolderContents( + this.userId, + this.projectId, + this.folderId, + this.symlinkPath, + false ) - .should.equal(true) - ) - }) - - it('should not call addEntity for the ignored files', function() { - return Array.from(this.ignoredEntries).map(name => - this.FileSystemImportManager.addEntity - .calledWith( - this.user_id, - this.project_id, - this.folder_id, - name, - `${this.path_on_disk}/${name}`, - this.replace - ) - .should.equal(false) - ) - }) - - it('should look in the correct directory', function() { - return this.fs.readdir.calledWith(this.path_on_disk).should.equal(true) + ).to.be.rejectedWith('path is symlink') + this.EditorController.promises.addFolder.should.not.have.been.called + this.EditorController.promises.addDoc.should.not.have.been.called + this.EditorController.promises.addFile.should.not.have.been.called + }) }) }) describe('addEntity', function() { describe('with directory', function() { - beforeEach(function() { - this.FileTypeManager.isDirectory = sinon - .stub() - .callsArgWith(1, null, true) - this.FileSystemImportManager.addFolder = sinon.stub().callsArg(6) - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - return this.FileSystemImportManager.addEntity( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace, - this.callback - ) - }) - - it('should call addFolder', function() { - return this.FileSystemImportManager.addFolder - .calledWith( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace + describe('successfully', function() { + beforeEach(async function() { + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.folderName, + this.folderPath, + false ) - .should.equal(true) + }) + + it('should add a folder to the project', function() { + this.EditorController.promises.addFolder.should.have.been.calledWith( + this.projectId, + this.folderId, + this.folderName, + 'upload' + ) + }) + + it("should add the folder's contents", function() { + this.EditorController.promises.addDoc.should.have.been.calledWith( + this.projectId, + this.newFolderId, + this.docName, + this.docLines, + 'upload', + this.userId + ) + this.EditorController.promises.addFile.should.have.been.calledWith( + this.projectId, + this.newFolderId, + this.fileName, + this.filePath, + null, + 'upload', + this.userId + ) + }) }) }) describe('with binary file', function() { - beforeEach(function() { - this.FileTypeManager.isDirectory = sinon - .stub() - .callsArgWith(1, null, false) - this.FileTypeManager.getType = sinon - .stub() - .yields(null, { binary: true }) - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - this.FileSystemImportManager.addFile = sinon.stub().callsArg(6) - return this.FileSystemImportManager.addEntity( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace, - this.callback - ) + describe('with replace set to false', function() { + beforeEach(async function() { + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.fileName, + this.filePath, + false + ) + }) + + it('should add the file', function() { + this.EditorController.promises.addFile.should.have.been.calledWith( + this.projectId, + this.folderId, + this.fileName, + this.filePath, + null, + 'upload', + this.userId + ) + }) }) - it('should call addFile', function() { - return this.FileSystemImportManager.addFile - .calledWith( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace + describe('with replace set to true', function() { + beforeEach(async function() { + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.fileName, + this.filePath, + true ) - .should.equal(true) + }) + + it('should add the file', function() { + this.EditorController.promises.upsertFile.should.have.been.calledWith( + this.projectId, + this.folderId, + this.fileName, + this.filePath, + null, + 'upload', + this.userId + ) + }) + }) + + describe('with text file', function() { + describe('with replace set to false', function() { + beforeEach(async function() { + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.docName, + this.docPath, + false + ) + }) + + it('should insert the doc', function() { + this.EditorController.promises.addDoc.should.have.been.calledWith( + this.projectId, + this.folderId, + this.docName, + this.docLines, + 'upload', + this.userId + ) + }) + }) + + describe('with windows line ending', function() { + beforeEach(async function() { + this.docContent = 'one\r\ntwo\r\nthree' + this.docLines = ['one', 'two', 'three'] + this.fs.promises.readFile + .withArgs(this.docPath, this.encoding) + .resolves(this.docContent) + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.docName, + this.docPath, + false + ) + }) + + it('should strip the \\r characters before adding', function() { + this.EditorController.promises.addDoc.should.have.been.calledWith( + this.projectId, + this.folderId, + this.docName, + this.docLines, + 'upload', + this.userId + ) + }) + }) + + describe('with \r line endings', function() { + beforeEach(async function() { + this.docContent = 'one\rtwo\rthree' + this.docLines = ['one', 'two', 'three'] + this.fs.promises.readFile + .withArgs(this.docPath, this.encoding) + .resolves(this.docContent) + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.docName, + this.docPath, + false + ) + }) + + it('should treat the \\r characters as newlines', function() { + this.EditorController.promises.addDoc.should.have.been.calledWith( + this.projectId, + this.folderId, + this.docName, + this.docLines, + 'upload', + this.userId + ) + }) + }) + + describe('with replace set to true', function() { + beforeEach(async function() { + await this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.docName, + this.docPath, + true + ) + }) + + it('should upsert the doc', function() { + this.EditorController.promises.upsertDoc.should.have.been.calledWith( + this.projectId, + this.folderId, + this.docName, + this.docLines, + 'upload', + this.userId + ) + }) + }) }) }) - describe('with text file', function() { - beforeEach(function() { - this.FileTypeManager.isDirectory = sinon - .stub() - .callsArgWith(1, null, false) - this.FileTypeManager.getType = sinon - .stub() - .yields(null, { binary: false, encoding: 'latin1' }) - this.FileSystemImportManager.addDoc = sinon.stub().callsArg(7) - this.FileSystemImportManager._isSafeOnFileSystem = sinon - .stub() - .callsArgWith(1, null, true) - return this.FileSystemImportManager.addEntity( - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - this.replace, - this.callback - ) - }) - - it('should call addFile', function() { - return sinon.assert.calledWith( - this.FileSystemImportManager.addDoc, - this.user_id, - this.project_id, - this.folder_id, - this.name, - this.path_on_disk, - 'latin1', - this.replace - ) + describe('with symlink', function() { + it('should stop with an error', async function() { + await expect( + this.FileSystemImportManager.promises.addEntity( + this.userId, + this.projectId, + this.folderId, + this.symlinkName, + this.symlinkPath, + false + ) + ).to.be.rejectedWith('path is symlink') + this.EditorController.promises.addFolder.should.not.have.been.called + this.EditorController.promises.addDoc.should.not.have.been.called + this.EditorController.promises.addFile.should.not.have.been.called }) }) })