added extra isSymLink checks into reading files

This commit is contained in:
Henry Oswald 2016-03-12 12:01:36 +00:00
parent f108c01ea5
commit da242d90e6
4 changed files with 178 additions and 81 deletions

View file

@ -6,24 +6,31 @@ settings = require("settings-sharelatex")
oneMinInMs = 60 * 1000
fiveMinsInMs = oneMinInMs * 5
module.exports =
module.exports = FileStoreHandler =
uploadFileFromDisk: (project_id, file_id, fsPath, callback)->
logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "uploading file from disk"
readStream = fs.createReadStream(fsPath)
opts =
method: "post"
uri: @_buildUrl(project_id, file_id)
timeout:fiveMinsInMs
writeStream = request(opts)
readStream.pipe writeStream
writeStream.on "end", callback
readStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the read stream of uploadFileFromDisk"
callback err
writeStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the write stream of uploadFileFromDisk"
callback err
fs.lstat fsPath, (err, stat)->
if err?
logger.err err:err, "error with path symlink check"
return callback(err)
if stat.isSymbolicLink()
logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "error uploading file from disk, file path is symlink"
return callback('file is from symlink')
logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "uploading file from disk"
readStream = fs.createReadStream(fsPath)
opts =
method: "post"
uri: FileStoreHandler._buildUrl(project_id, file_id)
timeout:fiveMinsInMs
writeStream = request(opts)
readStream.pipe writeStream
writeStream.on "end", callback
readStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the read stream of uploadFileFromDisk"
callback err
writeStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the write stream of uploadFileFromDisk"
callback err
getFileStream: (project_id, file_id, query, callback)->
logger.log project_id:project_id, file_id:file_id, query:query, "getting file stream from file store"

View file

@ -4,76 +4,108 @@ _ = require "underscore"
FileTypeManager = require "./FileTypeManager"
EditorController = require "../Editor/EditorController"
ProjectLocator = require "../Project/ProjectLocator"
logger = require("logger-sharelatex")
module.exports = FileSystemImportManager =
addDoc: (user_id, project_id, folder_id, name, path, replace, callback = (error, doc)-> )->
fs.readFile path, "utf8", (error, content = "") ->
return callback(error) if error?
content = content.replace(/\r/g, "")
lines = content.split("\n")
if replace
FileSystemImportManager._isSymLink path, (err, isSymLink)->
if isSymLink
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, name:name, path:path, "add doc is from symlink, stopping process"
return callback("path is symlink")
fs.readFile path, "utf8", (error, content = "") ->
return callback(error) if error?
content = content.replace(/\r/g, "")
lines = content.split("\n")
if replace
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
existingDoc = null
for doc in folder.docs
if doc.name == name
existingDoc = doc
break
if existingDoc?
EditorController.setDoc project_id, existingDoc._id, user_id, lines, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
addFile: (user_id, project_id, folder_id, name, path, replace, callback = (error, file)-> )->
FileSystemImportManager._isSymLink path, (err, isSymLink)->
if isSymLink
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, name:name, path:path, "add file is from symlink, stopping insert"
return callback("path is symlink")
if !replace
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
else
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
existingDoc = null
for doc in folder.docs
if doc.name == name
existingDoc = doc
existingFile = null
for fileRef in folder.fileRefs
if fileRef.name == name
existingFile = fileRef
break
if existingDoc?
EditorController.setDoc project_id, existingDoc._id, user_id, lines, "upload", callback
if existingFile?
EditorController.replaceFile project_id, existingFile._id, path, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
addFile: (user_id, project_id, folder_id, name, path, replace, callback = (error, file)-> )->
if replace
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
existingFile = null
for fileRef in folder.fileRefs
if fileRef.name == name
existingFile = fileRef
break
if existingFile?
EditorController.replaceFile project_id, existingFile._id, path, "upload", callback
else
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
else
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
EditorController.addFolderWithoutLock project_id, folder_id, name, "upload", (error, new_folder) =>
return callback(error) if error?
@addFolderContents user_id, project_id, new_folder._id, path, replace, (error) ->
FileSystemImportManager._isSymLink path, (err, isSymLink)->
if isSymLink
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, path:path, "add folder is from symlink, stopping insert"
return callback("path is symlink")
EditorController.addFolderWithoutLock project_id, folder_id, name, "upload", (error, new_folder) =>
return callback(error) if error?
callback null, new_folder
FileSystemImportManager.addFolderContents user_id, project_id, new_folder._id, path, replace, (error) ->
return callback(error) if error?
callback null, new_folder
addFolderContents: (user_id, project_id, parent_folder_id, folderPath, replace, callback = (error)-> ) ->
fs.readdir folderPath, (error, entries = []) =>
return callback(error) if error?
jobs = _.map entries, (entry) =>
(callback) =>
FileTypeManager.shouldIgnore entry, (error, ignore) =>
return callback(error) if error?
if !ignore
@addEntity user_id, project_id, parent_folder_id, entry, "#{folderPath}/#{entry}", replace, callback
else
callback()
async.parallelLimit jobs, 5, callback
FileSystemImportManager._isSymLink folderPath, (err, isSymLink)->
if isSymLink
logger.log user_id:user_id, project_id:project_id, parent_folder_id:parent_folder_id, folderPath:folderPath, "add folder contents is from symlink, stopping insert"
return callback("path is symlink")
fs.readdir folderPath, (error, entries = []) =>
return callback(error) if error?
jobs = _.map entries, (entry) =>
(callback) =>
FileTypeManager.shouldIgnore entry, (error, ignore) =>
return callback(error) if error?
if !ignore
FileSystemImportManager.addEntity user_id, project_id, parent_folder_id, entry, "#{folderPath}/#{entry}", replace, callback
else
callback()
async.parallelLimit jobs, 5, callback
addEntity: (user_id, project_id, folder_id, name, path, replace, callback = (error, entity)-> ) ->
FileTypeManager.isDirectory path, (error, isDirectory) =>
return callback(error) if error?
if isDirectory
@addFolder user_id, project_id, folder_id, name, path, replace, callback
else
FileTypeManager.isBinary name, path, (error, isBinary) =>
return callback(error) if error?
if isBinary
@addFile user_id, project_id, folder_id, name, path, replace, callback
else
@addDoc user_id, project_id, folder_id, name, path, replace, callback
FileSystemImportManager._isSymLink path, (err, isSymLink)->
if isSymLink
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, path:path, "add entry is from symlink, stopping insert"
return callback("path is symlink")
FileTypeManager.isDirectory path, (error, isDirectory) =>
return callback(error) if error?
if isDirectory
FileSystemImportManager.addFolder user_id, project_id, folder_id, name, path, replace, callback
else
FileTypeManager.isBinary name, path, (error, isBinary) =>
return callback(error) if error?
if isBinary
FileSystemImportManager.addFile user_id, project_id, folder_id, name, path, replace, callback
else
FileSystemImportManager.addDoc user_id, project_id, folder_id, name, path, replace, callback
_isSymLink: (path, callback = (err, isSymLink)->)->
fs.lstat path, (err, stat)->
if err?
logger.err err:err, "error with path symlink check"
return callback(err)
isSymLink = stat.isSymbolicLink()
callback(err, isSymLink)

View file

@ -10,6 +10,7 @@ describe "FileStoreHandler", ->
beforeEach ->
@fs =
createReadStream : sinon.stub()
lstat: sinon.stub().callsArgWith(1, null, {isSymbolicLink:=>@isSymlink})
@writeStream =
my:"writeStream"
on: (type, cb)->
@ -31,6 +32,7 @@ describe "FileStoreHandler", ->
describe "uploadFileFromDisk", ->
beforeEach ->
@request.returns(@writeStream)
@isSymlink = false
it "should create read stream", (done)->
@fs.createReadStream.returns
@ -74,6 +76,13 @@ describe "FileStoreHandler", ->
@handler._buildUrl.calledWith(@project_id, @file_id).should.equal true
done()
describe "symlink", ->
it "should not read file if it is symlink", (done)->
@isSymlink = true
@handler.uploadFileFromDisk @project_id, @file_id, @fsPath, =>
@fs.createReadStream.called.should.equal false
done()
describe "deleteFile", ->
it "should send a delete request to filestore api", (done)->

View file

@ -18,12 +18,29 @@ describe "FileSystemImportManager", ->
"../Editor/EditorController": @EditorController = {}
"./FileTypeManager": @FileTypeManager = {}
"../Project/ProjectLocator": @ProjectLocator = {}
"logger-sharelatex":
log:->
err:->
describe "addDoc", ->
beforeEach ->
@docContent = "one\ntwo\nthree"
@docLines = @docContent.split("\n")
@fs.readFile = sinon.stub().callsArgWith(2, null, @docContent)
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
describe "when path is symlink", ->
beforeEach ->
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, true)
@EditorController.addDocWithoutLock = sinon.stub()
@FileSystemImportManager.addDoc @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
it "should not read the file from disk", ->
@fs.readFile.called.should.equal false
it "should not insert the doc", ->
@EditorController.addDocWithoutLock.called.should.equal false
describe "with replace set to false", ->
beforeEach ->
@ -98,12 +115,24 @@ describe "FileSystemImportManager", ->
describe "addFile with replace set to false", ->
beforeEach ->
@EditorController.addFileWithoutLock = sinon.stub().callsArg(5)
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
it "should add the file", ->
@EditorController.addFileWithoutLock.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload")
.should.equal true
describe "addFile with symlink", ->
beforeEach ->
@EditorController.addFileWithoutLock = sinon.stub()
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, true)
@EditorController.replaceFile = sinon.stub()
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
it "should node add the file", ->
@EditorController.addFileWithoutLock.called.should.equal false
@EditorController.replaceFile.called.should.equal false
describe "addFile with replace set to true", ->
describe "when the file doesn't exist", ->
beforeEach ->
@ -113,6 +142,7 @@ describe "FileSystemImportManager", ->
name: "not-the-right-file.tex"
}]
}
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder)
@EditorController.addFileWithoutLock = sinon.stub().callsArg(5)
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, true, @callback
@ -137,6 +167,7 @@ describe "FileSystemImportManager", ->
name: "not-the-right-file.tex"
}]
}
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder)
@EditorController.replaceFile = sinon.stub().callsArg(4)
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, true, @callback
@ -151,20 +182,34 @@ describe "FileSystemImportManager", ->
.should.equal true
describe "addFolder", ->
beforeEach ->
@new_folder_id = "new-folder-id"
@EditorController.addFolderWithoutLock = sinon.stub().callsArgWith(4, null, _id: @new_folder_id)
@FileSystemImportManager.addFolderContents = sinon.stub().callsArg(5)
@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should add a folder to the project", ->
@EditorController.addFolderWithoutLock.calledWith(@project_id, @folder_id, @name, "upload")
.should.equal true
describe "successfully", ->
beforeEach ->
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should add the folders contents", ->
@FileSystemImportManager.addFolderContents.calledWith(@user_id, @project_id, @new_folder_id, @path_on_disk, @replace)
.should.equal true
it "should add a folder to the project", ->
@EditorController.addFolderWithoutLock.calledWith(@project_id, @folder_id, @name, "upload")
.should.equal true
it "should add the folders contents", ->
@FileSystemImportManager.addFolderContents.calledWith(@user_id, @project_id, @new_folder_id, @path_on_disk, @replace)
.should.equal true
describe "with symlink", ->
beforeEach ->
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should not add a folder to the project", ->
@EditorController.addFolderWithoutLock.called.should.equal false
@FileSystemImportManager.addFolderContents.called.should.equal false
describe "addFolderContents", ->
beforeEach ->
@folderEntries = ["path1", "path2", "path3"]
@ -173,6 +218,7 @@ describe "FileSystemImportManager", ->
@FileSystemImportManager.addEntity = sinon.stub().callsArg(6)
@FileTypeManager.shouldIgnore = (path, callback) =>
callback null, @ignoredEntries.indexOf(require("path").basename(path)) != -1
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addFolderContents @user_id, @project_id, @folder_id, @path_on_disk, @replace, @callback
it "should call addEntity for each file in the folder which is not ignored", ->
@ -188,11 +234,12 @@ describe "FileSystemImportManager", ->
it "should look in the correct directory", ->
@fs.readdir.calledWith(@path_on_disk).should.equal true
describe "addEntity", ->
describe "addEntity dddddddd", ->
describe "with directory", ->
beforeEach ->
@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFolder = sinon.stub().callsArg(6)
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should call addFolder", ->
@ -203,6 +250,7 @@ describe "FileSystemImportManager", ->
beforeEach ->
@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, false)
@FileTypeManager.isBinary = sinon.stub().callsArgWith(2, null, true)
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addFile = sinon.stub().callsArg(6)
@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
@ -215,6 +263,7 @@ describe "FileSystemImportManager", ->
@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, false)
@FileTypeManager.isBinary = sinon.stub().callsArgWith(2, null, false)
@FileSystemImportManager.addDoc = sinon.stub().callsArg(6)
@FileSystemImportManager._isSymLink = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should call addFile", ->