Merge branch 'fschecks'

This commit is contained in:
Henry Oswald 2016-03-13 15:52:13 +00:00
commit 88f8b701c8
6 changed files with 308 additions and 92 deletions

View file

@ -6,24 +6,32 @@ settings = require("settings-sharelatex")
oneMinInMs = 60 * 1000
fiveMinsInMs = oneMinInMs * 5
module.exports =
module.exports = FileStoreHandler =
uploadFileFromDisk: (project_id, file_id, fsPath, callback)->
logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "uploading file from disk"
readStream = fs.createReadStream(fsPath)
opts =
method: "post"
uri: @_buildUrl(project_id, file_id)
timeout:fiveMinsInMs
writeStream = request(opts)
readStream.pipe writeStream
writeStream.on "end", callback
readStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the read stream of uploadFileFromDisk"
callback err
writeStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the write stream of uploadFileFromDisk"
callback err
fs.lstat fsPath, (err, stat)->
if err?
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "error stating file"
callback(err)
if !stat.isFile()
logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "tried to upload symlink, not contining"
return callback(new Error("can not upload symlink"))
logger.log project_id:project_id, file_id:file_id, fsPath:fsPath, "uploading file from disk"
readStream = fs.createReadStream(fsPath)
opts =
method: "post"
uri: FileStoreHandler._buildUrl(project_id, file_id)
timeout:fiveMinsInMs
writeStream = request(opts)
readStream.pipe writeStream
writeStream.on "end", callback
readStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the read stream of uploadFileFromDisk"
callback err
writeStream.on "error", (err)->
logger.err err:err, project_id:project_id, file_id:file_id, fsPath:fsPath, "something went wrong on the write stream of uploadFileFromDisk"
callback err
getFileStream: (project_id, file_id, query, callback)->
logger.log project_id:project_id, file_id:file_id, query:query, "getting file stream from file store"

View file

@ -3,21 +3,21 @@ logger = require "logger-sharelatex"
metrics = require "../../infrastructure/Metrics"
fs = require "fs"
Path = require "path"
_ = require("underscore")
ONE_MEG = 1024 * 1024
module.exports = ArchiveManager =
extractZipArchive: (source, destination, _callback = (err) ->) ->
callback = (args...) ->
_callback(args...)
_callback = () ->
timer = new metrics.Timer("unzipDirectory")
logger.log source: source, destination: destination, "unzipping file"
unzip = child.spawn("unzip", [source, "-d", destination])
_isZipTooLarge: (source, callback = (err, isTooLarge)->)->
callback = _.once callback
# don't remove this line, some zips need
# us to listen on this for some unknow reason
unzip = child.spawn("unzip", ["-l", source])
output = ""
unzip.stdout.on "data", (d)->
output += d
error = null
unzip.stderr.on "data", (chunk) ->
@ -31,11 +31,68 @@ module.exports = ArchiveManager =
callback(err)
unzip.on "exit", () ->
timer.done()
if error?
error = new Error(error)
logger.error err:error, source: source, destination: destination, "error unzipping file"
callback(error)
logger.error err:error, source: source, destination: destination, "error checking zip size"
lines = output.split("\n")
lastLine = lines[lines.length - 2]?.trim()
totalSizeInBytes = lastLine?.split(" ")?[0]
totalSizeInBytes = parseInt(totalSizeInBytes)
if !totalSizeInBytes? or isNaN(totalSizeInBytes)
logger.err source:source, "error getting bytes of zip"
return callback(new Error("something went wrong"))
isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
callback(error, isTooLarge)
extractZipArchive: (source, destination, _callback = (err) ->) ->
callback = (args...) ->
_callback(args...)
_callback = () ->
ArchiveManager._isZipTooLarge source, (err, isTooLarge)->
if err?
logger.err err:err, "error checking size of zip file"
return callback(err)
if isTooLarge
return callback(new Error("zip_too_large"))
timer = new metrics.Timer("unzipDirectory")
logger.log source: source, destination: destination, "unzipping file"
unzip = child.spawn("unzip", [source, "-d", destination])
# don't remove this line, some zips need
# us to listen on this for some unknow reason
unzip.stdout.on "data", (d)->
error = null
unzip.stderr.on "data", (chunk) ->
error ||= ""
error += chunk
unzip.on "error", (err) ->
logger.error {err, source, destination}, "unzip failed"
if err.code == "ENOENT"
logger.error "unzip command not found. Please check the unzip command is installed"
callback(err)
unzip.on "exit", () ->
timer.done()
if error?
error = new Error(error)
logger.error err:error, source: source, destination: destination, "error unzipping file"
callback(error)
findTopLevelDirectory: (directory, callback = (error, topLevelDir) ->) ->
fs.readdir directory, (error, files) ->

View file

@ -4,76 +4,108 @@ _ = require "underscore"
FileTypeManager = require "./FileTypeManager"
EditorController = require "../Editor/EditorController"
ProjectLocator = require "../Project/ProjectLocator"
logger = require("logger-sharelatex")
module.exports = FileSystemImportManager =
addDoc: (user_id, project_id, folder_id, name, path, replace, callback = (error, doc)-> )->
fs.readFile path, "utf8", (error, content = "") ->
return callback(error) if error?
content = content.replace(/\r/g, "")
lines = content.split("\n")
if replace
FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
if !isSafe
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, name:name, path:path, "add doc is from symlink, stopping process"
return callback("path is symlink")
fs.readFile path, "utf8", (error, content = "") ->
return callback(error) if error?
content = content.replace(/\r/g, "")
lines = content.split("\n")
if replace
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
existingDoc = null
for doc in folder.docs
if doc.name == name
existingDoc = doc
break
if existingDoc?
EditorController.setDoc project_id, existingDoc._id, user_id, lines, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
addFile: (user_id, project_id, folder_id, name, path, replace, callback = (error, file)-> )->
FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
if !isSafe
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, name:name, path:path, "add file is from symlink, stopping insert"
return callback("path is symlink")
if !replace
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
else
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
existingDoc = null
for doc in folder.docs
if doc.name == name
existingDoc = doc
existingFile = null
for fileRef in folder.fileRefs
if fileRef.name == name
existingFile = fileRef
break
if existingDoc?
EditorController.setDoc project_id, existingDoc._id, user_id, lines, "upload", callback
if existingFile?
EditorController.replaceFile project_id, existingFile._id, path, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
else
EditorController.addDocWithoutLock project_id, folder_id, name, lines, "upload", callback
addFile: (user_id, project_id, folder_id, name, path, replace, callback = (error, file)-> )->
if replace
ProjectLocator.findElement project_id: project_id, element_id: folder_id, type: "folder", (error, folder) ->
return callback(error) if error?
return callback(new Error("Couldn't find folder")) if !folder?
existingFile = null
for fileRef in folder.fileRefs
if fileRef.name == name
existingFile = fileRef
break
if existingFile?
EditorController.replaceFile project_id, existingFile._id, path, "upload", callback
else
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
else
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
EditorController.addFileWithoutLock project_id, folder_id, name, path, "upload", callback
addFolder: (user_id, project_id, folder_id, name, path, replace, callback = (error)-> ) ->
EditorController.addFolderWithoutLock project_id, folder_id, name, "upload", (error, new_folder) =>
return callback(error) if error?
@addFolderContents user_id, project_id, new_folder._id, path, replace, (error) ->
FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
if !isSafe
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, path:path, "add folder is from symlink, stopping insert"
return callback("path is symlink")
EditorController.addFolderWithoutLock project_id, folder_id, name, "upload", (error, new_folder) =>
return callback(error) if error?
callback null, new_folder
FileSystemImportManager.addFolderContents user_id, project_id, new_folder._id, path, replace, (error) ->
return callback(error) if error?
callback null, new_folder
addFolderContents: (user_id, project_id, parent_folder_id, folderPath, replace, callback = (error)-> ) ->
fs.readdir folderPath, (error, entries = []) =>
return callback(error) if error?
jobs = _.map entries, (entry) =>
(callback) =>
FileTypeManager.shouldIgnore entry, (error, ignore) =>
return callback(error) if error?
if !ignore
@addEntity user_id, project_id, parent_folder_id, entry, "#{folderPath}/#{entry}", replace, callback
else
callback()
async.parallelLimit jobs, 5, callback
FileSystemImportManager._isSafeOnFileSystem folderPath, (err, isSafe)->
if !isSafe
logger.log user_id:user_id, project_id:project_id, parent_folder_id:parent_folder_id, folderPath:folderPath, "add folder contents is from symlink, stopping insert"
return callback("path is symlink")
fs.readdir folderPath, (error, entries = []) =>
return callback(error) if error?
jobs = _.map entries, (entry) =>
(callback) =>
FileTypeManager.shouldIgnore entry, (error, ignore) =>
return callback(error) if error?
if !ignore
FileSystemImportManager.addEntity user_id, project_id, parent_folder_id, entry, "#{folderPath}/#{entry}", replace, callback
else
callback()
async.parallelLimit jobs, 5, callback
addEntity: (user_id, project_id, folder_id, name, path, replace, callback = (error, entity)-> ) ->
FileTypeManager.isDirectory path, (error, isDirectory) =>
return callback(error) if error?
if isDirectory
@addFolder user_id, project_id, folder_id, name, path, replace, callback
else
FileTypeManager.isBinary name, path, (error, isBinary) =>
return callback(error) if error?
if isBinary
@addFile user_id, project_id, folder_id, name, path, replace, callback
else
@addDoc user_id, project_id, folder_id, name, path, replace, callback
FileSystemImportManager._isSafeOnFileSystem path, (err, isSafe)->
if !isSafe
logger.log user_id:user_id, project_id:project_id, folder_id:folder_id, path:path, "add entry is from symlink, stopping insert"
return callback("path is symlink")
FileTypeManager.isDirectory path, (error, isDirectory) =>
return callback(error) if error?
if isDirectory
FileSystemImportManager.addFolder user_id, project_id, folder_id, name, path, replace, callback
else
FileTypeManager.isBinary name, path, (error, isBinary) =>
return callback(error) if error?
if isBinary
FileSystemImportManager.addFile user_id, project_id, folder_id, name, path, replace, callback
else
FileSystemImportManager.addDoc user_id, project_id, folder_id, name, path, replace, callback
_isSafeOnFileSystem: (path, callback = (err, isSafe)->)->
fs.lstat path, (err, stat)->
if err?
logger.err err:err, "error with path symlink check"
return callback(err)
isSafe = stat.isFile() or stat.isDirectory()
callback(err, isSafe)

View file

@ -10,6 +10,10 @@ describe "FileStoreHandler", ->
beforeEach ->
@fs =
createReadStream : sinon.stub()
lstat: sinon.stub().callsArgWith(1, null, {
isFile:=> @isSafeOnFileSystem
isDirectory:-> return false
})
@writeStream =
my:"writeStream"
on: (type, cb)->
@ -31,6 +35,7 @@ describe "FileStoreHandler", ->
describe "uploadFileFromDisk", ->
beforeEach ->
@request.returns(@writeStream)
@isSafeOnFileSystem = true
it "should create read stream", (done)->
@fs.createReadStream.returns
@ -74,6 +79,13 @@ describe "FileStoreHandler", ->
@handler._buildUrl.calledWith(@project_id, @file_id).should.equal true
done()
describe "symlink", ->
it "should not read file if it is symlink", (done)->
@isSafeOnFileSystem = false
@handler.uploadFileFromDisk @project_id, @file_id, @fsPath, =>
@fs.createReadStream.called.should.equal false
done()
describe "deleteFile", ->
it "should send a delete request to filestore api", (done)->

View file

@ -1,4 +1,5 @@
sinon = require('sinon')
expect = require("chai").expect
chai = require('chai')
should = chai.should()
modulePath = "../../../../app/js/Features/Uploads/ArchiveManager.js"
@ -9,12 +10,16 @@ describe "ArchiveManager", ->
beforeEach ->
@logger =
error: sinon.stub()
err:->
log: sinon.stub()
@process = new events.EventEmitter
@process.stdout = new events.EventEmitter
@process.stderr = new events.EventEmitter
@child =
spawn: sinon.stub().returns(@process)
@metrics =
Timer: class Timer
done: sinon.stub()
@ -29,6 +34,7 @@ describe "ArchiveManager", ->
@source = "/path/to/zip/source.zip"
@destination = "/path/to/zip/destination"
@callback = sinon.stub()
@ArchiveManager._isZipTooLarge = sinon.stub().callsArgWith(1, null, false)
describe "successfully", ->
beforeEach (done) ->
@ -58,6 +64,19 @@ describe "ArchiveManager", ->
it "should log out the error", ->
@logger.error.called.should.equal true
describe "with a zip that is too large", ->
beforeEach (done) ->
@ArchiveManager._isZipTooLarge = sinon.stub().callsArgWith(1, null, true)
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
@callback(error)
done()
it "should return the callback with an error", ->
@callback.calledWithExactly(new Error("zip_too_large")).should.equal true
it "should not call spawn", ->
@child.spawn.called.should.equal false
describe "with an error on the process", ->
beforeEach (done) ->
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
@ -71,6 +90,45 @@ describe "ArchiveManager", ->
it "should log out the error", ->
@logger.error.called.should.equal true
describe "_isZipTooLarge", ->
beforeEach ->
@output = (totalSize)->" Length Date Time Name \n-------- ---- ---- ---- \n241 03-12-16 12:20 main.tex \n108801 03-12-16 12:20 ddd/x1J5kHh.jpg \n-------- ------- \n#{totalSize} 2 files\n"
it "should return false with small output", (done)->
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
isTooLarge.should.equal false
done()
@process.stdout.emit "data", @output("109042")
@process.emit "exit"
it "should return true with large bytes", (done)->
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
isTooLarge.should.equal true
done()
@process.stdout.emit "data", @output("1090000000000000042")
@process.emit "exit"
it "should return error on no data", (done)->
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
expect(error).to.exist
done()
@process.stdout.emit "data", ""
@process.emit "exit"
it "should return error if it didn't get a number", (done)->
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
expect(error).to.exist
done()
@process.stdout.emit "data", @output("total_size_string")
@process.emit "exit"
it "should return error if the is only a bit of data", (done)->
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
expect(error).to.exist
done()
@process.stdout.emit "data", " Length Date Time Name \n--------"
@process.emit "exit"
describe "findTopLevelDirectory", ->
beforeEach ->
@fs.readdir = sinon.stub()

View file

@ -18,12 +18,29 @@ describe "FileSystemImportManager", ->
"../Editor/EditorController": @EditorController = {}
"./FileTypeManager": @FileTypeManager = {}
"../Project/ProjectLocator": @ProjectLocator = {}
"logger-sharelatex":
log:->
err:->
describe "addDoc", ->
beforeEach ->
@docContent = "one\ntwo\nthree"
@docLines = @docContent.split("\n")
@fs.readFile = sinon.stub().callsArgWith(2, null, @docContent)
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
describe "when path is symlink", ->
beforeEach ->
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, false)
@EditorController.addDocWithoutLock = sinon.stub()
@FileSystemImportManager.addDoc @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
it "should not read the file from disk", ->
@fs.readFile.called.should.equal false
it "should not insert the doc", ->
@EditorController.addDocWithoutLock.called.should.equal false
describe "with replace set to false", ->
beforeEach ->
@ -98,12 +115,24 @@ describe "FileSystemImportManager", ->
describe "addFile with replace set to false", ->
beforeEach ->
@EditorController.addFileWithoutLock = sinon.stub().callsArg(5)
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
it "should add the file", ->
@EditorController.addFileWithoutLock.calledWith(@project_id, @folder_id, @name, @path_on_disk, "upload")
.should.equal true
describe "addFile with symlink", ->
beforeEach ->
@EditorController.addFileWithoutLock = sinon.stub()
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, false)
@EditorController.replaceFile = sinon.stub()
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, false, @callback
it "should node add the file", ->
@EditorController.addFileWithoutLock.called.should.equal false
@EditorController.replaceFile.called.should.equal false
describe "addFile with replace set to true", ->
describe "when the file doesn't exist", ->
beforeEach ->
@ -113,6 +142,7 @@ describe "FileSystemImportManager", ->
name: "not-the-right-file.tex"
}]
}
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder)
@EditorController.addFileWithoutLock = sinon.stub().callsArg(5)
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, true, @callback
@ -137,6 +167,7 @@ describe "FileSystemImportManager", ->
name: "not-the-right-file.tex"
}]
}
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@ProjectLocator.findElement = sinon.stub().callsArgWith(1, null, @folder)
@EditorController.replaceFile = sinon.stub().callsArg(4)
@FileSystemImportManager.addFile @user_id, @project_id, @folder_id, @name, @path_on_disk, true, @callback
@ -151,20 +182,34 @@ describe "FileSystemImportManager", ->
.should.equal true
describe "addFolder", ->
beforeEach ->
@new_folder_id = "new-folder-id"
@EditorController.addFolderWithoutLock = sinon.stub().callsArgWith(4, null, _id: @new_folder_id)
@FileSystemImportManager.addFolderContents = sinon.stub().callsArg(5)
@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should add a folder to the project", ->
@EditorController.addFolderWithoutLock.calledWith(@project_id, @folder_id, @name, "upload")
.should.equal true
describe "successfully", ->
beforeEach ->
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should add the folders contents", ->
@FileSystemImportManager.addFolderContents.calledWith(@user_id, @project_id, @new_folder_id, @path_on_disk, @replace)
.should.equal true
it "should add a folder to the project", ->
@EditorController.addFolderWithoutLock.calledWith(@project_id, @folder_id, @name, "upload")
.should.equal true
it "should add the folders contents", ->
@FileSystemImportManager.addFolderContents.calledWith(@user_id, @project_id, @new_folder_id, @path_on_disk, @replace)
.should.equal true
describe "with symlink", ->
beforeEach ->
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, false)
@FileSystemImportManager.addFolder @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should not add a folder to the project", ->
@EditorController.addFolderWithoutLock.called.should.equal false
@FileSystemImportManager.addFolderContents.called.should.equal false
describe "addFolderContents", ->
beforeEach ->
@folderEntries = ["path1", "path2", "path3"]
@ -173,6 +218,7 @@ describe "FileSystemImportManager", ->
@FileSystemImportManager.addEntity = sinon.stub().callsArg(6)
@FileTypeManager.shouldIgnore = (path, callback) =>
callback null, @ignoredEntries.indexOf(require("path").basename(path)) != -1
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFolderContents @user_id, @project_id, @folder_id, @path_on_disk, @replace, @callback
it "should call addEntity for each file in the folder which is not ignored", ->
@ -193,6 +239,7 @@ describe "FileSystemImportManager", ->
beforeEach ->
@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFolder = sinon.stub().callsArg(6)
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should call addFolder", ->
@ -203,6 +250,7 @@ describe "FileSystemImportManager", ->
beforeEach ->
@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, false)
@FileTypeManager.isBinary = sinon.stub().callsArgWith(2, null, true)
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addFile = sinon.stub().callsArg(6)
@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
@ -215,6 +263,7 @@ describe "FileSystemImportManager", ->
@FileTypeManager.isDirectory = sinon.stub().callsArgWith(1, null, false)
@FileTypeManager.isBinary = sinon.stub().callsArgWith(2, null, false)
@FileSystemImportManager.addDoc = sinon.stub().callsArg(6)
@FileSystemImportManager._isSafeOnFileSystem = sinon.stub().callsArgWith(1, null, true)
@FileSystemImportManager.addEntity @user_id, @project_id, @folder_id, @name, @path_on_disk, @replace, @callback
it "should call addFile", ->