mirror of
https://github.com/overleaf/overleaf.git
synced 2025-04-08 08:01:57 +00:00
Merge pull request #60 from sharelatex/bg-fix-unzip-permission
replace unzip with yauzl (connects to #219)
This commit is contained in:
Brian Gough
GitHub
commit
f8662d8aaa
4 changed files with 4169 additions and 1786 deletions
|
|
@ -1,56 +1,105 @@
|
|||
child = require "child_process"
|
||||
logger = require "logger-sharelatex"
|
||||
metrics = require "metrics-sharelatex"
|
||||
fs = require "fs"
|
||||
Path = require "path"
|
||||
fse = require "fs-extra"
|
||||
yauzl = require "yauzl"
|
||||
Settings = require "settings-sharelatex"
|
||||
_ = require("underscore")
|
||||
|
||||
ONE_MEG = 1024 * 1024
|
||||
|
||||
module.exports = ArchiveManager =
|
||||
|
||||
|
||||
_isZipTooLarge: (source, callback = (err, isTooLarge)->)->
|
||||
callback = _.once callback
|
||||
|
||||
unzip = child.spawn("unzip", ["-l", source])
|
||||
totalSizeInBytes = null
|
||||
yauzl.open source, {lazyEntries: true}, (err, zipfile) ->
|
||||
return callback(err) if err?
|
||||
|
||||
output = ""
|
||||
unzip.stdout.on "data", (d)->
|
||||
output += d
|
||||
if Settings.maxEntitiesPerProject? and zipfile.entryCount > Settings.maxEntitiesPerProject
|
||||
return callback(null, true) # too many files in zip file
|
||||
|
||||
error = null
|
||||
unzip.stderr.on "data", (chunk) ->
|
||||
error ||= ""
|
||||
error += chunk
|
||||
zipfile.on "error", callback
|
||||
|
||||
unzip.on "error", (err) ->
|
||||
logger.error {err, source}, "unzip failed"
|
||||
if err.code == "ENOENT"
|
||||
logger.error "unzip command not found. Please check the unzip command is installed"
|
||||
callback(err)
|
||||
# read all the entries
|
||||
zipfile.readEntry()
|
||||
zipfile.on "entry", (entry) ->
|
||||
totalSizeInBytes += entry.uncompressedSize
|
||||
zipfile.readEntry() # get the next entry
|
||||
|
||||
unzip.on "close", (exitCode) ->
|
||||
if error?
|
||||
error = new Error(error)
|
||||
logger.warn err:error, source: source, "error checking zip size"
|
||||
# no more entries to read
|
||||
zipfile.on "end", () ->
|
||||
if !totalSizeInBytes? or isNaN(totalSizeInBytes)
|
||||
logger.err source:source, totalSizeInBytes:totalSizeInBytes, "error getting bytes of zip"
|
||||
return callback(new Error("error getting bytes of zip"))
|
||||
isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
|
||||
callback(null, isTooLarge)
|
||||
|
||||
lines = output.split("\n")
|
||||
lastLine = lines[lines.length - 2]?.trim()
|
||||
totalSizeInBytes = lastLine?.split(" ")?[0]
|
||||
_checkFilePath: (entry, destination, callback = (err, destFile) ->) ->
|
||||
# check if the entry is a directory
|
||||
endsWithSlash = /\/$/
|
||||
if endsWithSlash.test(entry.fileName)
|
||||
return callback() # don't give a destfile for directory
|
||||
# check that the file does not use a relative path
|
||||
for dir in entry.fileName.split('/')
|
||||
if dir == '..'
|
||||
return callback(new Error("relative path"))
|
||||
# check that the destination file path is normalized
|
||||
dest = "#{destination}/#{entry.fileName}"
|
||||
if dest != Path.normalize(dest)
|
||||
return callback(new Error("unnormalized path"))
|
||||
else
|
||||
return callback(null, dest)
|
||||
|
||||
totalSizeInBytesAsInt = parseInt(totalSizeInBytes)
|
||||
_writeFileEntry: (zipfile, entry, destFile, callback = (err)->) ->
|
||||
callback = _.once callback
|
||||
|
||||
if !totalSizeInBytesAsInt? or isNaN(totalSizeInBytesAsInt)
|
||||
logger.err source:source, totalSizeInBytes:totalSizeInBytes, totalSizeInBytesAsInt:totalSizeInBytesAsInt, lastLine:lastLine, exitCode:exitCode, "error getting bytes of zip"
|
||||
return callback(new Error("error getting bytes of zip"))
|
||||
zipfile.openReadStream entry, (err, readStream) ->
|
||||
return callback(err) if err?
|
||||
readStream.on "error", callback
|
||||
readStream.on "end", callback
|
||||
|
||||
isTooLarge = totalSizeInBytes > (ONE_MEG * 300)
|
||||
errorHandler = (err) -> # clean up before calling callback
|
||||
readStream.unpipe()
|
||||
readStream.destroy()
|
||||
callback(err)
|
||||
|
||||
callback(error, isTooLarge)
|
||||
fse.ensureDir Path.dirname(destFile), (err) ->
|
||||
return errorHandler(err) if err?
|
||||
writeStream = fs.createWriteStream destFile
|
||||
writeStream.on 'error', errorHandler
|
||||
readStream.pipe(writeStream)
|
||||
|
||||
_extractZipFiles: (source, destination, callback = (err) ->) ->
|
||||
callback = _.once callback
|
||||
|
||||
yauzl.open source, {lazyEntries: true}, (err, zipfile) ->
|
||||
return callback(err) if err?
|
||||
zipfile.on "error", callback
|
||||
# read all the entries
|
||||
zipfile.readEntry()
|
||||
zipfile.on "entry", (entry) ->
|
||||
logger.log {source:source, fileName: entry.fileName}, "processing zip file entry"
|
||||
ArchiveManager._checkFilePath entry, destination, (err, destFile) ->
|
||||
if err?
|
||||
logger.warn err:err, source:source, destination:destination, "skipping bad file path"
|
||||
zipfile.readEntry() # bad path, just skip to the next file
|
||||
return
|
||||
if destFile? # only write files
|
||||
ArchiveManager._writeFileEntry zipfile, entry, destFile, (err) ->
|
||||
if err?
|
||||
logger.error err:err, source:source, destFile:destFile, "error unzipping file entry"
|
||||
zipfile.close() # bail out, stop reading file entries
|
||||
return callback(err)
|
||||
else
|
||||
zipfile.readEntry() # continue to the next file
|
||||
else # if it's a directory, continue
|
||||
zipfile.readEntry()
|
||||
# no more entries to read
|
||||
zipfile.on "end", callback
|
||||
|
||||
|
||||
extractZipArchive: (source, destination, _callback = (err) ->) ->
|
||||
callback = (args...) ->
|
||||
_callback(args...)
|
||||
|
|
@ -62,36 +111,19 @@ module.exports = ArchiveManager =
|
|||
return callback(err)
|
||||
|
||||
if isTooLarge
|
||||
return callback(new Error("zip_too_large"))
|
||||
|
||||
return callback(new Error("zip_too_large"))
|
||||
|
||||
timer = new metrics.Timer("unzipDirectory")
|
||||
logger.log source: source, destination: destination, "unzipping file"
|
||||
|
||||
unzip = child.spawn("unzip", [source, "-d", destination])
|
||||
|
||||
# don't remove this line, some zips need
|
||||
# us to listen on this for some unknow reason
|
||||
unzip.stdout.on "data", (d)->
|
||||
|
||||
error = null
|
||||
unzip.stderr.on "data", (chunk) ->
|
||||
error ||= ""
|
||||
error += chunk
|
||||
|
||||
unzip.on "error", (err) ->
|
||||
logger.error {err, source, destination}, "unzip failed"
|
||||
if err.code == "ENOENT"
|
||||
logger.error "unzip command not found. Please check the unzip command is installed"
|
||||
callback(err)
|
||||
|
||||
unzip.on "close", () ->
|
||||
ArchiveManager._extractZipFiles source, destination, (err) ->
|
||||
timer.done()
|
||||
if error?
|
||||
error = new Error(error)
|
||||
logger.error err:error, source: source, destination: destination, "error unzipping file"
|
||||
callback(error)
|
||||
|
||||
if err?
|
||||
logger.error {err, source, destination}, "unzip failed"
|
||||
callback(err)
|
||||
else
|
||||
callback()
|
||||
|
||||
findTopLevelDirectory: (directory, callback = (error, topLevelDir) ->) ->
|
||||
fs.readdir directory, (error, files) ->
|
||||
return callback(error) if error?
|
||||
|
|
|
|||
5631
services/web/npm-shrinkwrap.json
generated
5631
services/web/npm-shrinkwrap.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -25,6 +25,7 @@
|
|||
"dateformat": "1.0.4-1.2.3",
|
||||
"express": "4.13.0",
|
||||
"express-session": "^1.14.2",
|
||||
"fs-extra": "^4.0.2",
|
||||
"heapdump": "^0.3.7",
|
||||
"helmet": "^3.8.1",
|
||||
"http-proxy": "^1.8.1",
|
||||
|
|
@ -69,7 +70,8 @@
|
|||
"underscore": "1.6.0",
|
||||
"uuid": "^3.0.1",
|
||||
"v8-profiler": "^5.2.3",
|
||||
"xml2js": "0.2.0"
|
||||
"xml2js": "0.2.0",
|
||||
"yauzl": "^2.8.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"autoprefixer": "^6.6.1",
|
||||
|
|
|
|||
|
|
@ -10,24 +10,22 @@ describe "ArchiveManager", ->
|
|||
beforeEach ->
|
||||
@logger =
|
||||
error: sinon.stub()
|
||||
warn: sinon.stub()
|
||||
err:->
|
||||
log: sinon.stub()
|
||||
@process = new events.EventEmitter
|
||||
@process.stdout = new events.EventEmitter
|
||||
@process.stderr = new events.EventEmitter
|
||||
|
||||
@child =
|
||||
spawn: sinon.stub().returns(@process)
|
||||
|
||||
|
||||
@metrics =
|
||||
Timer: class Timer
|
||||
done: sinon.stub()
|
||||
@zipfile = new events.EventEmitter
|
||||
@zipfile.readEntry = sinon.stub()
|
||||
@zipfile.close = sinon.stub()
|
||||
|
||||
@ArchiveManager = SandboxedModule.require modulePath, requires:
|
||||
"child_process": @child
|
||||
"yauzl": @yauzl = {open: sinon.stub().callsArgWith(2, null, @zipfile)}
|
||||
"logger-sharelatex": @logger
|
||||
"metrics-sharelatex": @metrics
|
||||
"fs": @fs = {}
|
||||
"fs-extra": @fse = {}
|
||||
|
||||
describe "extractZipArchive", ->
|
||||
beforeEach ->
|
||||
|
|
@ -39,10 +37,10 @@ describe "ArchiveManager", ->
|
|||
describe "successfully", ->
|
||||
beforeEach (done) ->
|
||||
@ArchiveManager.extractZipArchive @source, @destination, done
|
||||
@process.emit "close"
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should run unzip", ->
|
||||
@child.spawn.calledWithExactly("unzip", [@source, "-d", @destination]).should.equal true
|
||||
it "should run yauzl", ->
|
||||
@yauzl.open.calledWith(@source).should.equal true
|
||||
|
||||
it "should time the unzip", ->
|
||||
@metrics.Timer::done.called.should.equal true
|
||||
|
|
@ -50,13 +48,12 @@ describe "ArchiveManager", ->
|
|||
it "should log the unzip", ->
|
||||
@logger.log.calledWith(sinon.match.any, "unzipping file").should.equal true
|
||||
|
||||
describe "with an error on stderr", ->
|
||||
describe "with an error in the zip file header", ->
|
||||
beforeEach (done) ->
|
||||
@yauzl.open = sinon.stub().callsArgWith(2, new Error("Something went wrong"))
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@process.stderr.emit "data", "Something went wrong"
|
||||
@process.emit "close"
|
||||
|
||||
it "should return the callback with an error", ->
|
||||
@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
|
||||
|
|
@ -74,60 +71,177 @@ describe "ArchiveManager", ->
|
|||
it "should return the callback with an error", ->
|
||||
@callback.calledWithExactly(new Error("zip_too_large")).should.equal true
|
||||
|
||||
it "should not call spawn", ->
|
||||
@child.spawn.called.should.equal false
|
||||
it "should not call yauzl.open", ->
|
||||
@yauzl.open.called.should.equal false
|
||||
|
||||
describe "with an error on the process", ->
|
||||
describe "with an error in the extracted files", ->
|
||||
beforeEach (done) ->
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@process.emit "error", new Error("Something went wrong")
|
||||
@zipfile.emit "error", new Error("Something went wrong")
|
||||
|
||||
it "should return the callback with an error", ->
|
||||
@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
|
||||
|
||||
it "should log out the error", ->
|
||||
@logger.error.called.should.equal true
|
||||
|
||||
|
||||
describe "with a relative extracted file path", ->
|
||||
beforeEach (done) ->
|
||||
@zipfile.openReadStream = sinon.stub()
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@zipfile.emit "entry", {fileName: "../testfile.txt"}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should not write try to read the file entry", ->
|
||||
@zipfile.openReadStream.called.should.equal false
|
||||
|
||||
it "should log out a warning", ->
|
||||
@logger.warn.called.should.equal true
|
||||
|
||||
describe "with an unnormalized extracted file path", ->
|
||||
beforeEach (done) ->
|
||||
@zipfile.openReadStream = sinon.stub()
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@zipfile.emit "entry", {fileName: "foo/./testfile.txt"}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should not write try to read the file entry", ->
|
||||
@zipfile.openReadStream.called.should.equal false
|
||||
|
||||
it "should log out a warning", ->
|
||||
@logger.warn.called.should.equal true
|
||||
|
||||
describe "with a directory entry", ->
|
||||
beforeEach (done) ->
|
||||
@zipfile.openReadStream = sinon.stub()
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@zipfile.emit "entry", {fileName: "testdir/"}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should not write try to read the entry", ->
|
||||
@zipfile.openReadStream.called.should.equal false
|
||||
|
||||
it "should not log out a warning", ->
|
||||
@logger.warn.called.should.equal false
|
||||
|
||||
describe "with an error opening the file read stream", ->
|
||||
beforeEach (done) ->
|
||||
@zipfile.openReadStream = sinon.stub().callsArgWith(1, new Error("Something went wrong"))
|
||||
@writeStream = new events.EventEmitter
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@zipfile.emit "entry", {fileName: "testfile.txt"}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return the callback with an error", ->
|
||||
@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
|
||||
|
||||
it "should log out the error", ->
|
||||
@logger.error.called.should.equal true
|
||||
|
||||
it "should close the zipfile", ->
|
||||
@zipfile.close.called.should.equal true
|
||||
|
||||
describe "with an error in the file read stream", ->
|
||||
beforeEach (done) ->
|
||||
@readStream = new events.EventEmitter
|
||||
@readStream.pipe = sinon.stub()
|
||||
@zipfile.openReadStream = sinon.stub().callsArgWith(1, null, @readStream)
|
||||
@writeStream = new events.EventEmitter
|
||||
@fs.createWriteStream = sinon.stub().returns @writeStream
|
||||
@fse.ensureDir = sinon.stub().callsArg(1)
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@zipfile.emit "entry", {fileName: "testfile.txt"}
|
||||
@readStream.emit "error", new Error("Something went wrong")
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return the callback with an error", ->
|
||||
@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
|
||||
|
||||
it "should log out the error", ->
|
||||
@logger.error.called.should.equal true
|
||||
|
||||
it "should close the zipfile", ->
|
||||
@zipfile.close.called.should.equal true
|
||||
|
||||
describe "with an error in the file write stream", ->
|
||||
beforeEach (done) ->
|
||||
@readStream = new events.EventEmitter
|
||||
@readStream.pipe = sinon.stub()
|
||||
@readStream.unpipe = sinon.stub()
|
||||
@readStream.destroy = sinon.stub()
|
||||
@zipfile.openReadStream = sinon.stub().callsArgWith(1, null, @readStream)
|
||||
@writeStream = new events.EventEmitter
|
||||
@fs.createWriteStream = sinon.stub().returns @writeStream
|
||||
@fse.ensureDir = sinon.stub().callsArg(1)
|
||||
@ArchiveManager.extractZipArchive @source, @destination, (error) =>
|
||||
@callback(error)
|
||||
done()
|
||||
@zipfile.emit "entry", {fileName: "testfile.txt"}
|
||||
@writeStream.emit "error", new Error("Something went wrong")
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return the callback with an error", ->
|
||||
@callback.calledWithExactly(new Error("Something went wrong")).should.equal true
|
||||
|
||||
it "should log out the error", ->
|
||||
@logger.error.called.should.equal true
|
||||
|
||||
it "should unpipe from the readstream", ->
|
||||
@readStream.unpipe.called.should.equal true
|
||||
|
||||
it "should destroy the readstream", ->
|
||||
@readStream.destroy.called.should.equal true
|
||||
|
||||
it "should close the zipfile", ->
|
||||
@zipfile.close.called.should.equal true
|
||||
|
||||
describe "_isZipTooLarge", ->
|
||||
beforeEach ->
|
||||
@output = (totalSize)->" Length Date Time Name \n-------- ---- ---- ---- \n241 03-12-16 12:20 main.tex \n108801 03-12-16 12:20 ddd/x1J5kHh.jpg \n-------- ------- \n#{totalSize} 2 files\n"
|
||||
|
||||
it "should return false with small output", (done)->
|
||||
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
|
||||
isTooLarge.should.equal false
|
||||
done()
|
||||
@process.stdout.emit "data", @output("109042")
|
||||
@process.emit "close"
|
||||
@zipfile.emit "entry", {uncompressedSize: 109042}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return true with large bytes", (done)->
|
||||
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
|
||||
isTooLarge.should.equal true
|
||||
done()
|
||||
@process.stdout.emit "data", @output("1090000000000000042")
|
||||
@process.emit "close"
|
||||
@zipfile.emit "entry", {uncompressedSize: 1090000000000000042}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return error on no data", (done)->
|
||||
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
|
||||
expect(error).to.exist
|
||||
done()
|
||||
@process.stdout.emit "data", ""
|
||||
@process.emit "close"
|
||||
@zipfile.emit "entry", {}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return error if it didn't get a number", (done)->
|
||||
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
|
||||
expect(error).to.exist
|
||||
done()
|
||||
@process.stdout.emit "data", @output("total_size_string")
|
||||
@process.emit "close"
|
||||
@zipfile.emit "entry", {uncompressedSize:"random-error"}
|
||||
@zipfile.emit "end"
|
||||
|
||||
it "should return error if the is only a bit of data", (done)->
|
||||
it "should return error if there is no data", (done)->
|
||||
@ArchiveManager._isZipTooLarge @source, (error, isTooLarge) =>
|
||||
expect(error).to.exist
|
||||
done()
|
||||
@process.stdout.emit "data", " Length Date Time Name \n--------"
|
||||
@process.emit "close"
|
||||
@zipfile.emit "end"
|
||||
|
||||
describe "findTopLevelDirectory", ->
|
||||
beforeEach ->
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue