github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' into csh-ho-docker

Browse source
This commit is contained in:
Christopher Hoskin 2019-01-02 09:51:35 +00:00
commit aab5a1af6a
11 changed files with 303 additions and 174 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
services/filestore/README.md
View file

@ -5,6 +5,18 @@ An API for CRUD operations on binary files stored in S3
[![Build Status](https://travis-ci.org/sharelatex/filestore-sharelatex.png?branch=master)](https://travis-ci.org/sharelatex/filestore-sharelatex)
filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, presenting a RESTful HTTP interface to the CLSIs on port 3009 by default. Urls are mapped to node functions in https://github.com/sharelatex/filestore-sharelatex/blob/master/app.coffee . URLs are of the form:
* `/project/:project_id/file/:file_id`
* `/template/:template_id/v/:version/:format`
* `/project/:project_id/public/:public_file_id`
* `/project/:project_id/size`
* `/bucket/:bucket/key/*`
* `/heapdump`
* `/shutdown`
* `/status` - returns `filestore sharelatex up` or `server is being shut down` (HTTP 500)
* `/health_check`
License
-------

20
services/filestore/app/coffee/FSPersistorManager.coffee
View file

@ -13,10 +13,19 @@ module.exports =
sendFile: ( location, target, source, callback = (err)->) ->
filteredTarget = filterName target
logger.log location:location, target:filteredTarget, source:source, "sending file"
fs.rename source, "#{location}/#{filteredTarget}", (err) ->
if err!=null
done = _.once (err) ->
if err?
logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file"
callback err
callback(err)
# actually copy the file (instead of moving it) to maintain consistent behaviour
# between the different implementations
sourceStream = fs.createReadStream source
sourceStream.on 'error', done
targetStream = fs.createWriteStream "#{location}/#{filteredTarget}"
targetStream.on 'error', done
targetStream.on 'finish', () ->
done()
sourceStream.pipe targetStream
sendStream: ( location, target, sourceStream, callback = (err)->) ->
logger.log location:location, target:target, "sending file stream"
@ -26,7 +35,10 @@ module.exports =
if err?
logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback err
@sendFile location, target, fsPath, callback
@sendFile location, target, fsPath, (err) ->
# delete the temporary file created above and return the original error
LocalFileWriter.deleteFile fsPath, () ->
callback(err)
# opts may be {start: Number, end: Number}
getFileStream: (location, name, opts, _callback = (err, res)->) ->

2
services/filestore/app/coffee/FileController.coffee
View file

@ -29,10 +29,10 @@ module.exports = FileController =
logger.log start: range.start, end: range.end, "getting range of bytes from file"
FileHandler.getFile bucket, key, options, (err, fileStream)->
if err?
logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
if err instanceof Errors.NotFoundError
return res.send 404
else
logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
return res.send 500
else if req.query.cacheWarm
logger.log key:key, bucket:bucket, format:format, style:style, "request is only for cache warm so not sending stream"

22
services/filestore/app/coffee/FileHandler.coffee
View file

@ -6,6 +6,7 @@ FileConverter = require("./FileConverter")
KeyBuilder = require("./KeyBuilder")
async = require("async")
ImageOptimiser = require("./ImageOptimiser")
Errors = require('./Errors')
module.exports = FileHandler =
@ -32,7 +33,7 @@ module.exports = FileHandler =
_getStandardFile: (bucket, key, opts, callback)->
PersistorManager.getFileStream bucket, key, opts, (err, fileStream)->
if err?
if err? and !(err instanceof Errors.NotFoundError)
logger.err bucket:bucket, key:key, opts:FileHandler._scrubSecrets(opts), "error getting fileStream"
callback err, fileStream
@ -64,7 +65,24 @@ module.exports = FileHandler =
LocalFileWriter.deleteFile convertedFsPath, ->
LocalFileWriter.deleteFile originalFsPath, ->
return callback(err)
PersistorManager.getFileStream bucket, convertedKey, opts, callback
# Send back the converted file from the local copy to avoid problems
# with the file not being present in S3 yet. As described in the
# documentation below, we have already made a 'HEAD' request in
# checkIfFileExists so we only have "eventual consistency" if we try
# to stream it from S3 here. This was a cause of many 403 errors.
#
# "Amazon S3 provides read-after-write consistency for PUTS of new
# objects in your S3 bucket in all regions with one caveat. The
# caveat is that if you make a HEAD or GET request to the key name
# (to find if the object exists) before creating the object, Amazon
# S3 provides eventual consistency for read-after-write.""
# https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
LocalFileWriter.getStream convertedFsPath, (err, readStream) ->
return callback(err) if err?
readStream.on 'end', () ->
logger.log {convertedFsPath: convertedFsPath}, "deleting temporary file"
LocalFileWriter.deleteFile convertedFsPath, ->
callback(null, readStream)
_convertFile: (bucket, originalKey, opts, callback)->
@_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)->

17
services/filestore/app/coffee/LocalFileWriter.coffee
View file

@ -5,6 +5,7 @@ _ = require("underscore")
logger = require("logger-sharelatex")
metrics = require("metrics-sharelatex")
Settings = require("settings-sharelatex")
Errors = require "./Errors"
module.exports =
@ -26,6 +27,22 @@ module.exports =
callback err
stream.pipe writeStream
getStream: (fsPath, _callback = (err, res)->) ->
callback = _.once _callback
timer = new metrics.Timer("readingFile")
logger.log fsPath:fsPath, "reading file locally"
readStream = fs.createReadStream(fsPath)
readStream.on "end", ->
timer.done()
logger.log fsPath:fsPath, "finished reading file locally"
readStream.on "error", (err)->
logger.err err:err, fsPath:fsPath, "problem reading file locally, with read stream"
if err.code == 'ENOENT'
callback new Errors.NotFoundError(err.message), null
else
callback err
callback null, readStream
deleteFile: (fsPath, callback)->
if !fsPath? or fsPath == ""
return callback()

14
services/filestore/app/coffee/S3PersistorManager.coffee
View file

@ -42,9 +42,8 @@ module.exports =
if res.statusCode != 200
logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file"
return callback("non 200 response from s3 on put file")
LocalFileWriter.deleteFile fsPath, (err)->
logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
callback(err)
logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
callback(err)
putEventEmiter.on "error", (err)->
logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file"
callback err
@ -57,7 +56,10 @@ module.exports =
if err?
logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback(err)
@sendFile bucketName, key, fsPath, callback
@sendFile bucketName, key, fsPath, (err) ->
# delete the temporary file created above and return the original error
LocalFileWriter.deleteFile fsPath, () ->
callback(err)
# opts may be {start: Number, end: Number}
getFileStream: (bucketName, key, opts, callback = (err, res)->)->
@ -74,7 +76,9 @@ module.exports =
s3Stream = s3Client.get(key, headers)
s3Stream.end()
s3Stream.on 'response', (res) ->
if res.statusCode == 404
if res.statusCode in [403, 404]
# S3 returns a 403 instead of a 404 when the user doesn't have
# permission to list the bucket contents.
logger.log bucketName:bucketName, key:key, "file not found in s3"
return callback new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null
else if res.statusCode not in [200, 206]

333
services/filestore/npm-shrinkwrap.json generated
View file

@ -3,24 +3,36 @@
"version": "0.1.4",
"dependencies": {
"@google-cloud/common": {
"version": "0.23.0",
"from": "@google-cloud/common@>=0.23.0 <0.24.0",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.23.0.tgz"
"version": "0.27.0",
"from": "@google-cloud/common@>=0.27.0 <0.28.0",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz"
},
"@google-cloud/debug-agent": {
"version": "3.0.0",
"version": "3.0.1",
"from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.0.tgz",
"resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.1.tgz",
"dependencies": {
"coffeescript": {
"version": "2.3.2",
"from": "coffeescript@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz"
}
}
},
"@google-cloud/profiler": {
"version": "0.2.3",
"from": "@google-cloud/profiler@>=0.2.3 <0.3.0",
"resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz",
"dependencies": {
"@google-cloud/common": {
"version": "0.26.2",
"from": "@google-cloud/common@>=0.26.0 <0.27.0",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz"
},
"lodash": {
"version": "4.17.11",
"from": "lodash@>=4.12.0 <5.0.0",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz"
"through2": {
"version": "3.0.0",
"from": "through2@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz"
}
}
},
@ -35,41 +47,14 @@
"resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz"
},
"@google-cloud/trace-agent": {
"version": "3.3.1",
"version": "3.5.0",
"from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0",
"resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.3.1.tgz",
"resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.0.tgz",
"dependencies": {
"@google-cloud/common": {
"version": "0.26.2",
"from": "@google-cloud/common@>=0.26.0 <0.27.0",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz"
},
"gcp-metadata": {
"version": "0.9.0",
"from": "gcp-metadata@>=0.9.0 <0.10.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.0.tgz"
},
"google-auth-library": {
"version": "2.0.1",
"from": "google-auth-library@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.1.tgz",
"dependencies": {
"gcp-metadata": {
"version": "0.7.0",
"from": "gcp-metadata@^0.7.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
}
}
},
"lru-cache": {
"version": "4.1.4",
"from": "lru-cache@>=4.1.3 <5.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.4.tgz"
},
"through2": {
"version": "3.0.0",
"from": "through2@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz"
"version": "0.28.0",
"from": "@google-cloud/common@>=0.28.0 <0.29.0",
"resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.28.0.tgz"
},
"uuid": {
"version": "3.3.2",
@ -78,6 +63,61 @@
}
}
},
"@protobufjs/aspromise": {
"version": "1.1.2",
"from": "@protobufjs/aspromise@>=1.1.2 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz"
},
"@protobufjs/base64": {
"version": "1.1.2",
"from": "@protobufjs/base64@>=1.1.2 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz"
},
"@protobufjs/codegen": {
"version": "2.0.4",
"from": "@protobufjs/codegen@>=2.0.4 <3.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz"
},
"@protobufjs/eventemitter": {
"version": "1.1.0",
"from": "@protobufjs/eventemitter@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz"
},
"@protobufjs/fetch": {
"version": "1.1.0",
"from": "@protobufjs/fetch@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz"
},
"@protobufjs/float": {
"version": "1.0.2",
"from": "@protobufjs/float@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz"
},
"@protobufjs/inquire": {
"version": "1.1.0",
"from": "@protobufjs/inquire@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz"
},
"@protobufjs/path": {
"version": "1.1.2",
"from": "@protobufjs/path@>=1.1.2 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz"
},
"@protobufjs/pool": {
"version": "1.1.0",
"from": "@protobufjs/pool@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz"
},
"@protobufjs/utf8": {
"version": "1.1.0",
"from": "@protobufjs/utf8@>=1.1.0 <2.0.0",
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz"
},
"@sindresorhus/is": {
"version": "0.13.0",
"from": "@sindresorhus/is@>=0.13.0 <0.14.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz"
},
"@sinonjs/commons": {
"version": "1.3.0",
"from": "@sinonjs/commons@>=1.2.0 <2.0.0",
@ -105,6 +145,11 @@
"from": "@types/caseless@*",
"resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz"
},
"@types/console-log-level": {
"version": "1.4.0",
"from": "@types/console-log-level@>=1.4.0 <2.0.0",
"resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz"
},
"@types/duplexify": {
"version": "3.6.0",
"from": "@types/duplexify@>=3.5.0 <4.0.0",
@ -115,16 +160,26 @@
"from": "@types/form-data@*",
"resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz"
},
"@types/long": {
"version": "4.0.0",
"from": "@types/long@>=4.0.0 <5.0.0",
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz"
},
"@types/node": {
"version": "10.12.10",
"version": "10.12.18",
"from": "@types/node@*",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz"
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz"
},
"@types/request": {
"version": "2.48.1",
"from": "@types/request@>=2.47.0 <3.0.0",
"resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz"
},
"@types/semver": {
"version": "5.5.0",
"from": "@types/semver@>=5.5.0 <6.0.0",
"resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz"
},
"@types/tough-cookie": {
"version": "2.3.4",
"from": "@types/tough-cookie@*",
@ -254,6 +309,11 @@
"from": "bignumber.js@>=7.0.0 <8.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz"
},
"bindings": {
"version": "1.3.1",
"from": "bindings@>=1.2.1 <2.0.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz"
},
"bintrees": {
"version": "1.0.1",
"from": "bintrees@1.0.1",
@ -431,10 +491,10 @@
"from": "deep-eql@>=3.0.1 <4.0.0",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz"
},
"define-properties": {
"version": "1.1.3",
"from": "define-properties@>=1.1.2 <2.0.0",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz"
"delay": {
"version": "4.1.0",
"from": "delay@>=4.0.1 <5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz"
},
"delayed-stream": {
"version": "0.0.5",
@ -497,16 +557,6 @@
"from": "ent@>=2.2.0 <3.0.0",
"resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz"
},
"es-abstract": {
"version": "1.12.0",
"from": "es-abstract@>=1.5.1 <2.0.0",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz"
},
"es-to-primitive": {
"version": "1.2.0",
"from": "es-to-primitive@>=1.1.1 <2.0.0",
"resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz"
},
"es6-promise": {
"version": "4.2.5",
"from": "es6-promise@>=4.0.3 <5.0.0",
@ -609,9 +659,9 @@
}
},
"follow-redirects": {
"version": "1.5.10",
"version": "1.6.0",
"from": "follow-redirects@>=1.3.0 <2.0.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.0.tgz",
"dependencies": {
"debug": {
"version": "3.1.0",
@ -657,15 +707,15 @@
"from": "fs.realpath@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
},
"function-bind": {
"version": "1.1.1",
"from": "function-bind@>=1.1.1 <2.0.0",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz"
"gaxios": {
"version": "1.0.4",
"from": "gaxios@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.0.4.tgz"
},
"gcp-metadata": {
"version": "0.7.0",
"from": "gcp-metadata@>=0.7.0 <0.8.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
"version": "0.9.3",
"from": "gcp-metadata@>=0.9.0 <0.10.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz"
},
"get-func-name": {
"version": "2.0.0",
@ -689,33 +739,26 @@
"optional": true
},
"google-auth-library": {
"version": "1.6.1",
"from": "google-auth-library@>=1.6.0 <2.0.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.6.1.tgz",
"version": "2.0.2",
"from": "google-auth-library@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz",
"dependencies": {
"gcp-metadata": {
"version": "0.6.3",
"from": "gcp-metadata@>=0.6.3 <0.7.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz"
"version": "0.7.0",
"from": "gcp-metadata@>=0.7.0 <0.8.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
},
"lru-cache": {
"version": "4.1.4",
"from": "lru-cache@>=4.1.3 <5.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.4.tgz"
"version": "5.1.1",
"from": "lru-cache@>=5.0.0 <6.0.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz"
}
}
},
"google-p12-pem": {
"version": "1.0.2",
"version": "1.0.3",
"from": "google-p12-pem@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.2.tgz",
"dependencies": {
"pify": {
"version": "3.0.0",
"from": "pify@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz"
}
}
"resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz"
},
"graceful-fs": {
"version": "4.1.15",
@ -920,11 +963,6 @@
}
}
},
"has": {
"version": "1.0.3",
"from": "has@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz"
},
"has-ansi": {
"version": "0.1.0",
"from": "has-ansi@>=0.1.0 <0.2.0",
@ -935,11 +973,6 @@
"from": "has-flag@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz"
},
"has-symbols": {
"version": "1.0.0",
"from": "has-symbols@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz"
},
"hawk": {
"version": "0.10.2",
"from": "hawk@>=0.10.2 <0.11.0",
@ -956,9 +989,9 @@
"resolved": "https://registry.npmjs.org/heapdump/-/heapdump-0.3.12.tgz"
},
"hex2dec": {
"version": "1.1.0",
"version": "1.1.1",
"from": "hex2dec@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.0.tgz"
"resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.1.tgz"
},
"hoek": {
"version": "0.7.6",
@ -1018,35 +1051,15 @@
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz"
},
"is": {
"version": "3.2.1",
"from": "is@>=3.2.1 <4.0.0",
"resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz"
"version": "3.3.0",
"from": "is@>=3.2.0 <4.0.0",
"resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz"
},
"is-buffer": {
"version": "1.1.6",
"from": "is-buffer@>=1.1.5 <2.0.0",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz"
},
"is-callable": {
"version": "1.1.4",
"from": "is-callable@>=1.1.3 <2.0.0",
"resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz"
},
"is-date-object": {
"version": "1.0.1",
"from": "is-date-object@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz"
},
"is-regex": {
"version": "1.0.4",
"from": "is-regex@>=1.0.4 <2.0.0",
"resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz"
},
"is-symbol": {
"version": "1.0.2",
"from": "is-symbol@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz"
},
"isarray": {
"version": "1.0.0",
"from": "isarray@>=1.0.0 <2.0.0",
@ -1136,10 +1149,10 @@
"from": "lodash.get@>=4.4.2 <5.0.0",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz"
},
"lodash.isstring": {
"version": "4.0.1",
"from": "lodash.isstring@>=4.0.1 <5.0.0",
"resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz"
"lodash.pickby": {
"version": "4.6.0",
"from": "lodash.pickby@>=4.6.0 <5.0.0",
"resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz"
},
"logger-sharelatex": {
"version": "1.5.8",
@ -1158,6 +1171,11 @@
"from": "lolex@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/lolex/-/lolex-3.0.0.tgz"
},
"long": {
"version": "4.0.0",
"from": "long@>=4.0.0 <5.0.0",
"resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz"
},
"lru-cache": {
"version": "2.7.3",
"from": "lru-cache@>=2.0.0 <3.0.0",
@ -1194,9 +1212,9 @@
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
},
"metrics-sharelatex": {
"version": "2.0.7",
"from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7",
"resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3c7dd668d1153c13acee9cceb3a8ce24495b7c86",
"version": "2.0.12",
"from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
"resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3ac1621ef049e2f2d88a83b3a41011333d609662",
"dependencies": {
"coffee-script": {
"version": "1.6.0",
@ -1407,16 +1425,6 @@
"from": "oauth-sign@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.2.0.tgz"
},
"object-keys": {
"version": "1.0.12",
"from": "object-keys@>=1.0.12 <2.0.0",
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz"
},
"object.getownpropertydescriptors": {
"version": "2.0.3",
"from": "object.getownpropertydescriptors@>=2.0.3 <3.0.0",
"resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz"
},
"on-finished": {
"version": "2.3.0",
"from": "on-finished@>=2.3.0 <2.4.0",
@ -1428,15 +1436,25 @@
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz"
},
"p-limit": {
"version": "2.0.0",
"version": "2.1.0",
"from": "p-limit@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz"
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz"
},
"p-try": {
"version": "2.0.0",
"from": "p-try@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz"
},
"parse-duration": {
"version": "0.1.1",
"from": "parse-duration@>=0.1.1 <0.2.0",
"resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz"
},
"parse-ms": {
"version": "2.0.0",
"from": "parse-ms@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.0.0.tgz"
},
"parseurl": {
"version": "1.3.2",
"from": "parseurl@>=1.3.2 <1.4.0",
@ -1472,26 +1490,31 @@
"from": "pngcrush@0.0.3",
"resolved": "https://registry.npmjs.org/pngcrush/-/pngcrush-0.0.3.tgz"
},
"pretty-ms": {
"version": "4.0.0",
"from": "pretty-ms@>=4.0.0 <5.0.0",
"resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz"
},
"process-nextick-args": {
"version": "2.0.0",
"from": "process-nextick-args@>=2.0.0 <2.1.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz"
},
"prom-client": {
"version": "11.2.0",
"version": "11.2.1",
"from": "prom-client@>=11.1.3 <12.0.0",
"resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.0.tgz"
"resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.1.tgz"
},
"protobufjs": {
"version": "6.8.8",
"from": "protobufjs@>=6.8.6 <6.9.0",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz"
},
"proxy-addr": {
"version": "2.0.4",
"from": "proxy-addr@>=2.0.4 <2.1.0",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz"
},
"pseudomap": {
"version": "1.0.2",
"from": "pseudomap@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz"
},
"punycode": {
"version": "1.3.2",
"from": "punycode@1.3.2",
@ -1581,9 +1604,9 @@
"resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz"
},
"resolve": {
"version": "1.8.1",
"version": "1.9.0",
"from": "resolve@>=1.5.0 <2.0.0",
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz"
"resolved": "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz"
},
"response": {
"version": "0.14.0",
@ -1769,15 +1792,20 @@
"from": "supports-color@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz"
},
"symbol-observable": {
"version": "1.2.0",
"from": "symbol-observable@>=1.2.0 <2.0.0",
"resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz"
},
"tdigest": {
"version": "0.1.1",
"from": "tdigest@>=0.1.1 <0.2.0",
"resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz"
},
"teeny-request": {
"version": "3.9.1",
"from": "teeny-request@>=3.6.0 <4.0.0",
"resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.9.1.tgz",
"version": "3.11.3",
"from": "teeny-request@>=3.11.1 <4.0.0",
"resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz",
"dependencies": {
"uuid": {
"version": "3.3.2",
@ -1846,11 +1874,6 @@
"from": "util-deprecate@>=1.0.1 <1.1.0",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
},
"util.promisify": {
"version": "1.0.0",
"from": "util.promisify@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz"
},
"utils-merge": {
"version": "1.0.1",
"from": "utils-merge@1.0.1",

2
services/filestore/package.json
View file

@ -31,7 +31,7 @@
"heapdump": "^0.3.2",
"knox": "~0.9.1",
"logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.8",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7",
"metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
"mocha": "5.2.0",
"node-transloadit": "0.0.4",
"node-uuid": "~1.4.1",

30
services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee
View file

@ -25,6 +25,7 @@ describe "FSPersistorManagerTests", ->
@Rimraf = sinon.stub()
@LocalFileWriter =
writeStream: sinon.stub()
deleteFile: sinon.stub()
@requires =
"./LocalFileWriter":@LocalFileWriter
"fs":@Fs
@ -41,10 +42,32 @@ describe "FSPersistorManagerTests", ->
@FSPersistorManager = SandboxedModule.require modulePath, requires: @requires
describe "sendFile", ->
it "should put the file", (done) ->
@Fs.rename.callsArgWith(2,@error)
beforeEach ->
@Fs.createReadStream = sinon.stub().returns({
on: ->
pipe: ->
})
it "should copy the file", (done) ->
@Fs.createWriteStream =sinon.stub().returns({
on: (event, handler) ->
process.nextTick(handler) if event is 'finish'
})
@FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
@Fs.rename.calledWith( @name2, "#{@location}/#{@name1Filtered}" ).should.equal true
@Fs.createReadStream.calledWith(@name2).should.equal true
@Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
done()
it "should return an error if the file cannot be stored", (done) ->
@Fs.createWriteStream =sinon.stub().returns({
on: (event, handler) =>
if event is 'error'
process.nextTick () =>
handler(@error)
})
@FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
@Fs.createReadStream.calledWith(@name2).should.equal true
@Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
err.should.equal @error
done()
@ -52,6 +75,7 @@ describe "FSPersistorManagerTests", ->
beforeEach ->
@FSPersistorManager.sendFile = sinon.stub().callsArgWith(3)
@LocalFileWriter.writeStream.callsArgWith(2, null, @name1)
@LocalFileWriter.deleteFile.callsArg(1)
@SourceStream =
on:->

10
services/filestore/test/unit/coffee/FileHandlerTests.coffee
View file

@ -23,6 +23,7 @@ describe "FileHandler", ->
directorySize: sinon.stub()
@LocalFileWriter =
writeStream: sinon.stub()
getStream: sinon.stub()
deleteFile: sinon.stub()
@FileConverter =
convert: sinon.stub()
@ -152,17 +153,20 @@ describe "FileHandler", ->
it "should _convertFile ", (done)->
@stubbedStream = {"something":"here"}
@localStream = {
on: ->
}
@PersistorManager.sendFile = sinon.stub().callsArgWith(3)
@PersistorManager.getFileStream = sinon.stub().callsArgWith(3, null, @stubbedStream)
@LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, @localStream)
@convetedKey = @key+"converted"
@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, (err, fsStream)=>
@handler._convertFile.called.should.equal true
@PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
@PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
fsStream.should.equal @stubbedStream
@LocalFileWriter.getStream.calledWith(@stubbedPath).should.equal true
fsStream.should.equal @localStream
done()
describe "_convertFile", ->

15
services/filestore/test/unit/coffee/LocalFileWriterTests.coffee
View file

@ -15,8 +15,11 @@ describe "LocalFileWriter", ->
on: (type, cb)->
if type == "finish"
cb()
@readStream =
on: ->
@fs =
createWriteStream : sinon.stub().returns(@writeStream)
createReadStream: sinon.stub().returns(@readStream)
unlink: sinon.stub()
@settings =
path:
@ -56,6 +59,18 @@ describe "LocalFileWriter", ->
fsPath.should.equal @stubbedFsPath
done()
describe "getStream", ->
it "should read the stream from the file ", (done)->
@writer.getStream @stubbedFsPath, (err, stream)=>
@fs.createReadStream.calledWith(@stubbedFsPath).should.equal true
done()
it "should send the stream in the callback", (done)->
@writer.getStream @stubbedFsPath, (err, readStream)=>
readStream.should.equal @readStream
done()
describe "delete file", ->
it "should unlink the file", (done)->