diff --git a/services/filestore/README.md b/services/filestore/README.md
index 420214b56f..cd126ab053 100644
--- a/services/filestore/README.md
+++ b/services/filestore/README.md
@@ -5,6 +5,18 @@ An API for CRUD operations on binary files stored in S3
[](https://travis-ci.org/sharelatex/filestore-sharelatex)
+filestore acts as a proxy between the CLSIs and (currently) Amazon S3 storage, presenting a RESTful HTTP interface to the CLSIs on port 3009 by default. Urls are mapped to node functions in https://github.com/sharelatex/filestore-sharelatex/blob/master/app.coffee . URLs are of the form:
+
+* `/project/:project_id/file/:file_id`
+* `/template/:template_id/v/:version/:format`
+* `/project/:project_id/public/:public_file_id`
+* `/project/:project_id/size`
+* `/bucket/:bucket/key/*`
+* `/heapdump`
+* `/shutdown`
+* `/status` - returns `filestore sharelatex up` or `server is being shut down` (HTTP 500)
+* `/health_check`
+
License
-------
diff --git a/services/filestore/app.coffee b/services/filestore/app.coffee
index b172e3ba88..c1484142b1 100644
--- a/services/filestore/app.coffee
+++ b/services/filestore/app.coffee
@@ -1,3 +1,5 @@
+Metrics = require "metrics-sharelatex"
+Metrics.initialize("filestore")
express = require('express')
bodyParser = require "body-parser"
logger = require('logger-sharelatex')
@@ -15,16 +17,12 @@ app = express()
if settings.sentry?.dsn?
logger.initializeErrorReporting(settings.sentry.dsn)
-Metrics = require "metrics-sharelatex"
-Metrics.initialize("filestore")
Metrics.open_sockets.monitor(logger)
Metrics.event_loop?.monitor(logger)
Metrics.memory.monitor(logger)
app.use Metrics.http.monitor(logger)
-Metrics.inc "startup"
-
app.use (req, res, next)->
Metrics.inc "http-request"
next()
diff --git a/services/filestore/app/coffee/FSPersistorManager.coffee b/services/filestore/app/coffee/FSPersistorManager.coffee
index 0868216a15..733202e4cd 100644
--- a/services/filestore/app/coffee/FSPersistorManager.coffee
+++ b/services/filestore/app/coffee/FSPersistorManager.coffee
@@ -13,10 +13,19 @@ module.exports =
sendFile: ( location, target, source, callback = (err)->) ->
filteredTarget = filterName target
logger.log location:location, target:filteredTarget, source:source, "sending file"
- fs.rename source, "#{location}/#{filteredTarget}", (err) ->
- if err!=null
+ done = _.once (err) ->
+ if err?
logger.err err:err, location:location, target:filteredTarget, source:source, "Error on put of file"
- callback err
+ callback(err)
+ # actually copy the file (instead of moving it) to maintain consistent behaviour
+ # between the different implementations
+ sourceStream = fs.createReadStream source
+ sourceStream.on 'error', done
+ targetStream = fs.createWriteStream "#{location}/#{filteredTarget}"
+ targetStream.on 'error', done
+ targetStream.on 'finish', () ->
+ done()
+ sourceStream.pipe targetStream
sendStream: ( location, target, sourceStream, callback = (err)->) ->
logger.log location:location, target:target, "sending file stream"
@@ -26,7 +35,10 @@ module.exports =
if err?
logger.err location:location, target:target, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback err
- @sendFile location, target, fsPath, callback
+ @sendFile location, target, fsPath, (err) ->
+ # delete the temporary file created above and return the original error
+ LocalFileWriter.deleteFile fsPath, () ->
+ callback(err)
# opts may be {start: Number, end: Number}
getFileStream: (location, name, opts, _callback = (err, res)->) ->
diff --git a/services/filestore/app/coffee/FileController.coffee b/services/filestore/app/coffee/FileController.coffee
index 27ac078379..24fd5229de 100644
--- a/services/filestore/app/coffee/FileController.coffee
+++ b/services/filestore/app/coffee/FileController.coffee
@@ -29,10 +29,10 @@ module.exports = FileController =
logger.log start: range.start, end: range.end, "getting range of bytes from file"
FileHandler.getFile bucket, key, options, (err, fileStream)->
if err?
- logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
if err instanceof Errors.NotFoundError
return res.send 404
else
+ logger.err err:err, key:key, bucket:bucket, format:format, style:style, "problem getting file"
return res.send 500
else if req.query.cacheWarm
logger.log key:key, bucket:bucket, format:format, style:style, "request is only for cache warm so not sending stream"
diff --git a/services/filestore/app/coffee/FileHandler.coffee b/services/filestore/app/coffee/FileHandler.coffee
index 87773e6807..6189c8a906 100644
--- a/services/filestore/app/coffee/FileHandler.coffee
+++ b/services/filestore/app/coffee/FileHandler.coffee
@@ -6,6 +6,7 @@ FileConverter = require("./FileConverter")
KeyBuilder = require("./KeyBuilder")
async = require("async")
ImageOptimiser = require("./ImageOptimiser")
+Errors = require('./Errors')
module.exports = FileHandler =
@@ -32,7 +33,7 @@ module.exports = FileHandler =
_getStandardFile: (bucket, key, opts, callback)->
PersistorManager.getFileStream bucket, key, opts, (err, fileStream)->
- if err?
+ if err? and !(err instanceof Errors.NotFoundError)
logger.err bucket:bucket, key:key, opts:FileHandler._scrubSecrets(opts), "error getting fileStream"
callback err, fileStream
@@ -64,7 +65,24 @@ module.exports = FileHandler =
LocalFileWriter.deleteFile convertedFsPath, ->
LocalFileWriter.deleteFile originalFsPath, ->
return callback(err)
- PersistorManager.getFileStream bucket, convertedKey, opts, callback
+ # Send back the converted file from the local copy to avoid problems
+ # with the file not being present in S3 yet. As described in the
+ # documentation below, we have already made a 'HEAD' request in
+ # checkIfFileExists so we only have "eventual consistency" if we try
+ # to stream it from S3 here. This was a cause of many 403 errors.
+ #
+ # "Amazon S3 provides read-after-write consistency for PUTS of new
+ # objects in your S3 bucket in all regions with one caveat. The
+ # caveat is that if you make a HEAD or GET request to the key name
+ # (to find if the object exists) before creating the object, Amazon
+ # S3 provides eventual consistency for read-after-write.""
+ # https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
+ LocalFileWriter.getStream convertedFsPath, (err, readStream) ->
+ return callback(err) if err?
+ readStream.on 'end', () ->
+ logger.log {convertedFsPath: convertedFsPath}, "deleting temporary file"
+ LocalFileWriter.deleteFile convertedFsPath, ->
+ callback(null, readStream)
_convertFile: (bucket, originalKey, opts, callback)->
@_writeS3FileToDisk bucket, originalKey, opts, (err, originalFsPath)->
diff --git a/services/filestore/app/coffee/LocalFileWriter.coffee b/services/filestore/app/coffee/LocalFileWriter.coffee
index 47b2b91e77..72422b7696 100644
--- a/services/filestore/app/coffee/LocalFileWriter.coffee
+++ b/services/filestore/app/coffee/LocalFileWriter.coffee
@@ -5,6 +5,7 @@ _ = require("underscore")
logger = require("logger-sharelatex")
metrics = require("metrics-sharelatex")
Settings = require("settings-sharelatex")
+Errors = require "./Errors"
module.exports =
@@ -26,6 +27,22 @@ module.exports =
callback err
stream.pipe writeStream
+ getStream: (fsPath, _callback = (err, res)->) ->
+ callback = _.once _callback
+ timer = new metrics.Timer("readingFile")
+ logger.log fsPath:fsPath, "reading file locally"
+ readStream = fs.createReadStream(fsPath)
+ readStream.on "end", ->
+ timer.done()
+ logger.log fsPath:fsPath, "finished reading file locally"
+ readStream.on "error", (err)->
+ logger.err err:err, fsPath:fsPath, "problem reading file locally, with read stream"
+ if err.code == 'ENOENT'
+ callback new Errors.NotFoundError(err.message), null
+ else
+ callback err
+ callback null, readStream
+
deleteFile: (fsPath, callback)->
if !fsPath? or fsPath == ""
return callback()
diff --git a/services/filestore/app/coffee/S3PersistorManager.coffee b/services/filestore/app/coffee/S3PersistorManager.coffee
index e3761d87e9..207fd1fe18 100644
--- a/services/filestore/app/coffee/S3PersistorManager.coffee
+++ b/services/filestore/app/coffee/S3PersistorManager.coffee
@@ -42,9 +42,8 @@ module.exports =
if res.statusCode != 200
logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file"
return callback("non 200 response from s3 on put file")
- LocalFileWriter.deleteFile fsPath, (err)->
- logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
- callback(err)
+ logger.log res:res, bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
+ callback(err)
putEventEmiter.on "error", (err)->
logger.err err:err, bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file"
callback err
@@ -57,7 +56,10 @@ module.exports =
if err?
logger.err bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
return callback(err)
- @sendFile bucketName, key, fsPath, callback
+ @sendFile bucketName, key, fsPath, (err) ->
+ # delete the temporary file created above and return the original error
+ LocalFileWriter.deleteFile fsPath, () ->
+ callback(err)
# opts may be {start: Number, end: Number}
getFileStream: (bucketName, key, opts, callback = (err, res)->)->
@@ -74,7 +76,9 @@ module.exports =
s3Stream = s3Client.get(key, headers)
s3Stream.end()
s3Stream.on 'response', (res) ->
- if res.statusCode == 404
+ if res.statusCode in [403, 404]
+ # S3 returns a 403 instead of a 404 when the user doesn't have
+ # permission to list the bucket contents.
logger.log bucketName:bucketName, key:key, "file not found in s3"
return callback new Errors.NotFoundError("File not found in S3: #{bucketName}:#{key}"), null
else if res.statusCode not in [200, 206]
diff --git a/services/filestore/npm-shrinkwrap.json b/services/filestore/npm-shrinkwrap.json
index 14a32672a7..43599f526b 100644
--- a/services/filestore/npm-shrinkwrap.json
+++ b/services/filestore/npm-shrinkwrap.json
@@ -3,24 +3,36 @@
"version": "0.1.4",
"dependencies": {
"@google-cloud/common": {
- "version": "0.23.0",
- "from": "@google-cloud/common@>=0.23.0 <0.24.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.23.0.tgz"
+ "version": "0.27.0",
+ "from": "@google-cloud/common@>=0.27.0 <0.28.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.27.0.tgz"
},
"@google-cloud/debug-agent": {
- "version": "3.0.0",
+ "version": "3.0.1",
"from": "@google-cloud/debug-agent@>=3.0.0 <4.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.0.tgz",
+ "resolved": "https://registry.npmjs.org/@google-cloud/debug-agent/-/debug-agent-3.0.1.tgz",
"dependencies": {
"coffeescript": {
"version": "2.3.2",
"from": "coffeescript@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/coffeescript/-/coffeescript-2.3.2.tgz"
+ }
+ }
+ },
+ "@google-cloud/profiler": {
+ "version": "0.2.3",
+ "from": "@google-cloud/profiler@>=0.2.3 <0.3.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/profiler/-/profiler-0.2.3.tgz",
+ "dependencies": {
+ "@google-cloud/common": {
+ "version": "0.26.2",
+ "from": "@google-cloud/common@>=0.26.0 <0.27.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz"
},
- "lodash": {
- "version": "4.17.11",
- "from": "lodash@>=4.12.0 <5.0.0",
- "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz"
+ "through2": {
+ "version": "3.0.0",
+ "from": "through2@>=3.0.0 <4.0.0",
+ "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz"
}
}
},
@@ -35,41 +47,14 @@
"resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-0.3.1.tgz"
},
"@google-cloud/trace-agent": {
- "version": "3.3.1",
+ "version": "3.5.0",
"from": "@google-cloud/trace-agent@>=3.2.0 <4.0.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.3.1.tgz",
+ "resolved": "https://registry.npmjs.org/@google-cloud/trace-agent/-/trace-agent-3.5.0.tgz",
"dependencies": {
"@google-cloud/common": {
- "version": "0.26.2",
- "from": "@google-cloud/common@>=0.26.0 <0.27.0",
- "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.26.2.tgz"
- },
- "gcp-metadata": {
- "version": "0.9.0",
- "from": "gcp-metadata@>=0.9.0 <0.10.0",
- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.0.tgz"
- },
- "google-auth-library": {
- "version": "2.0.1",
- "from": "google-auth-library@>=2.0.0 <3.0.0",
- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.1.tgz",
- "dependencies": {
- "gcp-metadata": {
- "version": "0.7.0",
- "from": "gcp-metadata@^0.7.0",
- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
- }
- }
- },
- "lru-cache": {
- "version": "4.1.4",
- "from": "lru-cache@>=4.1.3 <5.0.0",
- "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.4.tgz"
- },
- "through2": {
- "version": "3.0.0",
- "from": "through2@>=3.0.0 <4.0.0",
- "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.0.tgz"
+ "version": "0.28.0",
+ "from": "@google-cloud/common@>=0.28.0 <0.29.0",
+ "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-0.28.0.tgz"
},
"uuid": {
"version": "3.3.2",
@@ -78,6 +63,61 @@
}
}
},
+ "@protobufjs/aspromise": {
+ "version": "1.1.2",
+ "from": "@protobufjs/aspromise@>=1.1.2 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz"
+ },
+ "@protobufjs/base64": {
+ "version": "1.1.2",
+ "from": "@protobufjs/base64@>=1.1.2 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz"
+ },
+ "@protobufjs/codegen": {
+ "version": "2.0.4",
+ "from": "@protobufjs/codegen@>=2.0.4 <3.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz"
+ },
+ "@protobufjs/eventemitter": {
+ "version": "1.1.0",
+ "from": "@protobufjs/eventemitter@>=1.1.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz"
+ },
+ "@protobufjs/fetch": {
+ "version": "1.1.0",
+ "from": "@protobufjs/fetch@>=1.1.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz"
+ },
+ "@protobufjs/float": {
+ "version": "1.0.2",
+ "from": "@protobufjs/float@>=1.0.2 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz"
+ },
+ "@protobufjs/inquire": {
+ "version": "1.1.0",
+ "from": "@protobufjs/inquire@>=1.1.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz"
+ },
+ "@protobufjs/path": {
+ "version": "1.1.2",
+ "from": "@protobufjs/path@>=1.1.2 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz"
+ },
+ "@protobufjs/pool": {
+ "version": "1.1.0",
+ "from": "@protobufjs/pool@>=1.1.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz"
+ },
+ "@protobufjs/utf8": {
+ "version": "1.1.0",
+ "from": "@protobufjs/utf8@>=1.1.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz"
+ },
+ "@sindresorhus/is": {
+ "version": "0.13.0",
+ "from": "@sindresorhus/is@>=0.13.0 <0.14.0",
+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.13.0.tgz"
+ },
"@sinonjs/commons": {
"version": "1.3.0",
"from": "@sinonjs/commons@>=1.2.0 <2.0.0",
@@ -105,6 +145,11 @@
"from": "@types/caseless@*",
"resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.1.tgz"
},
+ "@types/console-log-level": {
+ "version": "1.4.0",
+ "from": "@types/console-log-level@>=1.4.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/@types/console-log-level/-/console-log-level-1.4.0.tgz"
+ },
"@types/duplexify": {
"version": "3.6.0",
"from": "@types/duplexify@>=3.5.0 <4.0.0",
@@ -115,16 +160,26 @@
"from": "@types/form-data@*",
"resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.2.1.tgz"
},
+ "@types/long": {
+ "version": "4.0.0",
+ "from": "@types/long@>=4.0.0 <5.0.0",
+ "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz"
+ },
"@types/node": {
- "version": "10.12.10",
+ "version": "10.12.18",
"from": "@types/node@*",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz"
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.18.tgz"
},
"@types/request": {
"version": "2.48.1",
"from": "@types/request@>=2.47.0 <3.0.0",
"resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.1.tgz"
},
+ "@types/semver": {
+ "version": "5.5.0",
+ "from": "@types/semver@>=5.5.0 <6.0.0",
+ "resolved": "https://registry.npmjs.org/@types/semver/-/semver-5.5.0.tgz"
+ },
"@types/tough-cookie": {
"version": "2.3.4",
"from": "@types/tough-cookie@*",
@@ -254,6 +309,11 @@
"from": "bignumber.js@>=7.0.0 <8.0.0",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz"
},
+ "bindings": {
+ "version": "1.3.1",
+ "from": "bindings@>=1.2.1 <2.0.0",
+ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.1.tgz"
+ },
"bintrees": {
"version": "1.0.1",
"from": "bintrees@1.0.1",
@@ -431,10 +491,10 @@
"from": "deep-eql@>=3.0.1 <4.0.0",
"resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz"
},
- "define-properties": {
- "version": "1.1.3",
- "from": "define-properties@>=1.1.2 <2.0.0",
- "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz"
+ "delay": {
+ "version": "4.1.0",
+ "from": "delay@>=4.0.1 <5.0.0",
+ "resolved": "https://registry.npmjs.org/delay/-/delay-4.1.0.tgz"
},
"delayed-stream": {
"version": "0.0.5",
@@ -497,16 +557,6 @@
"from": "ent@>=2.2.0 <3.0.0",
"resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz"
},
- "es-abstract": {
- "version": "1.12.0",
- "from": "es-abstract@>=1.5.1 <2.0.0",
- "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz"
- },
- "es-to-primitive": {
- "version": "1.2.0",
- "from": "es-to-primitive@>=1.1.1 <2.0.0",
- "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz"
- },
"es6-promise": {
"version": "4.2.5",
"from": "es6-promise@>=4.0.3 <5.0.0",
@@ -609,9 +659,9 @@
}
},
"follow-redirects": {
- "version": "1.5.10",
+ "version": "1.6.0",
"from": "follow-redirects@>=1.3.0 <2.0.0",
- "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.10.tgz",
+ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.0.tgz",
"dependencies": {
"debug": {
"version": "3.1.0",
@@ -657,15 +707,15 @@
"from": "fs.realpath@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz"
},
- "function-bind": {
- "version": "1.1.1",
- "from": "function-bind@>=1.1.1 <2.0.0",
- "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz"
+ "gaxios": {
+ "version": "1.0.4",
+ "from": "gaxios@>=1.0.2 <2.0.0",
+ "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-1.0.4.tgz"
},
"gcp-metadata": {
- "version": "0.7.0",
- "from": "gcp-metadata@>=0.7.0 <0.8.0",
- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
+ "version": "0.9.3",
+ "from": "gcp-metadata@>=0.9.0 <0.10.0",
+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.9.3.tgz"
},
"get-func-name": {
"version": "2.0.0",
@@ -689,33 +739,26 @@
"optional": true
},
"google-auth-library": {
- "version": "1.6.1",
- "from": "google-auth-library@>=1.6.0 <2.0.0",
- "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-1.6.1.tgz",
+ "version": "2.0.2",
+ "from": "google-auth-library@>=2.0.0 <3.0.0",
+ "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-2.0.2.tgz",
"dependencies": {
"gcp-metadata": {
- "version": "0.6.3",
- "from": "gcp-metadata@>=0.6.3 <0.7.0",
- "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.6.3.tgz"
+ "version": "0.7.0",
+ "from": "gcp-metadata@>=0.7.0 <0.8.0",
+ "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-0.7.0.tgz"
},
"lru-cache": {
- "version": "4.1.4",
- "from": "lru-cache@>=4.1.3 <5.0.0",
- "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.4.tgz"
+ "version": "5.1.1",
+ "from": "lru-cache@>=5.0.0 <6.0.0",
+ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz"
}
}
},
"google-p12-pem": {
- "version": "1.0.2",
+ "version": "1.0.3",
"from": "google-p12-pem@>=1.0.0 <2.0.0",
- "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.2.tgz",
- "dependencies": {
- "pify": {
- "version": "3.0.0",
- "from": "pify@>=3.0.0 <4.0.0",
- "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz"
- }
- }
+ "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-1.0.3.tgz"
},
"graceful-fs": {
"version": "4.1.15",
@@ -920,11 +963,6 @@
}
}
},
- "has": {
- "version": "1.0.3",
- "from": "has@>=1.0.1 <2.0.0",
- "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz"
- },
"has-ansi": {
"version": "0.1.0",
"from": "has-ansi@>=0.1.0 <0.2.0",
@@ -935,11 +973,6 @@
"from": "has-flag@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz"
},
- "has-symbols": {
- "version": "1.0.0",
- "from": "has-symbols@>=1.0.0 <2.0.0",
- "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz"
- },
"hawk": {
"version": "0.10.2",
"from": "hawk@>=0.10.2 <0.11.0",
@@ -956,9 +989,9 @@
"resolved": "https://registry.npmjs.org/heapdump/-/heapdump-0.3.12.tgz"
},
"hex2dec": {
- "version": "1.1.0",
+ "version": "1.1.1",
"from": "hex2dec@>=1.0.1 <2.0.0",
- "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.0.tgz"
+ "resolved": "https://registry.npmjs.org/hex2dec/-/hex2dec-1.1.1.tgz"
},
"hoek": {
"version": "0.7.6",
@@ -1018,35 +1051,15 @@
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.8.0.tgz"
},
"is": {
- "version": "3.2.1",
- "from": "is@>=3.2.1 <4.0.0",
- "resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz"
+ "version": "3.3.0",
+ "from": "is@>=3.2.0 <4.0.0",
+ "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz"
},
"is-buffer": {
"version": "1.1.6",
"from": "is-buffer@>=1.1.5 <2.0.0",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz"
},
- "is-callable": {
- "version": "1.1.4",
- "from": "is-callable@>=1.1.3 <2.0.0",
- "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz"
- },
- "is-date-object": {
- "version": "1.0.1",
- "from": "is-date-object@>=1.0.1 <2.0.0",
- "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz"
- },
- "is-regex": {
- "version": "1.0.4",
- "from": "is-regex@>=1.0.4 <2.0.0",
- "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz"
- },
- "is-symbol": {
- "version": "1.0.2",
- "from": "is-symbol@>=1.0.2 <2.0.0",
- "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz"
- },
"isarray": {
"version": "1.0.0",
"from": "isarray@>=1.0.0 <2.0.0",
@@ -1136,15 +1149,15 @@
"from": "lodash.get@>=4.4.2 <5.0.0",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz"
},
- "lodash.isstring": {
- "version": "4.0.1",
- "from": "lodash.isstring@>=4.0.1 <5.0.0",
- "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz"
+ "lodash.pickby": {
+ "version": "4.6.0",
+ "from": "lodash.pickby@>=4.6.0 <5.0.0",
+ "resolved": "https://registry.npmjs.org/lodash.pickby/-/lodash.pickby-4.6.0.tgz"
},
"logger-sharelatex": {
- "version": "1.5.7",
- "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7",
- "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#13562f8866708fc86aef8202bf5a2ce4d1c6eed7",
+ "version": "1.5.9",
+ "from": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9",
+ "resolved": "git+https://github.com/sharelatex/logger-sharelatex.git#e8e1b95052f62e107336053e4a983f81cdbdf589",
"dependencies": {
"coffee-script": {
"version": "1.12.4",
@@ -1158,6 +1171,11 @@
"from": "lolex@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/lolex/-/lolex-3.0.0.tgz"
},
+ "long": {
+ "version": "4.0.0",
+ "from": "long@>=4.0.0 <5.0.0",
+ "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz"
+ },
"lru-cache": {
"version": "2.7.3",
"from": "lru-cache@>=2.0.0 <3.0.0",
@@ -1194,14 +1212,19 @@
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
},
"metrics-sharelatex": {
- "version": "2.0.7",
- "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7",
- "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3c7dd668d1153c13acee9cceb3a8ce24495b7c86",
+ "version": "2.0.12",
+ "from": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
+ "resolved": "git+https://github.com/sharelatex/metrics-sharelatex.git#3ac1621ef049e2f2d88a83b3a41011333d609662",
"dependencies": {
"coffee-script": {
"version": "1.6.0",
"from": "coffee-script@1.6.0",
"resolved": "https://registry.npmjs.org/coffee-script/-/coffee-script-1.6.0.tgz"
+ },
+ "underscore": {
+ "version": "1.6.0",
+ "from": "underscore@>=1.6.0 <1.7.0",
+ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
}
}
},
@@ -1402,16 +1425,6 @@
"from": "oauth-sign@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.2.0.tgz"
},
- "object-keys": {
- "version": "1.0.12",
- "from": "object-keys@>=1.0.12 <2.0.0",
- "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz"
- },
- "object.getownpropertydescriptors": {
- "version": "2.0.3",
- "from": "object.getownpropertydescriptors@>=2.0.3 <3.0.0",
- "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz"
- },
"on-finished": {
"version": "2.3.0",
"from": "on-finished@>=2.3.0 <2.4.0",
@@ -1423,15 +1436,25 @@
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz"
},
"p-limit": {
- "version": "2.0.0",
+ "version": "2.1.0",
"from": "p-limit@>=2.0.0 <3.0.0",
- "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz"
+ "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz"
},
"p-try": {
"version": "2.0.0",
"from": "p-try@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz"
},
+ "parse-duration": {
+ "version": "0.1.1",
+ "from": "parse-duration@>=0.1.1 <0.2.0",
+ "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-0.1.1.tgz"
+ },
+ "parse-ms": {
+ "version": "2.0.0",
+ "from": "parse-ms@>=2.0.0 <3.0.0",
+ "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.0.0.tgz"
+ },
"parseurl": {
"version": "1.3.2",
"from": "parseurl@>=1.3.2 <1.4.0",
@@ -1467,26 +1490,31 @@
"from": "pngcrush@0.0.3",
"resolved": "https://registry.npmjs.org/pngcrush/-/pngcrush-0.0.3.tgz"
},
+ "pretty-ms": {
+ "version": "4.0.0",
+ "from": "pretty-ms@>=4.0.0 <5.0.0",
+ "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-4.0.0.tgz"
+ },
"process-nextick-args": {
"version": "2.0.0",
"from": "process-nextick-args@>=2.0.0 <2.1.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz"
},
"prom-client": {
- "version": "11.2.0",
+ "version": "11.2.1",
"from": "prom-client@>=11.1.3 <12.0.0",
- "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.0.tgz"
+ "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-11.2.1.tgz"
+ },
+ "protobufjs": {
+ "version": "6.8.8",
+ "from": "protobufjs@>=6.8.6 <6.9.0",
+ "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz"
},
"proxy-addr": {
"version": "2.0.4",
"from": "proxy-addr@>=2.0.4 <2.1.0",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.4.tgz"
},
- "pseudomap": {
- "version": "1.0.2",
- "from": "pseudomap@>=1.0.2 <2.0.0",
- "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz"
- },
"punycode": {
"version": "1.3.2",
"from": "punycode@1.3.2",
@@ -1576,9 +1604,9 @@
"resolved": "https://registry.npmjs.org/require-like/-/require-like-0.1.2.tgz"
},
"resolve": {
- "version": "1.8.1",
+ "version": "1.9.0",
"from": "resolve@>=1.5.0 <2.0.0",
- "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.8.1.tgz"
+ "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz"
},
"response": {
"version": "0.14.0",
@@ -1764,15 +1792,20 @@
"from": "supports-color@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz"
},
+ "symbol-observable": {
+ "version": "1.2.0",
+ "from": "symbol-observable@>=1.2.0 <2.0.0",
+ "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz"
+ },
"tdigest": {
"version": "0.1.1",
"from": "tdigest@>=0.1.1 <0.2.0",
"resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz"
},
"teeny-request": {
- "version": "3.9.1",
- "from": "teeny-request@>=3.6.0 <4.0.0",
- "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.9.1.tgz",
+ "version": "3.11.3",
+ "from": "teeny-request@>=3.11.1 <4.0.0",
+ "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-3.11.3.tgz",
"dependencies": {
"uuid": {
"version": "3.3.2",
@@ -1841,11 +1874,6 @@
"from": "util-deprecate@>=1.0.1 <1.1.0",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
},
- "util.promisify": {
- "version": "1.0.0",
- "from": "util.promisify@>=1.0.0 <2.0.0",
- "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz"
- },
"utils-merge": {
"version": "1.0.1",
"from": "utils-merge@1.0.1",
diff --git a/services/filestore/package.json b/services/filestore/package.json
index fab7033316..4f25056dee 100644
--- a/services/filestore/package.json
+++ b/services/filestore/package.json
@@ -30,8 +30,8 @@
"fs-extra": "^1.0.0",
"heapdump": "^0.3.2",
"knox": "~0.9.1",
- "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.7",
- "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.7",
+ "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.9",
+ "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v2.0.12",
"mocha": "5.2.0",
"node-transloadit": "0.0.4",
"node-uuid": "~1.4.1",
diff --git a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee
index da59f859b9..c4a7d83d06 100644
--- a/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee
+++ b/services/filestore/test/unit/coffee/FSPersistorManagerTests.coffee
@@ -25,6 +25,7 @@ describe "FSPersistorManagerTests", ->
@Rimraf = sinon.stub()
@LocalFileWriter =
writeStream: sinon.stub()
+ deleteFile: sinon.stub()
@requires =
"./LocalFileWriter":@LocalFileWriter
"fs":@Fs
@@ -41,10 +42,32 @@ describe "FSPersistorManagerTests", ->
@FSPersistorManager = SandboxedModule.require modulePath, requires: @requires
describe "sendFile", ->
- it "should put the file", (done) ->
- @Fs.rename.callsArgWith(2,@error)
+ beforeEach ->
+ @Fs.createReadStream = sinon.stub().returns({
+ on: ->
+ pipe: ->
+ })
+
+ it "should copy the file", (done) ->
+ @Fs.createWriteStream =sinon.stub().returns({
+ on: (event, handler) ->
+ process.nextTick(handler) if event is 'finish'
+ })
@FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
- @Fs.rename.calledWith( @name2, "#{@location}/#{@name1Filtered}" ).should.equal true
+ @Fs.createReadStream.calledWith(@name2).should.equal true
+ @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
+ done()
+
+ it "should return an error if the file cannot be stored", (done) ->
+ @Fs.createWriteStream =sinon.stub().returns({
+ on: (event, handler) =>
+ if event is 'error'
+ process.nextTick () =>
+ handler(@error)
+ })
+ @FSPersistorManager.sendFile @location, @name1, @name2, (err)=>
+ @Fs.createReadStream.calledWith(@name2).should.equal true
+ @Fs.createWriteStream.calledWith("#{@location}/#{@name1Filtered}" ).should.equal true
err.should.equal @error
done()
@@ -52,6 +75,7 @@ describe "FSPersistorManagerTests", ->
beforeEach ->
@FSPersistorManager.sendFile = sinon.stub().callsArgWith(3)
@LocalFileWriter.writeStream.callsArgWith(2, null, @name1)
+ @LocalFileWriter.deleteFile.callsArg(1)
@SourceStream =
on:->
diff --git a/services/filestore/test/unit/coffee/FileHandlerTests.coffee b/services/filestore/test/unit/coffee/FileHandlerTests.coffee
index ab757b9360..50b8a17524 100644
--- a/services/filestore/test/unit/coffee/FileHandlerTests.coffee
+++ b/services/filestore/test/unit/coffee/FileHandlerTests.coffee
@@ -23,6 +23,7 @@ describe "FileHandler", ->
directorySize: sinon.stub()
@LocalFileWriter =
writeStream: sinon.stub()
+ getStream: sinon.stub()
deleteFile: sinon.stub()
@FileConverter =
convert: sinon.stub()
@@ -152,17 +153,20 @@ describe "FileHandler", ->
it "should _convertFile ", (done)->
@stubbedStream = {"something":"here"}
+ @localStream = {
+ on: ->
+ }
@PersistorManager.sendFile = sinon.stub().callsArgWith(3)
- @PersistorManager.getFileStream = sinon.stub().callsArgWith(3, null, @stubbedStream)
+ @LocalFileWriter.getStream = sinon.stub().callsArgWith(1, null, @localStream)
@convetedKey = @key+"converted"
@handler._convertFile = sinon.stub().callsArgWith(3, null, @stubbedPath)
@ImageOptimiser.compressPng = sinon.stub().callsArgWith(1)
@handler._getConvertedFileAndCache @bucket, @key, @convetedKey, {}, (err, fsStream)=>
@handler._convertFile.called.should.equal true
@PersistorManager.sendFile.calledWith(@bucket, @convetedKey, @stubbedPath).should.equal true
- @PersistorManager.getFileStream.calledWith(@bucket, @convetedKey).should.equal true
@ImageOptimiser.compressPng.calledWith(@stubbedPath).should.equal true
- fsStream.should.equal @stubbedStream
+ @LocalFileWriter.getStream.calledWith(@stubbedPath).should.equal true
+ fsStream.should.equal @localStream
done()
describe "_convertFile", ->
diff --git a/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee b/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee
index 773289f08e..55cf1f551f 100644
--- a/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee
+++ b/services/filestore/test/unit/coffee/LocalFileWriterTests.coffee
@@ -15,8 +15,11 @@ describe "LocalFileWriter", ->
on: (type, cb)->
if type == "finish"
cb()
+ @readStream =
+ on: ->
@fs =
createWriteStream : sinon.stub().returns(@writeStream)
+ createReadStream: sinon.stub().returns(@readStream)
unlink: sinon.stub()
@settings =
path:
@@ -56,6 +59,18 @@ describe "LocalFileWriter", ->
fsPath.should.equal @stubbedFsPath
done()
+ describe "getStream", ->
+
+ it "should read the stream from the file ", (done)->
+ @writer.getStream @stubbedFsPath, (err, stream)=>
+ @fs.createReadStream.calledWith(@stubbedFsPath).should.equal true
+ done()
+
+ it "should send the stream in the callback", (done)->
+ @writer.getStream @stubbedFsPath, (err, readStream)=>
+ readStream.should.equal @readStream
+ done()
+
describe "delete file", ->
it "should unlink the file", (done)->