Add metric for s3 ingress

This commit is contained in:
Simon Detheridge 2019-12-05 15:46:14 +00:00
parent 1d1106bc67
commit 48aa141591
3 changed files with 37 additions and 5 deletions

View file

@ -132,6 +132,9 @@ module.exports =
logger.log({bucketName: bucketName, key: key }, "error getting file from s3: #{statusCode}") logger.log({bucketName: bucketName, key: key }, "error getting file from s3: #{statusCode}")
return callback(new Error("Got non-200 response from S3: #{statusCode} #{statusMessage}"), null) return callback(new Error("Got non-200 response from S3: #{statusCode} #{statusMessage}"), null)
stream = response.httpResponse.createUnbufferedStream() stream = response.httpResponse.createUnbufferedStream()
stream.on 'data', (data) ->
metrics.count 's3.ingress', data.byteLength
callback(null, stream) callback(null, stream)
s3Request.on 'error', (err) => s3Request.on 'error', (err) =>

View file

@ -9,6 +9,7 @@ fs = require("fs")
request = require("request") request = require("request")
settings = require("settings-sharelatex") settings = require("settings-sharelatex")
FilestoreApp = require "./FilestoreApp" FilestoreApp = require "./FilestoreApp"
async = require('async')
getMetric = (filestoreUrl, metric, cb) -> getMetric = (filestoreUrl, metric, cb) ->
@ -33,10 +34,19 @@ describe "Filestore", ->
beforeEach (done)-> beforeEach (done)->
FilestoreApp.ensureRunning => FilestoreApp.ensureRunning =>
fs.unlink @localFileWritePath, => async.parallel [
getMetric @filestoreUrl, 's3_egress', (metric) => (cb) =>
@previousEgress = metric fs.unlink @localFileWritePath, () ->
done() cb()
(cb) =>
getMetric @filestoreUrl, 's3_egress', (metric) =>
@previousEgress = metric
cb()
(cb) =>
getMetric @filestoreUrl, 's3_ingress', (metric) =>
@previousIngress = metric
cb()
], done
it "should send a 200 for status endpoint", (done)-> it "should send a 200 for status endpoint", (done)->
request "#{@filestoreUrl}/status", (err, response, body)-> request "#{@filestoreUrl}/status", (err, response, body)->
@ -83,7 +93,14 @@ describe "Filestore", ->
body.should.equal @constantFileContent body.should.equal @constantFileContent
done() done()
it "should be able to get back the first 8 bytes of the file", (done) -> it "should record an ingress metric when downloading the file", (done)->
@timeout(1000 * 10)
request.get @fileUrl, () =>
getMetric @filestoreUrl, 's3_ingress', (metric) =>
expect(metric - @previousIngress).to.equal @constantFileContent.length
done()
it "should be able to get back the first 9 bytes of the file", (done) ->
@timeout(1000 * 10) @timeout(1000 * 10)
options = options =
uri: @fileUrl uri: @fileUrl
@ -93,6 +110,17 @@ describe "Filestore", ->
body.should.equal 'hello wor' body.should.equal 'hello wor'
done() done()
it "should record an ingress metric for a partial download", (done)->
@timeout(1000 * 10)
options =
uri: @fileUrl
headers:
'Range': 'bytes=0-8'
request.get options, ()=>
getMetric @filestoreUrl, 's3_ingress', (metric) =>
expect(metric - @previousIngress).to.equal 9
done()
it "should be able to get back bytes 4 through 10 of the file", (done) -> it "should be able to get back bytes 4 through 10 of the file", (done) ->
@timeout(1000 * 10) @timeout(1000 * 10)
options = options =

View file

@ -62,6 +62,7 @@ describe "S3PersistorManagerTests", ->
describe "success", -> describe "success", ->
beforeEach () -> beforeEach () ->
@expectedStream = { expectedStream: true } @expectedStream = { expectedStream: true }
@expectedStream.on = sinon.stub()
@s3Request.send.callsFake () => @s3Request.send.callsFake () =>
@s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK") @s3EventHandlers.httpHeaders(200, {}, @s3Response, "OK")
@s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream) @s3Response.httpResponse.createUnbufferedStream.returns(@expectedStream)