Use external proxy

This commit is contained in:
James Allen 2018-02-21 11:19:21 +00:00
parent cae09028ff
commit a99f55891f
5 changed files with 37 additions and 30 deletions

View file

@ -2,6 +2,7 @@ request = require 'request'
FileWriter = require('../../infrastructure/FileWriter')
_ = require "underscore"
urlValidator = require 'valid-url'
Settings = require 'settings-sharelatex'
UrlFetchFailedError = (message) ->
error = new Error(message)
@ -17,27 +18,21 @@ InvalidUrlError = (message) ->
return error
InvalidUrlError.prototype.__proto__ = Error.prototype
module.exports = UrlAgent = {
UrlFetchFailedError: UrlFetchFailedError
InvalidUrlError: InvalidUrlError
sanitizeData: (data) ->
return {
url: data.url
url: @._prependHttpIfNeeded(data.url)
}
_prependHttpIfNeeded: (url) ->
if !url.match('://')
url = 'http://' + url
return url
writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
# TODO: Proxy through external API
callback = _.once(callback)
url = @._prependHttpIfNeeded(data.url)
url = data.url
if !urlValidator.isWebUri(url)
return callback(new InvalidUrlError())
url = @._wrapWithProxy(url)
readStream = request.get(url)
readStream.on "error", callback
readStream.on "response", (response) ->
@ -59,4 +54,15 @@ module.exports = UrlAgent = {
)
else
next(error)
_prependHttpIfNeeded: (url) ->
if !url.match('://')
url = 'http://' + url
return url
_wrapWithProxy: (url) ->
# TODO: Consider what to do for Community and Enterprise edition?
if !Settings.apis?.linkedUrlProxy?.url?
throw new Error('no linked url proxy configured')
return "#{Settings.apis.linkedUrlProxy.url}?url=#{encodeURIComponent(url)}"
}

View file

@ -152,6 +152,8 @@ module.exports = settings =
url: "http://#{process.env['NOTIFICATIONS_HOST'] or 'localhost'}:3042"
analytics:
url: "http://#{process.env['ANALYTICS_HOST'] or 'localhost'}:3050"
linkedUrlProxy:
url: process.env['LINKED_URL_PROXY']
templates:
user_id: process.env.TEMPLATES_USER_ID or "5395eb7aad1f29a88756c7f2"

View file

@ -15,6 +15,7 @@ services:
MONGO_URL: "mongodb://mongo/sharelatex"
SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
PROJECT_HISTORY_ENABLED: 'true'
LINKED_URL_PROXY: 'http://localhost:6543'
depends_on:
- redis
- mongo

View file

@ -3,18 +3,23 @@ expect = require("chai").expect
_ = require 'underscore'
MockFileStoreApi = require './helpers/MockFileStoreApi'
MockURLSource = require './helpers/MockURLSource'
request = require "./helpers/request"
User = require "./helpers/User"
MockURLSource.app.get "/foo", (req, res, next) =>
res.send('foo foo foo')
MockURLSource.app.get "/bar", (req, res, next) =>
res.send('bar bar bar')
express = require("express")
LinkedUrlProxy = express()
LinkedUrlProxy.get "/", (req, res, next) =>
if req.query.url == 'http://example.com/foo'
res.send('foo foo foo')
else if req.query.url == 'http://example.com/bar'
res.send('bar bar bar')
else
res.sendStatus(404)
describe "LinkedFiles", ->
before (done) ->
MockURLSource.run (error) =>
LinkedUrlProxy.listen 6543, (error) =>
return done(error) if error?
@owner = new User()
@owner.login done
@ -36,7 +41,7 @@ describe "LinkedFiles", ->
json:
provider: 'url'
data: {
url: "http://localhost:6543/foo"
url: 'http://example.com/foo'
}
parent_folder_id: @root_folder_id
name: 'url-test-file-1'
@ -48,7 +53,7 @@ describe "LinkedFiles", ->
file = project.rootFolder[0].fileRefs[0]
expect(file.linkedFileData).to.deep.equal({
provider: 'url'
url: "http://localhost:6543/foo"
url: 'http://example.com/foo'
})
@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
throw error if error?
@ -62,7 +67,7 @@ describe "LinkedFiles", ->
json:
provider: 'url'
data: {
url: "http://localhost:6543/foo"
url: 'http://example.com/foo'
}
parent_folder_id: @root_folder_id
name: 'url-test-file-2'
@ -74,7 +79,7 @@ describe "LinkedFiles", ->
json:
provider: 'url'
data: {
url: "http://localhost:6543/bar"
url: 'http://example.com/bar'
}
parent_folder_id: @root_folder_id
name: 'url-test-file-2'
@ -86,7 +91,7 @@ describe "LinkedFiles", ->
file = project.rootFolder[0].fileRefs[1]
expect(file.linkedFileData).to.deep.equal({
provider: 'url'
url: "http://localhost:6543/bar"
url: 'http://example.com/bar'
})
@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
throw error if error?
@ -100,7 +105,7 @@ describe "LinkedFiles", ->
json:
provider: 'url'
data: {
url: "http://localhost:6543/does-not-exist"
url: 'http://example.com/does-not-exist'
}
parent_folder_id: @root_folder_id
name: 'url-test-file-3'
@ -154,7 +159,7 @@ describe "LinkedFiles", ->
json:
provider: 'url'
data: {
url: "http://localhost:6543/foo"
url: 'example.com/foo'
}
parent_folder_id: @root_folder_id
name: 'url-test-file-6'
@ -167,7 +172,7 @@ describe "LinkedFiles", ->
file.name == 'url-test-file-6'
expect(file.linkedFileData).to.deep.equal({
provider: 'url'
url: "http://localhost:6543/foo"
url: 'http://example.com/foo'
})
@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
throw error if error?

View file

@ -1,7 +0,0 @@
express = require("express")
app = express()
module.exports = MockUrlSource =
app: app
run: (callback) ->
app.listen 6543, callback