Use external proxy

services/web/app/coffee/Features/LinkedFiles/UrlAgent.coffee

@@ -2,6 +2,7 @@ request = require 'request'
 FileWriter = require('../../infrastructure/FileWriter')
 _ = require "underscore"
 urlValidator = require 'valid-url'
+Settings = require 'settings-sharelatex'
 
 UrlFetchFailedError = (message) ->
 	error = new Error(message)
@@ -17,27 +18,21 @@ InvalidUrlError = (message) ->
 	return error
 InvalidUrlError.prototype.__proto__ = Error.prototype
 
-
 module.exports = UrlAgent = {
 	UrlFetchFailedError: UrlFetchFailedError
 	InvalidUrlError: InvalidUrlError
 
 	sanitizeData: (data) ->
 		return {
-			url: data.url
+			url: @._prependHttpIfNeeded(data.url)
 		}
 
-	_prependHttpIfNeeded: (url) ->
-		if !url.match('://')
-			url = 'http://' + url
-		return url
-
 	writeIncomingFileToDisk: (project_id, data, current_user_id, callback = (error, fsPath) ->) ->
-		# TODO: Proxy through external API
 		callback = _.once(callback)
-		url = @._prependHttpIfNeeded(data.url)
+		url = data.url
 		if !urlValidator.isWebUri(url)
 			return callback(new InvalidUrlError())
+		url = @._wrapWithProxy(url)
 		readStream = request.get(url)
 		readStream.on "error", callback
 		readStream.on "response", (response) ->
@@ -59,4 +54,15 @@ module.exports = UrlAgent = {
 			)
 		else
 			next(error)
+
+	_prependHttpIfNeeded: (url) ->
+		if !url.match('://')
+			url = 'http://' + url
+		return url
+
+	_wrapWithProxy: (url) ->
+		# TODO: Consider what to do for Community and Enterprise edition?
+		if !Settings.apis?.linkedUrlProxy?.url?
+			throw new Error('no linked url proxy configured')
+		return "#{Settings.apis.linkedUrlProxy.url}?url=#{encodeURIComponent(url)}"
 }

services/web/config/settings.defaults.coffee

@@ -152,6 +152,8 @@ module.exports = settings =
 			url: "http://#{process.env['NOTIFICATIONS_HOST'] or 'localhost'}:3042"
 		analytics:
 			url: "http://#{process.env['ANALYTICS_HOST'] or 'localhost'}:3050"
+		linkedUrlProxy:
+			url: process.env['LINKED_URL_PROXY']
 
 	templates:
 		user_id: process.env.TEMPLATES_USER_ID or "5395eb7aad1f29a88756c7f2"

services/web/docker-compose.yml

@@ -15,6 +15,7 @@ services:
       MONGO_URL: "mongodb://mongo/sharelatex"
       SHARELATEX_ALLOW_PUBLIC_ACCESS: 'true'
       PROJECT_HISTORY_ENABLED: 'true'
+      LINKED_URL_PROXY: 'http://localhost:6543'
     depends_on:
       - redis
       - mongo

services/web/test/acceptance/coffee/LinkedFilesTests.coffee

@@ -3,18 +3,23 @@ expect = require("chai").expect
 _ = require 'underscore'
 
 MockFileStoreApi = require './helpers/MockFileStoreApi'
-MockURLSource	= require './helpers/MockURLSource'
 request = require "./helpers/request"
 User = require "./helpers/User"
 
-MockURLSource.app.get "/foo", (req, res, next) =>
-	res.send('foo foo foo')
-MockURLSource.app.get "/bar", (req, res, next) =>
-	res.send('bar bar bar')
+
+express = require("express")
+LinkedUrlProxy = express()
+LinkedUrlProxy.get "/", (req, res, next) =>
+	if req.query.url == 'http://example.com/foo'
+		res.send('foo foo foo')
+	else if req.query.url == 'http://example.com/bar'
+		res.send('bar bar bar')
+	else
+		res.sendStatus(404)
 
 describe "LinkedFiles", ->
 	before (done) ->
-		MockURLSource.run (error) =>
+		LinkedUrlProxy.listen 6543, (error) =>
 			return done(error) if error?
 			@owner = new User() 
 			@owner.login done
@@ -36,7 +41,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-1'
@@ -48,7 +53,7 @@ describe "LinkedFiles", ->
 					file = project.rootFolder[0].fileRefs[0]
 					expect(file.linkedFileData).to.deep.equal({
 						provider: 'url'
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					})
 					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
 						throw error if error?
@@ -62,7 +67,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-2'
@@ -74,7 +79,7 @@ describe "LinkedFiles", ->
 					json:
 						provider: 'url'
 						data: {
-							url: "http://localhost:6543/bar"
+							url: 'http://example.com/bar'
 						}
 						parent_folder_id: @root_folder_id
 						name: 'url-test-file-2'
@@ -86,7 +91,7 @@ describe "LinkedFiles", ->
 						file = project.rootFolder[0].fileRefs[1]
 						expect(file.linkedFileData).to.deep.equal({
 							provider: 'url'
-							url: "http://localhost:6543/bar"
+							url: 'http://example.com/bar'
 						})
 						@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
 							throw error if error?
@@ -100,7 +105,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/does-not-exist"
+						url: 'http://example.com/does-not-exist'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-3'
@@ -154,7 +159,7 @@ describe "LinkedFiles", ->
 				json:
 					provider: 'url'
 					data: {
-						url: "http://localhost:6543/foo"
+						url: 'example.com/foo'
 					}
 					parent_folder_id: @root_folder_id
 					name: 'url-test-file-6'
@@ -167,7 +172,7 @@ describe "LinkedFiles", ->
 						file.name == 'url-test-file-6'
 					expect(file.linkedFileData).to.deep.equal({
 						provider: 'url'
-						url: "http://localhost:6543/foo"
+						url: 'http://example.com/foo'
 					})
 					@owner.request.get "/project/#{@project_id}/file/#{file._id}", (error, response, body) ->
 						throw error if error?

services/web/test/acceptance/coffee/helpers/MockURLSource.coffee

@@ -1,7 +0,0 @@
-express = require("express")
-app = express()
-
-module.exports = MockUrlSource =
-	app: app
-	run: (callback) ->
-		app.listen 6543, callback