Merge branch 'clsichecks'

services/web/app/coffee/Features/Compile/ClsiFormatChecker.coffee

@@ -0,0 +1,68 @@
+_ = require("lodash")
+async = require("async")
+
+module.exports = ClsiFormatChecker =
+
+	checkRecoursesForProblems: (resources, callback)->
+		jobs = 
+			conflictedPaths: (cb)->
+				ClsiFormatChecker._checkForConflictingPaths resources, cb
+
+			sizeCheck: (cb)->
+				ClsiFormatChecker._checkDocsAreUnderSizeLimit resources, cb
+
+		async.series jobs, (err, problems)->
+			if err? 
+				return callback(err)
+			
+			problems = _.omitBy(problems, _.isEmpty)
+
+			if _.isEmpty(problems)
+				return callback()
+			else
+				callback(null, problems)
+
+
+	_checkForConflictingPaths: (resources, callback)->
+		paths = _.map(resources, 'path')
+
+		conflicts = _.filter paths, (path)->
+			matchingPaths = _.filter paths, (checkPath)->
+				return checkPath.indexOf(path+"/") != -1
+
+			return matchingPaths.length > 0
+
+		conflictObjects = _.map conflicts, (conflict)->
+			path:conflict
+
+		callback null, conflictObjects
+
+	_checkDocsAreUnderSizeLimit: (resources, callback)->
+		
+		FIVEMB = 1000 * 1000 * 5
+
+		totalSize = 0
+
+		sizedResources = _.map resources, (resource)->
+			result = {path:resource.path}
+			if resource.content?
+				result.size = resource.content.replace(/\n/g).length
+				result.kbSize = Math.ceil(result.size / 1000)
+			else
+				result.size = 0
+			totalSize += result.size
+			return result
+
+		tooLarge = totalSize > FIVEMB
+		if !tooLarge
+			return callback()
+		else
+			sizedResources = _.sortBy(sizedResources, "size").reverse().slice(0, 10)
+			return callback(null, {resources:sizedResources, totalSize:totalSize})
+
+
+
+
+
+
+

services/web/app/coffee/Features/Compile/ClsiManager.coffee

@@ -7,25 +7,34 @@ ProjectEntityHandler = require("../Project/ProjectEntityHandler")
 logger = require "logger-sharelatex"
 Url = require("url")
 ClsiCookieManager = require("./ClsiCookieManager")
-
+_ = require("underscore")
+async = require("async")
+ClsiFormatChecker = require("./ClsiFormatChecker")
 
 module.exports = ClsiManager =
 
-	sendRequest: (project_id, options = {}, callback = (error, success) ->) ->
+	sendRequest: (project_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
 		ClsiManager._buildRequest project_id, options, (error, req) ->
 			return callback(error) if error?
 			logger.log project_id: project_id, "sending compile to CLSI"
-			ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) ->
-				if error?
-					logger.err err:error, project_id:project_id, "error sending request to clsi"
-					return callback(error)
-				logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, "received compile response from CLSI"
-				ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
-					if err?
-						logger.err err:err, project_id:project_id, "error getting server id"
-						return callback(err)
-					outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
-					callback(null, response?.compile?.status, outputFiles, clsiServerId)
+			ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)->
+				if err?
+					logger.err err, project_id, "could not check resources for potential problems before sending to clsi"
+					return callback(err)
+				if validationProblems?
+					logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted"
+					return callback(null, "validation-problems", null, null, validationProblems)
+				ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) ->
+					if error?
+						logger.err err:error, project_id:project_id, "error sending request to clsi"
+						return callback(error)
+					logger.log project_id: project_id, outputFilesLength: response?.outputFiles?.length, status: response?.status, "received compile response from CLSI"
+					ClsiCookieManager._getServerId project_id, (err, clsiServerId)->
+						if err?
+							logger.err err:err, project_id:project_id, "error getting server id"
+							return callback(err)
+						outputFiles = ClsiManager._parseOutputFiles(project_id, response?.compile?.outputFiles)
+						callback(null, response?.compile?.status, outputFiles, clsiServerId)
 
 	deleteAuxFiles: (project_id, options, callback = (error) ->) ->
 		compilerUrl = @_getCompilerUrl(options?.compileGroup)

services/web/app/coffee/Features/Compile/CompileController.coffee

@@ -30,7 +30,7 @@ module.exports = CompileController =
 			if req.body?.draft
 				options.draft = req.body.draft
 			logger.log {options, project_id}, "got compile request"
-			CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits) ->
+			CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits, validationProblems) ->
 				return next(error) if error?
 				res.contentType("application/json")
 				res.status(200).send JSON.stringify {
@@ -38,6 +38,7 @@ module.exports = CompileController =
 					outputFiles: outputFiles
 					compileGroup: limits?.compileGroup
 					clsiServerId:clsiServerId
+					validationProblems:validationProblems
 				}
 
 	downloadPdf: (req, res, next = (error) ->)->
@@ -138,10 +139,7 @@ module.exports = CompileController =
 			# expand any url parameter passed in as {url:..., qs:...}
 			if typeof url is "object"
 				{url, qs} = url
-			if limits.compileGroup == "priority"
-				compilerUrl = Settings.apis.clsi_priority.url
-			else
-				compilerUrl = Settings.apis.clsi.url
+			compilerUrl = Settings.apis.clsi.url
 			url = "#{compilerUrl}#{url}"
 			logger.log url: url, "proxying to CLSI"
 			oneMinute = 60 * 1000

services/web/app/coffee/Features/Compile/CompileManager.coffee

@@ -37,10 +37,10 @@ module.exports = CompileManager =
 							return callback(error) if error?
 							for key, value of limits
 								options[key] = value
-							ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId) ->
+							ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
 								return callback(error) if error?
 								logger.log files: outputFiles, "output files"
-								callback(null, status, outputFiles, clsiServerId, limits)
+								callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
 								
 	deleteAuxFiles: (project_id, callback = (error) ->) ->
 		CompileManager.getProjectCompileLimits project_id, (error, limits) ->

services/web/app/views/project/editor/pdf.jade

@@ -154,6 +154,27 @@ div.full-size.pdf(ng-controller="PdfController")
 				ng-if="settings.pdfViewer == 'native'"
 			)
 
+		.pdf-validation-problems(ng-switch-when="validation-problems")
+
+			.alert.alert-danger(ng-show="pdf.validation.duplicatePaths")
+				strong #{translate("latex_error")}
+				span #{translate("duplicate_paths_found")}
+
+
+			.alert.alert-danger(ng-show="pdf.validation.sizeCheck")
+				strong #{translate("project_too_large")} 
+				div #{translate("project_too_large_please_reduce")}
+				div
+					li(ng-repeat="entry in pdf.validation.sizeCheck.resources") {{ '/'+entry['path'] }} - {{entry['kbSize']}}kb
+	
+			.alert.alert-danger(ng-show="pdf.validation.conflictedPaths")
+				div
+					strong #{translate("conflicting_paths_found")}
+				div #{translate("following_paths_conflict")}
+				div
+					li(ng-repeat="entry in pdf.validation.conflictedPaths") {{ '/'+entry['path'] }}
+
+
 		.pdf-errors(ng-switch-when="errors")
 	
 			.alert.alert-danger(ng-show="pdf.error")

services/web/package.json

@@ -30,6 +30,7 @@
     "http-proxy": "^1.8.1",
     "jade": "~1.3.1",
     "ldapjs": "^1.0.0",
+    "lodash": "^4.13.1",
     "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.3.1",
     "lynx": "0.1.1",
     "marked": "^0.3.5",
@@ -57,7 +58,6 @@
     "temp": "^0.8.3",
     "translations-sharelatex": "git+https://github.com/sharelatex/translations-sharelatex.git#master",
     "underscore": "1.6.0",
-    "underscore.string": "^3.0.2",
     "v8-profiler": "^5.2.3",
     "xml2js": "0.2.0"
   },

services/web/public/coffee/ide/pdf/controllers/PdfController.coffee

@@ -42,11 +42,11 @@ define [
 			$scope.pdf.error      = false
 			$scope.pdf.timedout   = false
 			$scope.pdf.failure    = false
-			$scope.pdf.projectTooLarge = false
 			$scope.pdf.url        = null
 			$scope.pdf.clsiMaintenance = false
 			$scope.pdf.tooRecentlyCompiled = false
 			$scope.pdf.renderingError = false
+			$scope.pdf.projectTooLarge = false
 
 			# make a cache to look up files by name
 			fileByPath = {}
@@ -72,6 +72,9 @@ define [
 			else if response.status == "too-recently-compiled"
 				$scope.pdf.view = 'errors'
 				$scope.pdf.tooRecentlyCompiled = true
+			else if response.status == "validation-problems"
+				$scope.pdf.view = "validation-problems"
+				$scope.pdf.validation = response.validationProblems
 			else if response.status == "success"
 				$scope.pdf.view = 'pdf'
 				$scope.shouldShowLogs = false

services/web/public/stylesheets/app/editor/pdf.less

@@ -3,7 +3,7 @@
 	top: 58px;
 }
 
-.pdf-logs, .pdf-errors, .pdf-uncompiled {
+.pdf-logs, .pdf-errors, .pdf-uncompiled, .pdf-validation-problems{
 	padding: @line-height-computed / 2;
 }
 

services/web/test/UnitTests/coffee/Compile/ClsiFormatCheckerTests.coffee

@@ -0,0 +1,151 @@
+sinon = require('sinon')
+chai = require('chai')
+should = chai.should()
+expect = chai.expect
+modulePath = "../../../../app/js/Features/Compile/ClsiFormatChecker.js"
+SandboxedModule = require('sandboxed-module')
+
+describe "ClsiFormatChecker", ->
+	beforeEach ->
+		@ClsiFormatChecker = SandboxedModule.require modulePath, requires:
+			"settings-sharelatex": @settings ={}
+			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
+		@project_id = "project-id"
+
+
+	describe "checkRecoursesForProblems", ->
+
+		beforeEach ->
+			@resources = [{
+				path:    "main.tex"
+				content: "stuff"
+			}, {
+				path:    "chapters/chapter1"
+				content: "other stuff"
+			}, {
+				path: "stuff/image/image.png"
+				url:  "http:somewhere.com/project/#{@project_id}/file/1234124321312"
+				modified: "more stuff"
+			}]
+
+		it "should call _checkForDuplicatePaths and _checkForConflictingPaths", (done)->
+
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null)
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1)
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				@ClsiFormatChecker._checkForConflictingPaths.called.should.equal true
+				@ClsiFormatChecker._checkDocsAreUnderSizeLimit.called.should.equal true
+				done()
+
+		it "should remove undefined errors", (done)->
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {})
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				expect(problems).to.not.exist
+				expect(problems).to.not.exist
+				done()
+
+		it "should keep populated arrays", (done)->
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [{path:"somewhere/main.tex"}])
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {})
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				problems.conflictedPaths[0].path.should.equal "somewhere/main.tex"
+				expect(problems.sizeCheck).to.not.exist
+				done()
+
+		it "should keep populated object", (done)->
+			@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
+			@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {resources:[{"a.tex"},{"b.tex"}], totalSize:1000000})
+			@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
+				problems.sizeCheck.resources.length.should.equal 2
+				problems.sizeCheck.totalSize.should.equal 1000000
+				expect(problems.conflictedPaths).to.not.exist
+				done()
+
+		describe "_checkForConflictingPaths", ->
+
+			beforeEach ->
+
+				@resources.push({
+					path:    "chapters/chapter1.tex"
+					content: "other stuff"
+				})
+
+				@resources.push({
+					path:    "chapters.tex"
+					content: "other stuff"
+				})
+
+			it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)->
+				@resources.push({
+					path: "stuff/image"
+					url: "http://somwhere.com"
+				})
+
+				@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
+					conflictPathErrors.length.should.equal 1
+					conflictPathErrors[0].path.should.equal "stuff/image"
+					done()
+				
+			it "should flag up when a root level file has folder with same subpath as file elsewhere", (done)->
+				@resources.push({
+					path:    "stuff"
+					content: "other stuff"
+				})
+
+				@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
+					conflictPathErrors.length.should.equal 1
+					conflictPathErrors[0].path.should.equal "stuff"
+					done()
+
+			it "should not flag up when the file is a substring of a path", (done)->
+				@resources.push({
+					path:    "stuf"
+					content: "other stuff"
+				})
+
+				@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
+					conflictPathErrors.length.should.equal 0
+					done()
+				
+
+		describe "_checkDocsAreUnderSizeLimit", ->
+
+			it "should error when there is more than 5mb of data", (done)->
+
+				@resources.push({
+					path:    "massive.tex"
+					content: require("crypto").randomBytes(1000 * 1000 * 5).toString("hex")
+				})
+
+				while @resources.length < 20
+					@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
+
+				@ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
+					sizeError.totalSize.should.equal 10000016
+					sizeError.resources.length.should.equal 10
+					sizeError.resources[0].path.should.equal "massive.tex"
+					sizeError.resources[0].size.should.equal 1000 * 1000 * 10
+					done()
+			
+
+			it "should return nothing when project is correct size", (done)->
+
+				@resources.push({
+					path:    "massive.tex"
+					content: require("crypto").randomBytes(1000 * 1000 * 1).toString("hex")
+				})
+
+				while @resources.length < 20
+					@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
+
+				@ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
+					expect(sizeError).to.not.exist
+					done()
+
+
+
+
+
+
+

services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee

@@ -12,6 +12,8 @@ describe "ClsiManager", ->
 			getCookieJar: sinon.stub().callsArgWith(1, null, @jar)
 			setServerId: sinon.stub().callsArgWith(2)
 			_getServerId:sinon.stub()
+		@ClsiFormatChecker =
+			checkRecoursesForProblems:sinon.stub().callsArgWith(1)
 		@ClsiManager = SandboxedModule.require modulePath, requires:
 			"settings-sharelatex": @settings =
 				apis:
@@ -27,6 +29,7 @@ describe "ClsiManager", ->
 			"./ClsiCookieManager": @ClsiCookieManager
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
 			"request": @request = sinon.stub()
+			"./ClsiFormatChecker": @ClsiFormatChecker
 		@project_id = "project-id"
 		@callback = sinon.stub()
 
@@ -323,17 +326,3 @@ describe "ClsiManager", ->
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-

services/web/test/UnitTests/coffee/Compile/CompileControllerTests.coffee

@@ -206,16 +206,6 @@ describe "CompileController", ->
 					@CompileManager.getProjectCompileLimits = sinon.stub().callsArgWith(1, null, {compileGroup: "priority"})
 					@CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next)
 
-				it "should proxy to the priority url if the user has the feature", ()->
-					@request
-						.calledWith(
-							jar:@jar
-							method: @req.method
-							url: "#{@settings.apis.clsi_priority.url}#{@url}",
-							timeout: 60 * 1000
-						)
-						.should.equal true
-
 			describe "user with standard priority via query string", ->
 				beforeEach ->
 					@req.query = {compileGroup: 'standard'}
@@ -239,20 +229,6 @@ describe "CompileController", ->
 				it "should bind an error handle to the request proxy", ->
 					@proxy.on.calledWith("error").should.equal true
 
-			describe "user with priority compile via query string", ->
-				beforeEach ->
-					@req.query = {compileGroup: 'priority'}
-					@CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next)
-
-				it "should proxy to the priority url if the user has the feature", ()->
-					@request
-						.calledWith(
-							jar:@jar
-							method: @req.method
-							url: "#{@settings.apis.clsi_priority.url}#{@url}",
-							timeout: 60 * 1000
-						)
-						.should.equal true
 
 			describe "user with non-existent priority via query string", ->
 				beforeEach ->
@@ -316,25 +292,7 @@ describe "CompileController", ->
 				it "should bind an error handle to the request proxy", ->
 					@proxy.on.calledWith("error").should.equal true
 
-			describe "user with priority compile", ->
-				beforeEach ->
-					@CompileManager.getProjectCompileLimits = sinon.stub().callsArgWith(1, null, {compileGroup: "priority"})
-					@CompileController.proxyToClsi(@project_id, @url = "/test", @req, @res, @next)
 
-				it "should proxy to the priority url if the user has the feature", ()->
-					@request
-						.calledWith(
-							jar:@jar
-							method: @req.method
-							url: "#{@settings.apis.clsi_priority.url}#{@url}",
-							timeout: 60 * 1000
-							headers: {
-								'Range': '123-456'
-								'If-Range': 'abcdef'
-								'If-Modified-Since': 'Mon, 15 Dec 2014 15:23:56 GMT'
-							}
-						)
-						.should.equal true
 
 			describe "user with build parameter via query string", ->
 				beforeEach ->