github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-03-22 02:04:31 +00:00
Code Issues Projects Releases Packages Wiki Activity

added clsiformat checker, wired in

Browse source
This commit is contained in:
Henry Oswald 2016-06-02 13:09:11 +01:00
parent 8a5cb86c31
commit c284465ba5
9 changed files with 258 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
services/web/app/coffee/Features/Compile/ClsiFormatChecker.coffee Normal file
View file

@ -0,0 +1,68 @@
_ = require("lodash")
async = require("async")
module.exports = ClsiFormatChecker =
checkRecoursesForProblems: (resources, callback)->
jobs =
conflictedPaths: (cb)->
ClsiFormatChecker._checkForConflictingPaths resources, cb
sizeCheck: (cb)->
ClsiFormatChecker._checkDocsAreUnderSizeLimit resources, cb
async.series jobs, (err, problems)->
if err?
return callback(err)
problems = _.omitBy(problems, _.isEmpty)
if _.isEmpty(problems)
return callback()
else
callback(null, problems)
_checkForConflictingPaths: (resources, callback)->
paths = _.map(resources, 'path')
conflicts = _.filter paths, (path)->
matchingPaths = _.filter paths, (checkPath)->
return checkPath.indexOf(path+"/") != -1
return matchingPaths.length > 0
conflictObjects = _.map conflicts, (conflict)->
path:conflict
callback null, conflictObjects
_checkDocsAreUnderSizeLimit: (resources, callback)->
FIVEMB = 1000 * 1000 * 5
totalSize = 0
sizedResources = _.map resources, (resource)->
result = {path:resource.path}
if resource.content?
result.size = resource.content.replace(/\n/g).length
result.kbSize = Math.ceil(result.size / 1000)
else
result.size = 0
totalSize += result.size
return result
tooLarge = totalSize > FIVEMB
if !tooLarge
return callback()
else
sizedResources = _.sortBy(sizedResources, "size").reverse().slice(0, 10)
return callback(null, {resources:sizedResources, totalSize:totalSize})

88
services/web/app/coffee/Features/Compile/ClsiManager.coffee
View file

@ -9,21 +9,21 @@ url = require("url")
ClsiCookieManager = require("./ClsiCookieManager")
_ = require("underscore")
async = require("async")
ClsiFormatChecker = require("./ClsiFormatChecker")
module.exports = ClsiManager =
sendRequest: (project_id, options = {}, callback = (error, success) ->) ->
sendRequest: (project_id, options = {}, callback = (error, status, outputFiles, clsiServerId, validationProblems) ->) ->
ClsiManager._buildRequest project_id, options, (error, req) ->
return callback(error) if error?
logger.log project_id: project_id, "sending compile to CLSI"
ClsiManager._checkRecoursesForErrors req.compile.resources, (err, problems)->
ClsiFormatChecker.checkRecoursesForProblems req.compile?.resources, (err, validationProblems)->
if err?
logger.err err, project_id, "could not check resources for potential problems before sending to clsi"
return callback(err)
if problems?
logger.log project_id:project_id, problems:problems, "problems with users latex before compile was attempted"
return callback(null, problems)
if validationProblems?
logger.log project_id:project_id, validationProblems:validationProblems, "problems with users latex before compile was attempted"
return callback(null, "validation-problems", null, null, validationProblems)
ClsiManager._postToClsi project_id, req, options.compileGroup, (error, response) ->
if error?
logger.err err:error, project_id:project_id, "error sending request to clsi"
@ -162,79 +162,3 @@ module.exports = ClsiManager =
logger.error err: error, project_id: project_id, "CLSI returned failure code"
callback error, body
_checkRecoursesForErrors: (resources, callback)->
jobs =
duplicatePaths: (cb)->
ClsiManager._checkForDuplicatePaths resources, cb
conflictedPaths: (cb)->
ClsiManager._checkForConflictingPaths resources, cb
sizeCheck: (cb)->
ClsiManager._checkDocsAreUnderSizeLimit resources, cb
async.series jobs, (err, problems)->
if err?
return callback(err)
problems = _.omit(problems, _.isEmpty)
if _.isEmpty(problems)
return callback()
else
callback(null, problems)
_checkForDuplicatePaths: (resources, callback)->
paths = _.pluck(resources, 'path')
duplicates = _.filter paths, (path)->
return _.countBy(paths)[path] > 1
duplicates = _.uniq(duplicates)
duplicateObjects = _.map duplicates, (dup)->
path:dup
callback(null, duplicateObjects)
_checkForConflictingPaths: (resources, callback)->
paths = _.pluck(resources, 'path')
conflicts = _.filter paths, (path)->
matchingPaths = _.filter paths, (checkPath)->
return checkPath.indexOf(path+"/") != -1
return matchingPaths.length > 0
conflictObjects = _.map conflicts, (conflict)->
path:conflict
callback null, conflictObjects
_checkDocsAreUnderSizeLimit: (resources, callback)->
FIVEMB = 1000 * 1000 * 5
totalSize = 0
sizedResources = _.map resources, (resource)->
result = {path:resource.path}
if resource.content?
result.size = resource.content.join("").length
else
result.size = 0
totalSize += result.size
return result
tooLarge = totalSize > FIVEMB
if !tooLarge
return callback()
else
sizedResources = _.sortBy(sizedResources, "size").slice(10).reverse()
return callback(null, {resources:sizedResources, totalSize:totalSize})

3
services/web/app/coffee/Features/Compile/CompileController.coffee
View file

@ -30,7 +30,7 @@ module.exports = CompileController =
if req.body?.draft
options.draft = req.body.draft
logger.log {options, project_id}, "got compile request"
CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits) ->
CompileManager.compile project_id, user_id, options, (error, status, outputFiles, clsiServerId, limits, validationProblems) ->
return next(error) if error?
res.contentType("application/json")
res.status(200).send JSON.stringify {
@ -38,6 +38,7 @@ module.exports = CompileController =
outputFiles: outputFiles
compileGroup: limits?.compileGroup
clsiServerId:clsiServerId
validationProblems:validationProblems
}
downloadPdf: (req, res, next = (error) ->)->

4
services/web/app/coffee/Features/Compile/CompileManager.coffee
View file

@ -37,10 +37,10 @@ module.exports = CompileManager =
return callback(error) if error?
for key, value of limits
options[key] = value
ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId) ->
ClsiManager.sendRequest project_id, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
return callback(error) if error?
logger.log files: outputFiles, "output files"
callback(null, status, outputFiles, clsiServerId, limits)
callback(null, status, outputFiles, clsiServerId, limits, validationProblems)
deleteAuxFiles: (project_id, callback = (error) ->) ->
CompileManager.getProjectCompileLimits project_id, (error, limits) ->

21
services/web/app/views/project/editor/pdf.jade
View file

@ -154,6 +154,27 @@ div.full-size.pdf(ng-controller="PdfController")
ng-if="settings.pdfViewer == 'native'"
)
.pdf-validation-problems(ng-switch-when="validation-problems")
.alert.alert-danger(ng-show="pdf.validation.duplicatePaths")
strong #{translate("latex_error")}
span #{translate("duplicate_paths_found")}
.alert.alert-danger(ng-show="pdf.validation.sizeCheck")
strong #{translate("project_too_large")}
div #{translate("project_too_large_please_reduce")}
div
li(ng-repeat="entry in pdf.validation.sizeCheck.resources") {{ '/'+entry['path'] }} - {{entry['kbSize']}}kb
.alert.alert-danger(ng-show="pdf.validation.conflictedPaths")
div
strong #{translate("conflicting_paths_found")}
div #{translate("following_paths_conflict")}
div
li(ng-repeat="entry in pdf.validation.conflictedPaths") {{ '/'+entry['path'] }}
.pdf-errors(ng-switch-when="errors")
.alert.alert-danger(ng-show="pdf.error")

6
services/web/public/coffee/ide/pdf/controllers/PdfController.coffee
View file

@ -42,11 +42,11 @@ define [
$scope.pdf.error = false
$scope.pdf.timedout = false
$scope.pdf.failure = false
$scope.pdf.projectTooLarge = false
$scope.pdf.url = null
$scope.pdf.clsiMaintenance = false
$scope.pdf.tooRecentlyCompiled = false
$scope.pdf.renderingError = false
$scope.pdf.projectTooLarge = false
if response.status == "timedout"
$scope.pdf.view = 'errors'
@ -60,13 +60,15 @@ define [
$scope.pdf.view = 'errors'
$scope.pdf.failure = true
$scope.shouldShowLogs = true
fetchLogs()
else if response.status == 'clsi-maintenance'
$scope.pdf.view = 'errors'
$scope.pdf.clsiMaintenance = true
else if response.status == "too-recently-compiled"
$scope.pdf.view = 'errors'
$scope.pdf.tooRecentlyCompiled = true
else if response.status == "validation-problems"
$scope.pdf.view = "validation-problems"
$scope.pdf.validation = response.validationProblems
else if response.status == "success"
$scope.pdf.view = 'pdf'
$scope.shouldShowLogs = false

2
services/web/public/stylesheets/app/editor/pdf.less
View file

@ -3,7 +3,7 @@
top: 58px;
}
.pdf-logs, .pdf-errors, .pdf-uncompiled {
.pdf-logs, .pdf-errors, .pdf-uncompiled, .pdf-validation-problems{
padding: @line-height-computed / 2;
}

151
services/web/test/UnitTests/coffee/Compile/ClsiFormatCheckerTests.coffee Normal file
View file

@ -0,0 +1,151 @@
sinon = require('sinon')
chai = require('chai')
should = chai.should()
expect = chai.expect
modulePath = "../../../../app/js/Features/Compile/ClsiFormatChecker.js"
SandboxedModule = require('sandboxed-module')
describe "ClsiFormatChecker", ->
beforeEach ->
@ClsiFormatChecker = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings ={}
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
@project_id = "project-id"
describe "checkRecoursesForProblems", ->
beforeEach ->
@resources = [{
path: "main.tex"
content: "stuff"
}, {
path: "chapters/chapter1"
content: "other stuff"
}, {
path: "stuff/image/image.png"
url: "http:somewhere.com/project/#{@project_id}/file/1234124321312"
modified: "more stuff"
}]
it "should call _checkForDuplicatePaths and _checkForConflictingPaths", (done)->
@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null)
@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1)
@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
@ClsiFormatChecker._checkForConflictingPaths.called.should.equal true
@ClsiFormatChecker._checkDocsAreUnderSizeLimit.called.should.equal true
done()
it "should remove undefined errors", (done)->
@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {})
@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
expect(problems).to.not.exist
expect(problems).to.not.exist
done()
it "should keep populated arrays", (done)->
@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [{path:"somewhere/main.tex"}])
@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {})
@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
problems.conflictedPaths[0].path.should.equal "somewhere/main.tex"
expect(problems.sizeCheck).to.not.exist
done()
it "should keep populated object", (done)->
@ClsiFormatChecker._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
@ClsiFormatChecker._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null, {resources:[{"a.tex"},{"b.tex"}], totalSize:1000000})
@ClsiFormatChecker.checkRecoursesForProblems @resources, (err, problems)=>
problems.sizeCheck.resources.length.should.equal 2
problems.sizeCheck.totalSize.should.equal 1000000
expect(problems.conflictedPaths).to.not.exist
done()
describe "_checkForConflictingPaths", ->
beforeEach ->
@resources.push({
path: "chapters/chapter1.tex"
content: "other stuff"
})
@resources.push({
path: "chapters.tex"
content: "other stuff"
})
it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)->
@resources.push({
path: "stuff/image"
url: "http://somwhere.com"
})
@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
conflictPathErrors.length.should.equal 1
conflictPathErrors[0].path.should.equal "stuff/image"
done()
it "should flag up when a root level file has folder with same subpath as file elsewhere", (done)->
@resources.push({
path: "stuff"
content: "other stuff"
})
@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
conflictPathErrors.length.should.equal 1
conflictPathErrors[0].path.should.equal "stuff"
done()
it "should not flag up when the file is a substring of a path", (done)->
@resources.push({
path: "stuf"
content: "other stuff"
})
@ClsiFormatChecker._checkForConflictingPaths @resources, (err, conflictPathErrors)->
conflictPathErrors.length.should.equal 0
done()
describe "_checkDocsAreUnderSizeLimit", ->
it "should error when there is more than 5mb of data", (done)->
@resources.push({
path: "massive.tex"
content: require("crypto").randomBytes(1000 * 1000 * 5).toString("hex")
})
while @resources.length < 20
@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
@ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
sizeError.totalSize.should.equal 10000016
sizeError.resources.length.should.equal 10
sizeError.resources[0].path.should.equal "massive.tex"
sizeError.resources[0].size.should.equal 1000 * 1000 * 10
done()
it "should return nothing when project is correct size", (done)->
@resources.push({
path: "massive.tex"
content: require("crypto").randomBytes(1000 * 1000 * 1).toString("hex")
})
while @resources.length < 20
@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
@ClsiFormatChecker._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
expect(sizeError).to.not.exist
done()

103
services/web/test/UnitTests/coffee/Compile/ClsiManagerTests.coffee
View file

@ -12,6 +12,8 @@ describe "ClsiManager", ->
getCookieJar: sinon.stub().callsArgWith(1, null, @jar)
setServerId: sinon.stub().callsArgWith(2)
_getServerId:sinon.stub()
@ClsiFormatChecker =
checkRecoursesForProblems:sinon.stub().callsArgWith(1)
@ClsiManager = SandboxedModule.require modulePath, requires:
"settings-sharelatex": @settings =
apis:
@ -27,6 +29,7 @@ describe "ClsiManager", ->
"./ClsiCookieManager": @ClsiCookieManager
"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub(), warn: sinon.stub() }
"request": @request = sinon.stub()
"./ClsiFormatChecker": @ClsiFormatChecker
@project_id = "project-id"
@callback = sinon.stub()
@ -320,106 +323,6 @@ describe "ClsiManager", ->
describe "_checkRecoursesForErrors", ->
beforeEach ->
@resources = [{
path: "main.tex"
content: ["stuff"]
}, {
path: "chapters/chapter1"
content: ["other stuff"]
}, {
path: "stuff/image/image.png"
url: "#{@settings.apis.filestore.url}/project/#{@project_id}/file/1234124321312"
modified: ["more stuff"]
}]
it "should call _checkForDuplicatePaths and _checkForConflictingPaths", (done)->
@ClsiManager._checkForDuplicatePaths = sinon.stub().callsArgWith(1)
@ClsiManager._checkForConflictingPaths = sinon.stub().callsArgWith(1)
@ClsiManager._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1)
@ClsiManager._checkRecoursesForErrors @resources, (err, problems)=>
@ClsiManager._checkForDuplicatePaths.called.should.equal true
@ClsiManager._checkForConflictingPaths.called.should.equal true
@ClsiManager._checkDocsAreUnderSizeLimit.called.should.equal true
expect(problems).to.not.exist
done()
it "should remove undefined errors", (done)->
@ClsiManager._checkForDuplicatePaths = sinon.stub().callsArgWith(1, null, [path:"something/here"])
@ClsiManager._checkForConflictingPaths = sinon.stub().callsArgWith(1, null, [])
@ClsiManager._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1, null)
@ClsiManager._checkRecoursesForErrors @resources, (err, problems)=>
problems.duplicatePaths[0].path.should.equal "something/here"
expect(problems.conflictedPaths).to.not.exist
expect(problems.sizeCheck).to.not.exist
done()
describe "_checkForDuplicatePaths", ->
it "should flag up 2 nested files with same path", (done)->
@resources.push({
path: "chapters/chapter1"
url: "http://somwhere.com"
})
@ClsiManager._checkForDuplicatePaths @resources, (err, duplicateErrors)->
duplicateErrors.length.should.equal 1
duplicateErrors[0].path.should.equal "chapters/chapter1"
done()
describe "_checkForConflictingPaths", ->
it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)->
@resources.push({
path: "stuff/image"
url: "http://somwhere.com"
})
@resources.push({
path: "chapters/chapter1.tex"
content: ["other stuff"]
})
@resources.push({
path: "chapters.tex"
content: ["other stuff"]
})
@ClsiManager._checkForConflictingPaths @resources, (err, conflictPathErrors)->
conflictPathErrors.length.should.equal 1
conflictPathErrors[0].path.should.equal "stuff/image"
done()
describe "_checkDocsAreUnderSizeLimit", ->
it "should error when there is more than 2mb of data", (done)->
@resources.push({
path: "massive.tex"
content: [require("crypto").randomBytes(1000 * 1000 * 5).toString("hex")]
})
while @resources.length < 20
@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
@ClsiManager._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
sizeError.totalSize.should.equal 10000016
sizeError.resources.length.should.equal 10
sizeError.resources[0].path.should.equal "massive.tex"
sizeError.resources[0].size.should.equal 1000 * 1000 * 10
done()