spike to check latex for basic errors before compile

Aims to solve following problems which are currently not visible to user:

- project is too big, which files are worst offenders?
- when there are duplicate file names so an 'old' version keeps overrighting a new version
- when a file has the same path as a folder which blows up clsi i.e. images/research images/research/1.png
This commit is contained in:
Henry Oswald 2016-06-01 12:28:54 +01:00
parent a24f635531
commit 1437877b5a
2 changed files with 153 additions and 10 deletions

View file

@ -7,6 +7,8 @@ ProjectEntityHandler = require("../Project/ProjectEntityHandler")
logger = require "logger-sharelatex"
url = require("url")
ClsiCookieManager = require("./ClsiCookieManager")
_ = require("underscore")
async = require("async")
module.exports = ClsiManager =
@ -153,3 +155,70 @@ module.exports = ClsiManager =
logger.error err: error, project_id: project_id, "CLSI returned failure code"
callback error, body
_checkRecoursesForErrors: (resources, callback)->
jobs =
duplicatePaths: (cb)->
ClsiManager._checkForFilesWithSameName resources, cb
conflictedPaths: (cb)->
ClsiManager._checkForConflictingPaths resources, cb
sizeCheck: (cb)->
ClsiManager._checkDocsAreUnderSizeLimit resources, cb
async.series jobs, callback
_checkForFilesWithSameName: (resources, callback)->
paths = _.pluck(resources, 'path')
duplicates = _.filter paths, (path)->
return _.countBy(paths)[path] > 1
duplicates = _.uniq(duplicates)
duplicateObjects = _.map duplicates, (dup)->
path:dup
callback(null, duplicateObjects)
_checkForConflictingPaths: (resources, callback)->
paths = _.pluck(resources, 'path')
conflicts = _.filter paths, (path)->
matchingPaths = _.filter paths, (checkPath)->
return checkPath.indexOf(path+"/") != -1
return matchingPaths.length > 0
conflictObjects = _.map conflicts, (conflict)->
path:conflict
callback null, conflictObjects
_checkDocsAreUnderSizeLimit: (resources, callback)->
FIVEMB = 1000 * 1000 * 5
totalSize = 0
sizedResources = _.map resources, (resource)->
result = {path:resource.path}
if resource.content?
result.size = resource.content.join("").length
else
result.size = 0
totalSize += result.size
return result
tooLarge = totalSize > FIVEMB
sizedResources = _.sortBy(sizedResources, "size").slice(10).reverse()
callback(null, {resources:sizedResources, totalSize:totalSize, tooLarge:tooLarge})

View file

@ -320,16 +320,90 @@ describe "ClsiManager", ->
describe "_checkRecoursesForErrors", ->
beforeEach ->
@resources = [{
path: "main.tex"
content: ["stuff"]
}, {
path: "chapters/chapter1"
content: ["other stuff"]
}, {
path: "stuff/image/image.png"
url: "#{@settings.apis.filestore.url}/project/#{@project_id}/file/1234124321312"
modified: ["more stuff"]
}]
it "should call _checkForFilesWithSameName and _checkForConflictingPaths", (done)->
@ClsiManager._checkForFilesWithSameName = sinon.stub().callsArgWith(1)
@ClsiManager._checkForConflictingPaths = sinon.stub().callsArgWith(1)
@ClsiManager._checkDocsAreUnderSizeLimit = sinon.stub().callsArgWith(1)
@ClsiManager._checkRecoursesForErrors @resources, =>
@ClsiManager._checkForFilesWithSameName.called.should.equal true
@ClsiManager._checkForConflictingPaths.called.should.equal true
@ClsiManager._checkDocsAreUnderSizeLimit.called.should.equal true
done()
describe "_checkForFilesWithSameName", ->
it "should flag up 2 nested files with same path", (done)->
@resources.push({
path: "chapters/chapter1"
url: "http://somwhere.com"
})
@ClsiManager._checkForFilesWithSameName @resources, (err, duplicateErrors)->
duplicateErrors.length.should.equal 1
duplicateErrors[0].path.should.equal "chapters/chapter1"
done()
describe "_checkForConflictingPaths", ->
it "should flag up when a nested file has folder with same subpath as file elsewhere", (done)->
@resources.push({
path: "stuff/image"
url: "http://somwhere.com"
})
@resources.push({
path: "chapters/chapter1.tex"
content: ["other stuff"]
})
@resources.push({
path: "chapters.tex"
content: ["other stuff"]
})
@ClsiManager._checkForConflictingPaths @resources, (err, conflictPathErrors)->
conflictPathErrors.length.should.equal 1
conflictPathErrors[0].path.should.equal "stuff/image"
done()
describe "_checkDocsAreUnderSizeLimit", ->
it "should error when there is more than 2mb of data", (done)->
@resources.push({
path: "massive.tex"
content: [require("crypto").randomBytes(1000 * 1000 * 5).toString("hex")]
})
while @resources.length < 20
@resources.push({path:"chapters/chapter1.tex",url: "http://somwhere.com"})
@ClsiManager._checkDocsAreUnderSizeLimit @resources, (err, sizeError)->
sizeError.tooLarge.should.equal true
sizeError.totalSize.should.equal 10000016
sizeError.resources.length.should.equal 10
sizeError.resources[0].path.should.equal "massive.tex"
sizeError.resources[0].size.should.equal 1000 * 1000 * 10
done()