github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #37 from sharelatex/bg-rate-limit-autocompile

Browse source 
rate limit autocompile (connects to #18)
This commit is contained in:
Brian Gough 2017-10-12 09:25:59 +01:00 committed by GitHub
commit 45ed090326
5 changed files with 56 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
services/web/app/coffee/Features/Compile/CompileManager.coffee
View file

@ -18,7 +18,7 @@ module.exports = CompileManager =
timer.done() timer.done()
_callback(args...) _callback(args...)
@_checkIfAutoCompileLimitHasBeenHit options.isAutoCompile, (err, canCompile)-> @_checkIfAutoCompileLimitHasBeenHit options.isAutoCompile, "everyone", (err, canCompile)->
if !canCompile if !canCompile
return callback null, "autocompile-backoff", [] return callback null, "autocompile-backoff", []
logger.log project_id: project_id, user_id: user_id, "compiling project" logger.log project_id: project_id, user_id: user_id, "compiling project"
@ -34,6 +34,10 @@ module.exports = CompileManager =
return callback(error) if error? return callback(error) if error?
for key, value of limits for key, value of limits
options[key] = value options[key] = value
# Put a lower limit on autocompiles for free users, based on compileGroup
CompileManager._checkCompileGroupAutoCompileLimit options.isAutoCompile, limits.compileGroup, (err, canCompile)->
if !canCompile
return callback null, "autocompile-backoff", []
# only pass user_id down to clsi if this is a per-user compile # only pass user_id down to clsi if this is a per-user compile
compileAsUser = if Settings.disablePerUserCompiles then undefined else user_id compileAsUser = if Settings.disablePerUserCompiles then undefined else user_id
ClsiManager.sendRequest project_id, compileAsUser, options, (error, status, outputFiles, clsiServerId, validationProblems) -> ClsiManager.sendRequest project_id, compileAsUser, options, (error, status, outputFiles, clsiServerId, validationProblems) ->
@ -72,18 +76,27 @@ module.exports = CompileManager =
else else
return callback null, true return callback null, true
_checkIfAutoCompileLimitHasBeenHit: (isAutoCompile, callback = (err, canCompile)->)-> _checkCompileGroupAutoCompileLimit: (isAutoCompile, compileGroup, callback = (err, canCompile)->)->
if compileGroup is "default"
CompileManager._checkIfAutoCompileLimitHasBeenHit isAutoCompile, compileGroup, callback
else
Metrics.inc "auto-compile-#{compileGroup}"
return callback(null, true) # always allow priority group users to compile
_checkIfAutoCompileLimitHasBeenHit: (isAutoCompile, compileGroup, callback = (err, canCompile)->)->
if !isAutoCompile if !isAutoCompile
return callback(null, true) return callback(null, true)
Metrics.inc "auto-compile-#{compileGroup}"
opts = opts =
endpointName:"auto_compile" endpointName:"auto_compile"
timeInterval:20 timeInterval:20
subjectName:"everyone" subjectName:compileGroup
throttle: 25 throttle: Settings?.rateLimit?.autoCompile?[compileGroup] || 25
rateLimiter.addCount opts, (err, canCompile)-> rateLimiter.addCount opts, (err, canCompile)->
if err? if err?
canCompile = false canCompile = false
logger.log canCompile:canCompile, opts:opts, "checking if auto compile limit has been hit" if !canCompile
Metrics.inc "auto-compile-#{compileGroup}-limited"
callback err, canCompile callback err, canCompile
_ensureRootDocumentIsSet: (project_id, callback = (error) ->) -> _ensureRootDocumentIsSet: (project_id, callback = (error) ->) ->

6
services/web/app/views/project/editor/pdf.pug
View file

@ -393,6 +393,12 @@ div.full-size.pdf(ng-controller="PdfController")
ng-click="startFreeTrial('compile-timeout')" ng-click="startFreeTrial('compile-timeout')"
) #{translate("start_free_trial")} ) #{translate("start_free_trial")}
.alert.alert-danger(ng-show="pdf.autoCompileDisabled")
p
strong #{translate("autocompile_disabled")}.
span #{translate("autocompile_disabled_reason")}
.alert.alert-danger(ng-show="pdf.projectTooLarge") .alert.alert-danger(ng-show="pdf.projectTooLarge")
strong #{translate("project_too_large")} strong #{translate("project_too_large")}
span #{translate("project_too_large_please_reduce")} span #{translate("project_too_large_please_reduce")}

5
services/web/config/settings.defaults.coffee
View file

@ -437,3 +437,8 @@ module.exports = settings =
# name : "all projects", # name : "all projects",
# url: "/templates/all" # url: "/templates/all"
#}] #}]
rateLimits:
autoCompile:
everyone: 100
standard: 25

14
services/web/public/coffee/ide/pdf/controllers/PdfController.coffee
View file

@ -68,7 +68,7 @@ define [
$scope.$on "project:joined", () -> $scope.$on "project:joined", () ->
return if !autoCompile return if !autoCompile
autoCompile = false autoCompile = false
$scope.recompile(isAutoCompile: true) $scope.recompile(isAutoCompileOnLoad: true)
$scope.hasPremiumCompile = $scope.project.features.compileGroup == "priority" $scope.hasPremiumCompile = $scope.project.features.compileGroup == "priority"
$scope.$on "pdf:error:display", () -> $scope.$on "pdf:error:display", () ->
@ -86,7 +86,7 @@ define [
if isTimeNonMonotonic || timeSinceLastCompile >= AUTO_COMPILE_TIMEOUT if isTimeNonMonotonic || timeSinceLastCompile >= AUTO_COMPILE_TIMEOUT
if (!ide.$scope.hasLintingError) if (!ide.$scope.hasLintingError)
$scope.recompile(isBackgroundAutoCompile: true) $scope.recompile(isAutoCompileOnChange: true)
else else
# Extend remainder of timeout # Extend remainder of timeout
autoCompileTimeout = setTimeout () -> autoCompileTimeout = setTimeout () ->
@ -127,7 +127,7 @@ define [
sendCompileRequest = (options = {}) -> sendCompileRequest = (options = {}) ->
url = "/project/#{$scope.project_id}/compile" url = "/project/#{$scope.project_id}/compile"
params = {} params = {}
if options.isAutoCompile if options.isAutoCompileOnLoad or options.isAutoCompileOnChange
params["auto_compile"]=true params["auto_compile"]=true
# if the previous run was a check, clear the error logs # if the previous run was a check, clear the error logs
$scope.pdf.logEntries = [] if $scope.check $scope.pdf.logEntries = [] if $scope.check
@ -168,6 +168,7 @@ define [
$scope.pdf.compileExited = false $scope.pdf.compileExited = false
$scope.pdf.failedCheck = false $scope.pdf.failedCheck = false
$scope.pdf.compileInProgress = false $scope.pdf.compileInProgress = false
$scope.pdf.autoCompileDisabled = false
# make a cache to look up files by name # make a cache to look up files by name
fileByPath = {} fileByPath = {}
@ -206,7 +207,13 @@ define [
$scope.shouldShowLogs = true $scope.shouldShowLogs = true
fetchLogs(fileByPath) fetchLogs(fileByPath)
else if response.status == "autocompile-backoff" else if response.status == "autocompile-backoff"
if $scope.pdf.isAutoCompileOnLoad # initial autocompile
$scope.pdf.view = 'uncompiled' $scope.pdf.view = 'uncompiled'
else # background autocompile from typing
$scope.pdf.view = 'errors'
$scope.pdf.autoCompileDisabled = true
$scope.autocompile_enabled = false # disable any further autocompiles
event_tracking.sendMB "autocompile-rate-limited", {hasPremiumCompile: $scope.hasPremiumCompile}
else if response.status == "project-too-large" else if response.status == "project-too-large"
$scope.pdf.view = 'errors' $scope.pdf.view = 'errors'
$scope.pdf.projectTooLarge = true $scope.pdf.projectTooLarge = true
@ -418,6 +425,7 @@ define [
event_tracking.sendMBSampled "editor-recompile-sampled", options event_tracking.sendMBSampled "editor-recompile-sampled", options
$scope.pdf.compiling = true $scope.pdf.compiling = true
$scope.pdf.isAutoCompileOnLoad = options?.isAutoCompileOnLoad # initial autocompile
if options?.force if options?.force
# for forced compile, turn off validation check and ignore errors # for forced compile, turn off validation check and ignore errors

16
services/web/test/UnitTests/coffee/Compile/CompileManagerTests.coffee
View file

@ -44,7 +44,7 @@ describe "CompileManager", ->
describe "succesfully", -> describe "succesfully", ->
beforeEach -> beforeEach ->
@CompileManager._checkIfAutoCompileLimitHasBeenHit = (_, cb)-> cb(null, true) @CompileManager._checkIfAutoCompileLimitHasBeenHit = (isAutoCompile, compileGroup, cb)-> cb(null, true)
@CompileManager.compile @project_id, @user_id, {}, @callback @CompileManager.compile @project_id, @user_id, {}, @callback
it "should check the project has not been recently compiled", -> it "should check the project has not been recently compiled", ->
@ -84,7 +84,7 @@ describe "CompileManager", ->
describe "when the project has been recently compiled", -> describe "when the project has been recently compiled", ->
it "should return", (done)-> it "should return", (done)->
@CompileManager._checkIfAutoCompileLimitHasBeenHit = (_, cb)-> cb(null, true) @CompileManager._checkIfAutoCompileLimitHasBeenHit = (isAutoCompile, compileGroup, cb)-> cb(null, true)
@CompileManager._checkIfRecentlyCompiled = sinon.stub().callsArgWith(2, null, true) @CompileManager._checkIfRecentlyCompiled = sinon.stub().callsArgWith(2, null, true)
@CompileManager.compile @project_id, @user_id, {}, (err, status)-> @CompileManager.compile @project_id, @user_id, {}, (err, status)->
status.should.equal "too-recently-compiled" status.should.equal "too-recently-compiled"
@ -92,7 +92,7 @@ describe "CompileManager", ->
describe "should check the rate limit", -> describe "should check the rate limit", ->
it "should return", (done)-> it "should return", (done)->
@CompileManager._checkIfAutoCompileLimitHasBeenHit = sinon.stub().callsArgWith(1, null, false) @CompileManager._checkIfAutoCompileLimitHasBeenHit = sinon.stub().callsArgWith(2, null, false)
@CompileManager.compile @project_id, @user_id, {}, (err, status)-> @CompileManager.compile @project_id, @user_id, {}, (err, status)->
status.should.equal "autocompile-backoff" status.should.equal "autocompile-backoff"
done() done()
@ -222,14 +222,14 @@ describe "CompileManager", ->
describe "_checkIfAutoCompileLimitHasBeenHit", -> describe "_checkIfAutoCompileLimitHasBeenHit", ->
it "should be able to compile if it is not an autocompile", (done)-> it "should be able to compile if it is not an autocompile", (done)->
@ratelimiter.addCount.callsArgWith(1, null, true) @ratelimiter.addCount.callsArgWith(2, null, true)
@CompileManager._checkIfAutoCompileLimitHasBeenHit false, (err, canCompile)=> @CompileManager._checkIfAutoCompileLimitHasBeenHit false, "everyone", (err, canCompile)=>
canCompile.should.equal true canCompile.should.equal true
done() done()
it "should be able to compile if rate limit has remianing", (done)-> it "should be able to compile if rate limit has remianing", (done)->
@ratelimiter.addCount.callsArgWith(1, null, true) @ratelimiter.addCount.callsArgWith(1, null, true)
@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=> @CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
args = @ratelimiter.addCount.args[0][0] args = @ratelimiter.addCount.args[0][0]
args.throttle.should.equal 25 args.throttle.should.equal 25
args.subjectName.should.equal "everyone" args.subjectName.should.equal "everyone"
@ -240,13 +240,13 @@ describe "CompileManager", ->
it "should be not able to compile if rate limit has no remianing", (done)-> it "should be not able to compile if rate limit has no remianing", (done)->
@ratelimiter.addCount.callsArgWith(1, null, false) @ratelimiter.addCount.callsArgWith(1, null, false)
@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=> @CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
canCompile.should.equal false canCompile.should.equal false
done() done()
it "should return false if there is an error in the rate limit", (done)-> it "should return false if there is an error in the rate limit", (done)->
@ratelimiter.addCount.callsArgWith(1, "error") @ratelimiter.addCount.callsArgWith(1, "error")
@CompileManager._checkIfAutoCompileLimitHasBeenHit true, (err, canCompile)=> @CompileManager._checkIfAutoCompileLimitHasBeenHit true, "everyone", (err, canCompile)=>
canCompile.should.equal false canCompile.should.equal false
done() done()