github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-07 20:31:06 -05:00
Code Issues Projects Releases Packages Wiki Activity

add per-user routes for clearing cache and extend expiry methods

Browse source 
this adds separate functionality for clearing the cache (assets and
database) and the project compile directory for a specific user
This commit is contained in:
Brian Gough 2016-05-27 15:31:44 +01:00
parent df641549c4
commit 5367bc22e5
6 changed files with 59 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
services/clsi/app.coffee
View file

@ -65,7 +65,7 @@ app.get "/project/:project_id/status", CompileController.status
# Per-user containers
app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile
# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache
app.delete "/project/:project_id/user/:user_id", CompileController.clearCache
app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode
app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf

2
services/clsi/app/coffee/CompileController.coffee
View file

@ -45,7 +45,7 @@ module.exports = CompileController =
}
clearCache: (req, res, next = (error) ->) ->
ProjectPersistenceManager.clearProject req.params.project_id, (error) ->
ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) ->
return next(error) if error?
res.sendStatus(204) # No content

27
services/clsi/app/coffee/CompileManager.coffee
View file

@ -9,7 +9,9 @@ Metrics = require "./Metrics"
child_process = require "child_process"
DraftModeManager = require "./DraftModeManager"
fs = require("fs")
fse = require "fs-extra"
os = require("os")
async = require "async"
commandRunner = Settings.clsi?.commandRunner or "./CommandRunner"
logger.info commandRunner:commandRunner, "selecting command runner for clsi"
@ -76,12 +78,12 @@ module.exports = CompileManager =
OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) ->
callback null, newOutputFiles
clearProject: (project_id, _callback = (error) ->) ->
clearProject: (project_id, user_id, _callback = (error) ->) ->
callback = (error) ->
_callback(error)
_callback = () ->
compileDir = Path.join(Settings.path.compilesDir, project_id)
compileDir = getCompileDir(project_id, user_id)
CompileManager._checkDirectory compileDir, (err, exists) ->
return callback(err) if err?
@ -100,6 +102,27 @@ module.exports = CompileManager =
else
return callback(new Error("rm -r #{compileDir} failed: #{stderr}"))
_findAllDirs: (callback = (error, allDirs) ->) ->
root = Settings.path.compilesDir
fs.readdir root, (err, files) ->
return callback(err) if err?
allDirs = (Path.join(root, file) for file in files)
callback(null, allDirs)
clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) ->
now = Date.now()
# action for each directory
expireIfNeeded = (checkDir, cb) ->
fs.stat checkDir, (err, stats) ->
return cb() if err? # ignore errors checking directory
age = now - stats.mtime
hasExpired = (age > max_cache_age_ms)
if hasExpired then fse.remove(checkDir, cb) else cb()
# iterate over all project directories
CompileManager._findAllDirs (error, allDirs) ->
return callback() if error?
async.eachSeries allDirs, expireIfNeeded, callback
_checkDirectory: (compileDir, callback = (error, exists) ->) ->
fs.lstat compileDir, (err, stats) ->
if err?.code is 'ENOENT'

32
services/clsi/app/coffee/ProjectPersistenceManager.coffee
View file

@ -27,21 +27,30 @@ module.exports = ProjectPersistenceManager =
jobs = for project_id in (project_ids or [])
do (project_id) ->
(callback) ->
ProjectPersistenceManager.clearProject project_id, (err) ->
ProjectPersistenceManager.clearProjectFromCache project_id, (err) ->
if err?
logger.error err: err, project_id: project_id, "error clearing project"
callback()
async.series jobs, callback
clearProject: (project_id, callback = (error) ->) ->
logger.log project_id: project_id, "clearing project"
CompileManager.clearProject project_id, (error) ->
return callback(error) if error?
UrlCache.clearProject project_id, (error) ->
async.series jobs, (error) ->
return callback(error) if error?
ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
return callback(error) if error?
callback()
CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) ->
callback() # ignore any errors from deleting directories
clearProject: (project_id, user_id, callback = (error) ->) ->
logger.log project_id: project_id, "clearing project for user"
CompileManager.clearProject project_id, user_id, (error) ->
return callback(error) if error?
ProjectPersistenceManager.clearProjectFromCache project_id, (error) ->
return callback(error) if error?
callback()
clearProjectFromCache: (project_id, callback = (error) ->) ->
logger.log project_id: project_id, "clearing project from cache"
UrlCache.clearProject project_id, (error) ->
return callback(error) if error?
ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
return callback(error) if error?
callback()
_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
db.Project.destroy(where: {project_id: project_id})
@ -54,5 +63,4 @@ module.exports = ProjectPersistenceManager =
callback null, projects.map((project) -> project.project_id)
).error callback
logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout"

10
services/clsi/test/unit/coffee/CompileManagerTests.coffee
View file

@ -105,12 +105,12 @@ describe "CompileManager", ->
@proc.stdout = new EventEmitter()
@proc.stderr = new EventEmitter()
@child_process.spawn = sinon.stub().returns(@proc)
@CompileManager.clearProject @project_id, @callback
@CompileManager.clearProject @project_id, @user_id, @callback
@proc.emit "close", 0
it "should remove the project directory", ->
@child_process.spawn
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"])
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"])
.should.equal true
it "should call the callback", ->
@ -124,13 +124,13 @@ describe "CompileManager", ->
@proc.stdout = new EventEmitter()
@proc.stderr = new EventEmitter()
@child_process.spawn = sinon.stub().returns(@proc)
@CompileManager.clearProject @project_id, @callback
@CompileManager.clearProject @project_id, @user_id, @callback
@proc.stderr.emit "data", @error = "oops"
@proc.emit "close", 1
it "should remove the project directory", ->
@child_process.spawn
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"])
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"])
.should.equal true
it "should call the callback with an error from the stderr", ->
@ -138,7 +138,7 @@ describe "CompileManager", ->
.calledWith(new Error())
.should.equal true
@callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id} failed: #{@error}"
@callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id}-#{@user_id} failed: #{@error}"
describe "syncing", ->
beforeEach ->

12
services/clsi/test/unit/coffee/ProjectPersistenceManagerTests.coffee
View file

@ -13,6 +13,7 @@ describe "ProjectPersistenceManager", ->
"./db": @db = {}
@callback = sinon.stub()
@project_id = "project-id-123"
@user_id = "1234"
describe "clearExpiredProjects", ->
beforeEach ->
@ -21,12 +22,13 @@ describe "ProjectPersistenceManager", ->
"project-id-2"
]
@ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids)
@ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1)
@ProjectPersistenceManager.clearProjectFromCache = sinon.stub().callsArg(1)
@CompileManager.clearExpiredProjects = sinon.stub().callsArg(1)
@ProjectPersistenceManager.clearExpiredProjects @callback
it "should clear each expired project", ->
for project_id in @project_ids
@ProjectPersistenceManager.clearProject
@ProjectPersistenceManager.clearProjectFromCache
.calledWith(project_id)
.should.equal true
@ -37,8 +39,8 @@ describe "ProjectPersistenceManager", ->
beforeEach ->
@ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1)
@UrlCache.clearProject = sinon.stub().callsArg(1)
@CompileManager.clearProject = sinon.stub().callsArg(1)
@ProjectPersistenceManager.clearProject @project_id, @callback
@CompileManager.clearProject = sinon.stub().callsArg(2)
@ProjectPersistenceManager.clearProject @project_id, @user_id, @callback
it "should clear the project from the database", ->
@ProjectPersistenceManager._clearProjectFromDatabase
@ -52,7 +54,7 @@ describe "ProjectPersistenceManager", ->
it "should clear the project compile folder", ->
@CompileManager.clearProject
.calledWith(@project_id)
.calledWith(@project_id, @user_id)
.should.equal true
it "should call the callback", ->