Merge branch 'per-user-containers-part-2'

Conflicts:
	app/coffee/CompileController.coffee
This commit is contained in:
Brian Gough 2016-06-13 09:33:41 +01:00
commit fb12d04282
7 changed files with 126 additions and 67 deletions

View file

@ -42,8 +42,8 @@ app.param 'project_id', (req, res, next, project_id) ->
else
next new Error("invalid project id")
app.param 'user_id', (req, res, next, project_id) ->
if project_id?.match /^[a-zA-Z0-9_-]+$/
app.param 'user_id', (req, res, next, user_id) ->
if user_id?.match /^[0-9a-f]{24}$/
next()
else
next new Error("invalid user id")
@ -63,6 +63,14 @@ app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf
app.get "/project/:project_id/wordcount", CompileController.wordcount
app.get "/project/:project_id/status", CompileController.status
# Per-user containers
app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile
app.delete "/project/:project_id/user/:user_id", CompileController.clearCache
app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode
app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf
app.get "/project/:project_id/user/:user_id/wordcount", CompileController.wordcount
ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks"
# create a static server which does not allow access to any symlinks

View file

@ -11,6 +11,7 @@ module.exports = CompileController =
RequestParser.parse req.body, (error, request) ->
return next(error) if error?
request.project_id = req.params.project_id
request.user_id = req.params.user_id if req.params.user_id?
ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) ->
return next(error) if error?
CompileManager.doCompile request, (error, outputFiles = []) ->
@ -35,6 +36,7 @@ module.exports = CompileController =
outputFiles: outputFiles.map (file) ->
url:
"#{Settings.apis.clsi.url}/project/#{request.project_id}" +
(if request.user_id? then "/user/#{request.user_id}" else "") +
(if file.build? then "/build/#{file.build}" else "") +
"/output/#{file.path}"
path: file.path
@ -43,7 +45,7 @@ module.exports = CompileController =
}
clearCache: (req, res, next = (error) ->) ->
ProjectPersistenceManager.clearProject req.params.project_id, (error) ->
ProjectPersistenceManager.clearProject req.params.project_id, req.params.user_id, (error) ->
return next(error) if error?
res.sendStatus(204) # No content
@ -52,8 +54,9 @@ module.exports = CompileController =
line = parseInt(req.query.line, 10)
column = parseInt(req.query.column, 10)
project_id = req.params.project_id
user_id = req.params.user_id
CompileManager.syncFromCode project_id, file, line, column, (error, pdfPositions) ->
CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) ->
return next(error) if error?
res.send JSON.stringify {
pdf: pdfPositions
@ -64,8 +67,9 @@ module.exports = CompileController =
h = parseFloat(req.query.h)
v = parseFloat(req.query.v)
project_id = req.params.project_id
user_id = req.params.user_id
CompileManager.syncFromPdf project_id, page, h, v, (error, codePositions) ->
CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) ->
return next(error) if error?
res.send JSON.stringify {
code: codePositions
@ -74,10 +78,11 @@ module.exports = CompileController =
wordcount: (req, res, next = (error) ->) ->
file = req.query.file || "main.tex"
project_id = req.params.project_id
user_id = req.params.user_id
image = req.query.image
logger.log {image, file, project_id}, "word count request"
CompileManager.wordcount project_id, file, image, (error, result) ->
CompileManager.wordcount project_id, user_id, file, image, (error, result) ->
return next(error) if error?
res.send JSON.stringify {
texcount: result

View file

@ -9,23 +9,31 @@ Metrics = require "./Metrics"
child_process = require "child_process"
DraftModeManager = require "./DraftModeManager"
fs = require("fs")
fse = require "fs-extra"
os = require("os")
async = require "async"
commandRunner = Settings.clsi?.commandRunner or "./CommandRunner"
logger.info commandRunner:commandRunner, "selecting command runner for clsi"
CommandRunner = require(commandRunner)
getCompileName = (project_id, user_id) ->
if user_id? then "#{project_id}-#{user_id}" else project_id
getCompileDir = (project_id, user_id) ->
Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id))
module.exports = CompileManager =
doCompile: (request, callback = (error, outputFiles) ->) ->
compileDir = Path.join(Settings.path.compilesDir, request.project_id)
compileDir = getCompileDir(request.project_id, request.user_id)
timer = new Metrics.Timer("write-to-disk")
logger.log project_id: request.project_id, "starting doCompile"
logger.log project_id: request.project_id, user_id: request.user_id, "starting compile"
ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) ->
if error?
logger.err err:error, project_id: request.project_id, "error writing resources to disk"
logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk"
return callback(error)
logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "written files to disk"
logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk"
timer.done()
injectDraftModeIfRequired = (callback) ->
@ -42,7 +50,8 @@ module.exports = CompileManager =
tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test
Metrics.inc("compiles")
Metrics.inc("compiles-with-image.#{tag}")
LatexRunner.runLatex request.project_id, {
compileName = getCompileName(request.project_id, request.user_id)
LatexRunner.runLatex compileName, {
directory: compileDir
mainFile: request.rootResourcePath
compiler: request.compiler
@ -58,7 +67,7 @@ module.exports = CompileManager =
loadavg = os.loadavg?()
Metrics.gauge("load-avg", loadavg[0]) if loadavg?
ts = timer.done()
logger.log {project_id: request.project_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile"
logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile"
if stats?["latex-runs"] > 0
Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"])
if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0
@ -69,12 +78,12 @@ module.exports = CompileManager =
OutputCacheManager.saveOutputFiles outputFiles, compileDir, (error, newOutputFiles) ->
callback null, newOutputFiles
clearProject: (project_id, _callback = (error) ->) ->
clearProject: (project_id, user_id, _callback = (error) ->) ->
callback = (error) ->
_callback(error)
_callback = () ->
compileDir = Path.join(Settings.path.compilesDir, project_id)
compileDir = getCompileDir(project_id, user_id)
CompileManager._checkDirectory compileDir, (err, exists) ->
return callback(err) if err?
@ -93,6 +102,27 @@ module.exports = CompileManager =
else
return callback(new Error("rm -r #{compileDir} failed: #{stderr}"))
_findAllDirs: (callback = (error, allDirs) ->) ->
root = Settings.path.compilesDir
fs.readdir root, (err, files) ->
return callback(err) if err?
allDirs = (Path.join(root, file) for file in files)
callback(null, allDirs)
clearExpiredProjects: (max_cache_age_ms, callback = (error) ->) ->
now = Date.now()
# action for each directory
expireIfNeeded = (checkDir, cb) ->
fs.stat checkDir, (err, stats) ->
return cb() if err? # ignore errors checking directory
age = now - stats.mtime
hasExpired = (age > max_cache_age_ms)
if hasExpired then fse.remove(checkDir, cb) else cb()
# iterate over all project directories
CompileManager._findAllDirs (error, allDirs) ->
return callback() if error?
async.eachSeries allDirs, expireIfNeeded, callback
_checkDirectory: (compileDir, callback = (error, exists) ->) ->
fs.lstat compileDir, (err, stats) ->
if err?.code is 'ENOENT'
@ -106,24 +136,28 @@ module.exports = CompileManager =
else
callback(null, true) # directory exists
syncFromCode: (project_id, file_name, line, column, callback = (error, pdfPositions) ->) ->
syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) ->
# If LaTeX was run in a virtual environment, the file path that synctex expects
# might not match the file path on the host. The .synctex.gz file however, will be accessed
# wherever it is on the host.
base_dir = Settings.path.synctexBaseDir(project_id)
compileName = getCompileName(project_id, user_id)
base_dir = Settings.path.synctexBaseDir(compileName)
file_path = base_dir + "/" + file_name
synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf")
compileDir = getCompileDir(project_id, user_id)
synctex_path = Path.join(compileDir, "output.pdf")
CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) ->
return callback(error) if error?
logger.log project_id: project_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output"
callback null, CompileManager._parseSynctexFromCodeOutput(stdout)
syncFromPdf: (project_id, page, h, v, callback = (error, filePositions) ->) ->
base_dir = Settings.path.synctexBaseDir(project_id)
synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf")
syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) ->
compileName = getCompileName(project_id, user_id)
base_dir = Settings.path.synctexBaseDir(compileName)
compileDir = getCompileDir(project_id, user_id)
synctex_path = Path.join(compileDir, "output.pdf")
CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) ->
return callback(error) if error?
logger.log project_id: project_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output"
callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir)
_runSynctex: (args, callback = (error, stdout) ->) ->
@ -162,19 +196,20 @@ module.exports = CompileManager =
}
return results
wordcount: (project_id, file_name, image, callback = (error, pdfPositions) ->) ->
logger.log project_id:project_id, file_name:file_name, image:image, "running wordcount"
wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) ->
logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount"
file_path = "$COMPILE_DIR/" + file_name
command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"]
directory = Path.join(Settings.path.compilesDir, project_id)
directory = getCompileDir(project_id, user_id)
timeout = 10 * 1000
compileName = getCompileName(project_id, user_id)
CommandRunner.run project_id, command, directory, image, timeout, (error) ->
CommandRunner.run compileName, command, directory, image, timeout, (error) ->
return callback(error) if error?
try
stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8")
catch err
logger.err err:err, command:command, directory:directory, project_id:project_id, "error reading word count output"
logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output"
return callback(err)
callback null, CompileManager._parseWordcountFromOutput(stdout)

View file

@ -27,21 +27,30 @@ module.exports = ProjectPersistenceManager =
jobs = for project_id in (project_ids or [])
do (project_id) ->
(callback) ->
ProjectPersistenceManager.clearProject project_id, (err) ->
ProjectPersistenceManager.clearProjectFromCache project_id, (err) ->
if err?
logger.error err: err, project_id: project_id, "error clearing project"
callback()
async.series jobs, callback
clearProject: (project_id, callback = (error) ->) ->
logger.log project_id: project_id, "clearing project"
CompileManager.clearProject project_id, (error) ->
return callback(error) if error?
UrlCache.clearProject project_id, (error) ->
async.series jobs, (error) ->
return callback(error) if error?
ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
return callback(error) if error?
callback()
CompileManager.clearExpiredProjects ProjectPersistenceManager.EXPIRY_TIMEOUT, (error) ->
callback() # ignore any errors from deleting directories
clearProject: (project_id, user_id, callback = (error) ->) ->
logger.log project_id: project_id, user_id:user_id, "clearing project for user"
CompileManager.clearProject project_id, user_id, (error) ->
return callback(error) if error?
ProjectPersistenceManager.clearProjectFromCache project_id, (error) ->
return callback(error) if error?
callback()
clearProjectFromCache: (project_id, callback = (error) ->) ->
logger.log project_id: project_id, "clearing project from cache"
UrlCache.clearProject project_id, (error) ->
return callback(error) if error?
ProjectPersistenceManager._clearProjectFromDatabase project_id, (error) ->
return callback(error) if error?
callback()
_clearProjectFromDatabase: (project_id, callback = (error) ->) ->
db.Project.destroy(where: {project_id: project_id})
@ -54,5 +63,4 @@ module.exports = ProjectPersistenceManager =
callback null, projects.map((project) -> project.project_id)
).error callback
logger.log {EXPIRY_TIMEOUT: ProjectPersistenceManager.EXPIRY_TIMEOUT}, "project assets kept timeout"

View file

@ -146,12 +146,12 @@ describe "CompileController", ->
column: @column.toString()
@res.send = sinon.stub()
@CompileManager.syncFromCode = sinon.stub().callsArgWith(4, null, @pdfPositions = ["mock-positions"])
@CompileManager.syncFromCode = sinon.stub().callsArgWith(5, null, @pdfPositions = ["mock-positions"])
@CompileController.syncFromCode @req, @res, @next
it "should find the corresponding location in the PDF", ->
@CompileManager.syncFromCode
.calledWith(@project_id, @file, @line, @column)
.calledWith(@project_id, undefined, @file, @line, @column)
.should.equal true
it "should return the positions", ->
@ -175,12 +175,12 @@ describe "CompileController", ->
v: @v.toString()
@res.send = sinon.stub()
@CompileManager.syncFromPdf = sinon.stub().callsArgWith(4, null, @codePositions = ["mock-positions"])
@CompileManager.syncFromPdf = sinon.stub().callsArgWith(5, null, @codePositions = ["mock-positions"])
@CompileController.syncFromPdf @req, @res, @next
it "should find the corresponding location in the code", ->
@CompileManager.syncFromPdf
.calledWith(@project_id, @page, @h, @v)
.calledWith(@project_id, undefined, @page, @h, @v)
.should.equal true
it "should return the positions", ->
@ -201,12 +201,12 @@ describe "CompileController", ->
image: @image = "example.com/image"
@res.send = sinon.stub()
@CompileManager.wordcount = sinon.stub().callsArgWith(3, null, @texcount = ["mock-texcount"])
@CompileManager.wordcount = sinon.stub().callsArgWith(4, null, @texcount = ["mock-texcount"])
@CompileController.wordcount @req, @res, @next
it "should return the word count of a file", ->
@CompileManager.wordcount
.calledWith(@project_id, @file, @image)
.calledWith(@project_id, undefined, @file, @image)
.should.equal true
it "should return the texcount info", ->

View file

@ -43,11 +43,12 @@ describe "CompileManager", ->
resources: @resources = "mock-resources"
rootResourcePath: @rootResourcePath = "main.tex"
project_id: @project_id = "project-id-123"
user_id: @user_id = "1234"
compiler: @compiler = "pdflatex"
timeout: @timeout = 42000
imageName: @image = "example.com/image"
@Settings.compileDir = "compiles"
@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}"
@compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
@ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3)
@LatexRunner.runLatex = sinon.stub().callsArg(2)
@OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files)
@ -65,7 +66,7 @@ describe "CompileManager", ->
it "should run LaTeX", ->
@LatexRunner.runLatex
.calledWith(@project_id, {
.calledWith("#{@project_id}-#{@user_id}", {
directory: @compileDir
mainFile: @rootResourcePath
compiler: @compiler
@ -104,12 +105,12 @@ describe "CompileManager", ->
@proc.stdout = new EventEmitter()
@proc.stderr = new EventEmitter()
@child_process.spawn = sinon.stub().returns(@proc)
@CompileManager.clearProject @project_id, @callback
@CompileManager.clearProject @project_id, @user_id, @callback
@proc.emit "close", 0
it "should remove the project directory", ->
@child_process.spawn
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"])
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"])
.should.equal true
it "should call the callback", ->
@ -123,13 +124,13 @@ describe "CompileManager", ->
@proc.stdout = new EventEmitter()
@proc.stderr = new EventEmitter()
@child_process.spawn = sinon.stub().returns(@proc)
@CompileManager.clearProject @project_id, @callback
@CompileManager.clearProject @project_id, @user_id, @callback
@proc.stderr.emit "data", @error = "oops"
@proc.emit "close", 1
it "should remove the project directory", ->
@child_process.spawn
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}"])
.calledWith("rm", ["-r", "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"])
.should.equal true
it "should call the callback with an error from the stderr", ->
@ -137,7 +138,7 @@ describe "CompileManager", ->
.calledWith(new Error())
.should.equal true
@callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id} failed: #{@error}"
@callback.args[0][0].message.should.equal "rm -r #{@Settings.path.compilesDir}/#{@project_id}-#{@user_id} failed: #{@error}"
describe "syncing", ->
beforeEach ->
@ -150,17 +151,17 @@ describe "CompileManager", ->
@column = 3
@file_name = "main.tex"
@child_process.execFile = sinon.stub()
@Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}"
@Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
describe "syncFromCode", ->
beforeEach ->
@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n", "")
@CompileManager.syncFromCode @project_id, @file_name, @line, @column, @callback
@CompileManager.syncFromCode @project_id, @user_id, @file_name, @line, @column, @callback
it "should execute the synctex binary", ->
bin_path = Path.resolve(__dirname + "/../../../bin/synctex")
synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf"
file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}"
synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf"
file_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}"
@child_process.execFile
.calledWith(bin_path, ["code", synctex_path, file_path, @line, @column], timeout: 10000)
.should.equal true
@ -178,12 +179,12 @@ describe "CompileManager", ->
describe "syncFromPdf", ->
beforeEach ->
@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}\t#{@line}\t#{@column}\n", "")
@CompileManager.syncFromPdf @project_id, @page, @h, @v, @callback
@child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n", "")
@CompileManager.syncFromPdf @project_id, @user_id, @page, @h, @v, @callback
it "should execute the synctex binary", ->
bin_path = Path.resolve(__dirname + "/../../../bin/synctex")
synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf"
synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf"
@child_process.execFile
.calledWith(bin_path, ["pdf", synctex_path, @page, @h, @v], timeout: 10000)
.should.equal true
@ -209,15 +210,15 @@ describe "CompileManager", ->
@Settings.path.compilesDir = "/local/compile/directory"
@image = "example.com/image"
@CompileManager.wordcount @project_id, @file_name, @image, @callback
@CompileManager.wordcount @project_id, @user_id, @file_name, @image, @callback
it "should run the texcount command", ->
@directory = "#{@Settings.path.compilesDir}/#{@project_id}"
@directory = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}"
@file_path = "$COMPILE_DIR/#{@file_name}"
@command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"]
@CommandRunner.run
.calledWith(@project_id, @command, @directory, @image, @timeout)
.calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout)
.should.equal true
it "should call the callback with the parsed output", ->

View file

@ -13,6 +13,7 @@ describe "ProjectPersistenceManager", ->
"./db": @db = {}
@callback = sinon.stub()
@project_id = "project-id-123"
@user_id = "1234"
describe "clearExpiredProjects", ->
beforeEach ->
@ -21,12 +22,13 @@ describe "ProjectPersistenceManager", ->
"project-id-2"
]
@ProjectPersistenceManager._findExpiredProjectIds = sinon.stub().callsArgWith(0, null, @project_ids)
@ProjectPersistenceManager.clearProject = sinon.stub().callsArg(1)
@ProjectPersistenceManager.clearProjectFromCache = sinon.stub().callsArg(1)
@CompileManager.clearExpiredProjects = sinon.stub().callsArg(1)
@ProjectPersistenceManager.clearExpiredProjects @callback
it "should clear each expired project", ->
for project_id in @project_ids
@ProjectPersistenceManager.clearProject
@ProjectPersistenceManager.clearProjectFromCache
.calledWith(project_id)
.should.equal true
@ -37,8 +39,8 @@ describe "ProjectPersistenceManager", ->
beforeEach ->
@ProjectPersistenceManager._clearProjectFromDatabase = sinon.stub().callsArg(1)
@UrlCache.clearProject = sinon.stub().callsArg(1)
@CompileManager.clearProject = sinon.stub().callsArg(1)
@ProjectPersistenceManager.clearProject @project_id, @callback
@CompileManager.clearProject = sinon.stub().callsArg(2)
@ProjectPersistenceManager.clearProject @project_id, @user_id, @callback
it "should clear the project from the database", ->
@ProjectPersistenceManager._clearProjectFromDatabase
@ -52,7 +54,7 @@ describe "ProjectPersistenceManager", ->
it "should clear the project compile folder", ->
@CompileManager.clearProject
.calledWith(@project_id)
.calledWith(@project_id, @user_id)
.should.equal true
it "should call the callback", ->