From df641549c4e8828955e75dcb77855a1ce4cf9956 Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Fri, 27 May 2016 15:29:26 +0100 Subject: [PATCH] add per-user routes and methods --- services/clsi/app.coffee | 12 ++++- .../clsi/app/coffee/CompileController.coffee | 11 ++-- .../clsi/app/coffee/CompileManager.coffee | 50 ++++++++++++------- .../unit/coffee/CompileControllerTests.coffee | 12 ++--- .../unit/coffee/CompileManagerTests.coffee | 25 +++++----- 5 files changed, 68 insertions(+), 42 deletions(-) diff --git a/services/clsi/app.coffee b/services/clsi/app.coffee index 9f54f4a016..cfe8ebccdc 100644 --- a/services/clsi/app.coffee +++ b/services/clsi/app.coffee @@ -42,8 +42,8 @@ app.param 'project_id', (req, res, next, project_id) -> else next new Error("invalid project id") -app.param 'user_id', (req, res, next, project_id) -> - if project_id?.match /^[a-zA-Z0-9_-]+$/ +app.param 'user_id', (req, res, next, user_id) -> + if user_id?.match /^[0-9a-f]{24}$/ next() else next new Error("invalid user id") @@ -63,6 +63,14 @@ app.get "/project/:project_id/sync/pdf", CompileController.syncFromPdf app.get "/project/:project_id/wordcount", CompileController.wordcount app.get "/project/:project_id/status", CompileController.status +# Per-user containers +app.post "/project/:project_id/user/:user_id/compile", bodyParser.json(limit: "5mb"), CompileController.compile +# app.delete "/project/:project_id/user/:user_id", CompileController.clearCache + +app.get "/project/:project_id/user/:user_id/sync/code", CompileController.syncFromCode +app.get "/project/:project_id/user/:user_id/sync/pdf", CompileController.syncFromPdf +app.get "/project/:project_id/user/:user_id/wordcount", CompileController.wordcount + ForbidSymlinks = require "./app/js/StaticServerForbidSymlinks" # create a static server which does not allow access to any symlinks diff --git a/services/clsi/app/coffee/CompileController.coffee b/services/clsi/app/coffee/CompileController.coffee index 478d91bfa7..7e1f07890a 100644 --- a/services/clsi/app/coffee/CompileController.coffee +++ b/services/clsi/app/coffee/CompileController.coffee @@ -11,6 +11,7 @@ module.exports = CompileController = RequestParser.parse req.body, (error, request) -> return next(error) if error? request.project_id = req.params.project_id + request.user_id = req.params.user_id if req.params.user_id? ProjectPersistenceManager.markProjectAsJustAccessed request.project_id, (error) -> return next(error) if error? CompileManager.doCompile request, (error, outputFiles = []) -> @@ -35,6 +36,7 @@ module.exports = CompileController = outputFiles: outputFiles.map (file) -> url: "#{Settings.apis.clsi.url}/project/#{request.project_id}" + + (if request.user_id? then "/user/#{request.user_id}" else "") + (if file.build? then "/build/#{file.build}" else "") + "/output/#{file.path}" path: file.path @@ -52,8 +54,9 @@ module.exports = CompileController = line = parseInt(req.query.line, 10) column = parseInt(req.query.column, 10) project_id = req.params.project_id + user_id = req.params.user_id - CompileManager.syncFromCode project_id, file, line, column, (error, pdfPositions) -> + CompileManager.syncFromCode project_id, user_id, file, line, column, (error, pdfPositions) -> return next(error) if error? res.send JSON.stringify { pdf: pdfPositions @@ -64,8 +67,9 @@ module.exports = CompileController = h = parseFloat(req.query.h) v = parseFloat(req.query.v) project_id = req.params.project_id + user_id = req.params.user_id - CompileManager.syncFromPdf project_id, page, h, v, (error, codePositions) -> + CompileManager.syncFromPdf project_id, user_id, page, h, v, (error, codePositions) -> return next(error) if error? res.send JSON.stringify { code: codePositions @@ -74,10 +78,11 @@ module.exports = CompileController = wordcount: (req, res, next = (error) ->) -> file = req.query.file || "main.tex" project_id = req.params.project_id + user_id = req.params.user_id image = req.query.image logger.log {image, file, project_id}, "word count request" - CompileManager.wordcount project_id, file, image, (error, result) -> + CompileManager.wordcount project_id, user_id, file, image, (error, result) -> return next(error) if error? res.send JSON.stringify { texcount: result diff --git a/services/clsi/app/coffee/CompileManager.coffee b/services/clsi/app/coffee/CompileManager.coffee index 41377abaf8..cd332a4f9f 100644 --- a/services/clsi/app/coffee/CompileManager.coffee +++ b/services/clsi/app/coffee/CompileManager.coffee @@ -15,17 +15,23 @@ commandRunner = Settings.clsi?.commandRunner or "./CommandRunner" logger.info commandRunner:commandRunner, "selecting command runner for clsi" CommandRunner = require(commandRunner) +getCompileName = (project_id, user_id) -> + if user_id? then "#{project_id}-#{user_id}" else project_id + +getCompileDir = (project_id, user_id) -> + Path.join(Settings.path.compilesDir, getCompileName(project_id, user_id)) + module.exports = CompileManager = doCompile: (request, callback = (error, outputFiles) ->) -> - compileDir = Path.join(Settings.path.compilesDir, request.project_id) + compileDir = getCompileDir(request.project_id, request.user_id) timer = new Metrics.Timer("write-to-disk") - logger.log project_id: request.project_id, "starting doCompile" + logger.log project_id: request.project_id, user_id: request.user_id, "starting compile" ResourceWriter.syncResourcesToDisk request.project_id, request.resources, compileDir, (error) -> if error? - logger.err err:error, project_id: request.project_id, "error writing resources to disk" + logger.err err:error, project_id: request.project_id, user_id: request.user_id, "error writing resources to disk" return callback(error) - logger.log project_id: request.project_id, time_taken: Date.now() - timer.start, "written files to disk" + logger.log project_id: request.project_id, user_id: request.user_id, time_taken: Date.now() - timer.start, "written files to disk" timer.done() injectDraftModeIfRequired = (callback) -> @@ -42,7 +48,8 @@ module.exports = CompileManager = tag = "other" if not request.project_id.match(/^[0-9a-f]{24}$/) # exclude smoke test Metrics.inc("compiles") Metrics.inc("compiles-with-image.#{tag}") - LatexRunner.runLatex request.project_id, { + compileName = getCompileName(request.project_id, request.user_id) + LatexRunner.runLatex compileName, { directory: compileDir mainFile: request.rootResourcePath compiler: request.compiler @@ -58,7 +65,7 @@ module.exports = CompileManager = loadavg = os.loadavg?() Metrics.gauge("load-avg", loadavg[0]) if loadavg? ts = timer.done() - logger.log {project_id: request.project_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" + logger.log {project_id: request.project_id, user_id: request.user_id, time_taken: ts, stats:stats, timings:timings, loadavg:loadavg}, "done compile" if stats?["latex-runs"] > 0 Metrics.timing("run-compile-per-pass", ts / stats["latex-runs"]) if stats?["latex-runs"] > 0 and timings?["cpu-time"] > 0 @@ -106,24 +113,28 @@ module.exports = CompileManager = else callback(null, true) # directory exists - syncFromCode: (project_id, file_name, line, column, callback = (error, pdfPositions) ->) -> + syncFromCode: (project_id, user_id, file_name, line, column, callback = (error, pdfPositions) ->) -> # If LaTeX was run in a virtual environment, the file path that synctex expects # might not match the file path on the host. The .synctex.gz file however, will be accessed # wherever it is on the host. - base_dir = Settings.path.synctexBaseDir(project_id) + compileName = getCompileName(project_id, user_id) + base_dir = Settings.path.synctexBaseDir(compileName) file_path = base_dir + "/" + file_name - synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf") + compileDir = getCompileDir(project_id, user_id) + synctex_path = Path.join(compileDir, "output.pdf") CompileManager._runSynctex ["code", synctex_path, file_path, line, column], (error, stdout) -> return callback(error) if error? - logger.log project_id: project_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output" + logger.log project_id: project_id, user_id:user_id, file_name: file_name, line: line, column: column, stdout: stdout, "synctex code output" callback null, CompileManager._parseSynctexFromCodeOutput(stdout) - syncFromPdf: (project_id, page, h, v, callback = (error, filePositions) ->) -> - base_dir = Settings.path.synctexBaseDir(project_id) - synctex_path = Path.join(Settings.path.compilesDir, project_id, "output.pdf") + syncFromPdf: (project_id, user_id, page, h, v, callback = (error, filePositions) ->) -> + compileName = getCompileName(project_id, user_id) + base_dir = Settings.path.synctexBaseDir(compileName) + compileDir = getCompileDir(project_id, user_id) + synctex_path = Path.join(compileDir, "output.pdf") CompileManager._runSynctex ["pdf", synctex_path, page, h, v], (error, stdout) -> return callback(error) if error? - logger.log project_id: project_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" + logger.log project_id: project_id, user_id:user_id, page: page, h: h, v:v, stdout: stdout, "synctex pdf output" callback null, CompileManager._parseSynctexFromPdfOutput(stdout, base_dir) _runSynctex: (args, callback = (error, stdout) ->) -> @@ -162,19 +173,20 @@ module.exports = CompileManager = } return results - wordcount: (project_id, file_name, image, callback = (error, pdfPositions) ->) -> - logger.log project_id:project_id, file_name:file_name, image:image, "running wordcount" + wordcount: (project_id, user_id, file_name, image, callback = (error, pdfPositions) ->) -> + logger.log project_id:project_id, user_id:user_id, file_name:file_name, image:image, "running wordcount" file_path = "$COMPILE_DIR/" + file_name command = [ "texcount", '-inc', file_path, "-out=" + file_path + ".wc"] - directory = Path.join(Settings.path.compilesDir, project_id) + directory = getCompileDir(project_id, user_id) timeout = 10 * 1000 + compileName = getCompileName(project_id, user_id) - CommandRunner.run project_id, command, directory, image, timeout, (error) -> + CommandRunner.run compileName, command, directory, image, timeout, (error) -> return callback(error) if error? try stdout = fs.readFileSync(directory + "/" + file_name + ".wc", "utf-8") catch err - logger.err err:err, command:command, directory:directory, project_id:project_id, "error reading word count output" + logger.err err:err, command:command, directory:directory, project_id:project_id, user_id:user_id, "error reading word count output" return callback(err) callback null, CompileManager._parseWordcountFromOutput(stdout) diff --git a/services/clsi/test/unit/coffee/CompileControllerTests.coffee b/services/clsi/test/unit/coffee/CompileControllerTests.coffee index 7693799e4c..1fc6a99bd5 100644 --- a/services/clsi/test/unit/coffee/CompileControllerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileControllerTests.coffee @@ -146,12 +146,12 @@ describe "CompileController", -> column: @column.toString() @res.send = sinon.stub() - @CompileManager.syncFromCode = sinon.stub().callsArgWith(4, null, @pdfPositions = ["mock-positions"]) + @CompileManager.syncFromCode = sinon.stub().callsArgWith(5, null, @pdfPositions = ["mock-positions"]) @CompileController.syncFromCode @req, @res, @next it "should find the corresponding location in the PDF", -> @CompileManager.syncFromCode - .calledWith(@project_id, @file, @line, @column) + .calledWith(@project_id, undefined, @file, @line, @column) .should.equal true it "should return the positions", -> @@ -175,12 +175,12 @@ describe "CompileController", -> v: @v.toString() @res.send = sinon.stub() - @CompileManager.syncFromPdf = sinon.stub().callsArgWith(4, null, @codePositions = ["mock-positions"]) + @CompileManager.syncFromPdf = sinon.stub().callsArgWith(5, null, @codePositions = ["mock-positions"]) @CompileController.syncFromPdf @req, @res, @next it "should find the corresponding location in the code", -> @CompileManager.syncFromPdf - .calledWith(@project_id, @page, @h, @v) + .calledWith(@project_id, undefined, @page, @h, @v) .should.equal true it "should return the positions", -> @@ -201,12 +201,12 @@ describe "CompileController", -> image: @image = "example.com/image" @res.send = sinon.stub() - @CompileManager.wordcount = sinon.stub().callsArgWith(3, null, @texcount = ["mock-texcount"]) + @CompileManager.wordcount = sinon.stub().callsArgWith(4, null, @texcount = ["mock-texcount"]) @CompileController.wordcount @req, @res, @next it "should return the word count of a file", -> @CompileManager.wordcount - .calledWith(@project_id, @file, @image) + .calledWith(@project_id, undefined, @file, @image) .should.equal true it "should return the texcount info", -> diff --git a/services/clsi/test/unit/coffee/CompileManagerTests.coffee b/services/clsi/test/unit/coffee/CompileManagerTests.coffee index 55f5cc5200..0ba62b95c1 100644 --- a/services/clsi/test/unit/coffee/CompileManagerTests.coffee +++ b/services/clsi/test/unit/coffee/CompileManagerTests.coffee @@ -43,11 +43,12 @@ describe "CompileManager", -> resources: @resources = "mock-resources" rootResourcePath: @rootResourcePath = "main.tex" project_id: @project_id = "project-id-123" + user_id: @user_id = "1234" compiler: @compiler = "pdflatex" timeout: @timeout = 42000 imageName: @image = "example.com/image" @Settings.compileDir = "compiles" - @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}" + @compileDir = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @ResourceWriter.syncResourcesToDisk = sinon.stub().callsArg(3) @LatexRunner.runLatex = sinon.stub().callsArg(2) @OutputFileFinder.findOutputFiles = sinon.stub().callsArgWith(2, null, @output_files) @@ -65,7 +66,7 @@ describe "CompileManager", -> it "should run LaTeX", -> @LatexRunner.runLatex - .calledWith(@project_id, { + .calledWith("#{@project_id}-#{@user_id}", { directory: @compileDir mainFile: @rootResourcePath compiler: @compiler @@ -150,17 +151,17 @@ describe "CompileManager", -> @column = 3 @file_name = "main.tex" @child_process.execFile = sinon.stub() - @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}" + @Settings.path.synctexBaseDir = (project_id) => "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" describe "syncFromCode", -> beforeEach -> @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@page}\t#{@h}\t#{@v}\t#{@width}\t#{@height}\n", "") - @CompileManager.syncFromCode @project_id, @file_name, @line, @column, @callback + @CompileManager.syncFromCode @project_id, @user_id, @file_name, @line, @column, @callback it "should execute the synctex binary", -> bin_path = Path.resolve(__dirname + "/../../../bin/synctex") - synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf" - file_path = "#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}" + synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf" + file_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}" @child_process.execFile .calledWith(bin_path, ["code", synctex_path, file_path, @line, @column], timeout: 10000) .should.equal true @@ -178,12 +179,12 @@ describe "CompileManager", -> describe "syncFromPdf", -> beforeEach -> - @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}/#{@file_name}\t#{@line}\t#{@column}\n", "") - @CompileManager.syncFromPdf @project_id, @page, @h, @v, @callback + @child_process.execFile.callsArgWith(3, null, @stdout = "NODE\t#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/#{@file_name}\t#{@line}\t#{@column}\n", "") + @CompileManager.syncFromPdf @project_id, @user_id, @page, @h, @v, @callback it "should execute the synctex binary", -> bin_path = Path.resolve(__dirname + "/../../../bin/synctex") - synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}/output.pdf" + synctex_path = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}/output.pdf" @child_process.execFile .calledWith(bin_path, ["pdf", synctex_path, @page, @h, @v], timeout: 10000) .should.equal true @@ -209,15 +210,15 @@ describe "CompileManager", -> @Settings.path.compilesDir = "/local/compile/directory" @image = "example.com/image" - @CompileManager.wordcount @project_id, @file_name, @image, @callback + @CompileManager.wordcount @project_id, @user_id, @file_name, @image, @callback it "should run the texcount command", -> - @directory = "#{@Settings.path.compilesDir}/#{@project_id}" + @directory = "#{@Settings.path.compilesDir}/#{@project_id}-#{@user_id}" @file_path = "$COMPILE_DIR/#{@file_name}" @command =[ "texcount", "-inc", @file_path, "-out=" + @file_path + ".wc"] @CommandRunner.run - .calledWith(@project_id, @command, @directory, @image, @timeout) + .calledWith("#{@project_id}-#{@user_id}", @command, @directory, @image, @timeout) .should.equal true it "should call the callback with the parsed output", ->