From 2ce03f055468e9133e7d4452c8a8caa5eabce2fd Mon Sep 17 00:00:00 2001 From: Brian Gough Date: Thu, 11 Jun 2020 16:01:44 +0100 Subject: [PATCH] add initial compileGroup support --- services/clsi/app/js/CompileManager.js | 7 +- services/clsi/app/js/DockerRunner.js | 40 +++++++++++- services/clsi/app/js/LatexRunner.js | 14 +++- services/clsi/app/js/LocalCommandRunner.js | 11 +++- services/clsi/app/js/RequestParser.js | 12 +++- services/clsi/config/settings.defaults.js | 26 ++++++++ .../clsi/test/unit/js/CompileManagerTests.js | 18 ++++-- .../clsi/test/unit/js/DockerRunnerTests.js | 64 +++++++++++++++++++ .../clsi/test/unit/js/LatexRunnerTests.js | 9 ++- 9 files changed, 183 insertions(+), 18 deletions(-) diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js index 614c49aaf7..8ca80d8e8f 100644 --- a/services/clsi/app/js/CompileManager.js +++ b/services/clsi/app/js/CompileManager.js @@ -199,7 +199,8 @@ module.exports = CompileManager = { timeout: request.timeout, image: request.imageName, flags: request.flags, - environment: env + environment: env, + compileGroup: request.compileGroup }, function(error, output, stats, timings) { // request was for validation only @@ -536,6 +537,7 @@ module.exports = CompileManager = { const directory = getCompileDir(project_id, user_id) const timeout = 60 * 1000 // increased to allow for large projects const compileName = getCompileName(project_id, user_id) + const compileGroup = 'synctex' return CommandRunner.run( compileName, command, @@ -543,6 +545,7 @@ module.exports = CompileManager = { Settings.clsi != null ? Settings.clsi.docker.image : undefined, timeout, {}, + compileGroup, function(error, output) { if (error != null) { logger.err( @@ -606,6 +609,7 @@ module.exports = CompileManager = { const compileDir = getCompileDir(project_id, user_id) const timeout = 60 * 1000 const compileName = getCompileName(project_id, user_id) + const compileGroup = 'wordcount' return fse.ensureDir(compileDir, function(error) { if (error != null) { logger.err( @@ -621,6 +625,7 @@ module.exports = CompileManager = { image, timeout, {}, + compileGroup, function(error) { if (error != null) { return callback(error) diff --git a/services/clsi/app/js/DockerRunner.js b/services/clsi/app/js/DockerRunner.js index cb6ec2d2a7..2a6330f866 100644 --- a/services/clsi/app/js/DockerRunner.js +++ b/services/clsi/app/js/DockerRunner.js @@ -45,7 +45,16 @@ module.exports = DockerRunner = { ERR_EXITED: new Error('exited'), ERR_TIMED_OUT: new Error('container timed out'), - run(project_id, command, directory, image, timeout, environment, callback) { + run( + project_id, + command, + directory, + image, + timeout, + environment, + compileGroup, + callback + ) { let name if (callback == null) { callback = function(error, output) {} @@ -88,7 +97,8 @@ module.exports = DockerRunner = { image, volumes, timeout, - environment + environment, + compileGroup ) const fingerprint = DockerRunner._fingerprintContainer(options) options.name = name = `project-${project_id}-${fingerprint}` @@ -224,7 +234,14 @@ module.exports = DockerRunner = { ) }, - _getContainerOptions(command, image, volumes, timeout, environment) { + _getContainerOptions( + command, + image, + volumes, + timeout, + environment, + compileGroup + ) { let m, year let key, value, hostVol, dockerVol const timeoutInSeconds = timeout / 1000 @@ -311,6 +328,23 @@ module.exports = DockerRunner = { options.HostConfig.Runtime = Settings.clsi.docker.runtime } + if (Settings.clsi.docker.Readonly) { + options.HostConfig.ReadonlyRootfs = true + options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' } + } + + // Allow per-compile group overriding of individual settings + if ( + Settings.clsi.docker.compileGroupConfig && + Settings.clsi.docker.compileGroupConfig[compileGroup] + ) { + const override = Settings.clsi.docker.compileGroupConfig[compileGroup] + let key + for (key in override) { + _.set(options, key, override[key]) + } + } + return options }, diff --git a/services/clsi/app/js/LatexRunner.js b/services/clsi/app/js/LatexRunner.js index c3edb29e24..6d1591a213 100644 --- a/services/clsi/app/js/LatexRunner.js +++ b/services/clsi/app/js/LatexRunner.js @@ -36,7 +36,8 @@ module.exports = LatexRunner = { timeout, image, environment, - flags + flags, + compileGroup } = options if (!compiler) { compiler = 'pdflatex' @@ -46,7 +47,15 @@ module.exports = LatexRunner = { } // milliseconds logger.log( - { directory, compiler, timeout, mainFile, environment, flags }, + { + directory, + compiler, + timeout, + mainFile, + environment, + flags, + compileGroup + }, 'starting compile' ) @@ -79,6 +88,7 @@ module.exports = LatexRunner = { image, timeout, environment, + compileGroup, function(error, output) { delete ProcessTable[id] if (error != null) { diff --git a/services/clsi/app/js/LocalCommandRunner.js b/services/clsi/app/js/LocalCommandRunner.js index ccaf50784a..14a19986e2 100644 --- a/services/clsi/app/js/LocalCommandRunner.js +++ b/services/clsi/app/js/LocalCommandRunner.js @@ -20,7 +20,16 @@ const logger = require('logger-sharelatex') logger.info('using standard command runner') module.exports = CommandRunner = { - run(project_id, command, directory, image, timeout, environment, callback) { + run( + project_id, + command, + directory, + image, + timeout, + environment, + compileGroup, + callback + ) { let key, value if (callback == null) { callback = function(error) {} diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js index acfdc6689d..f2be556af1 100644 --- a/services/clsi/app/js/RequestParser.js +++ b/services/clsi/app/js/RequestParser.js @@ -74,7 +74,17 @@ module.exports = RequestParser = { default: [], type: 'object' }) - + if (settings.allowedCompileGroups) { + response.compileGroup = this._parseAttribute( + 'compileGroup', + compile.options.compileGroup, + { + validValues: settings.allowedCompileGroups, + default: '', + type: 'string' + } + ) + } // The syncType specifies whether the request contains all // resources (full) or only those resources to be updated // in-place (incremental). diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js index 9e44e1491d..f3bb73b007 100644 --- a/services/clsi/config/settings.defaults.js +++ b/services/clsi/config/settings.defaults.js @@ -63,6 +63,17 @@ module.exports = { } } +if (process.env.ALLOWED_COMPILE_GROUPS) { + try { + module.exports.allowedCompileGroups = process.env.ALLOWED_COMPILE_GROUPS.split( + ' ' + ) + } catch (error) { + console.error(error, 'could not apply allowed compile group setting') + process.exit(1) + } +} + if (process.env.DOCKER_RUNNER) { let seccompProfilePath module.exports.clsi = { @@ -82,6 +93,21 @@ if (process.env.DOCKER_RUNNER) { checkProjectsIntervalMs: 10 * 60 * 1000 } + try { + // Override individual docker settings using path-based keys, e.g.: + // compileGroupDockerConfigs = { + // priority: { 'HostConfig.CpuShares': 100 } + // beta: { 'dotted.path.here', 'value'} + // } + const compileGroupConfig = JSON.parse( + process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}' + ) + module.exports.clsi.docker.compileGroupConfig = compileGroupConfig + } catch (error) { + console.error(error, 'could not apply compile group docker configs') + process.exit(1) + } + try { seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json') module.exports.clsi.docker.seccomp_profile = JSON.stringify( diff --git a/services/clsi/test/unit/js/CompileManagerTests.js b/services/clsi/test/unit/js/CompileManagerTests.js index 74a0a47fff..90d572b2a2 100644 --- a/services/clsi/test/unit/js/CompileManagerTests.js +++ b/services/clsi/test/unit/js/CompileManagerTests.js @@ -160,7 +160,8 @@ describe('CompileManager', function() { compiler: (this.compiler = 'pdflatex'), timeout: (this.timeout = 42000), imageName: (this.image = 'example.com/image'), - flags: (this.flags = ['-file-line-error']) + flags: (this.flags = ['-file-line-error']), + compileGroup: (this.compileGroup = 'compile-group') } this.env = {} this.Settings.compileDir = 'compiles' @@ -199,7 +200,8 @@ describe('CompileManager', function() { timeout: this.timeout, image: this.image, flags: this.flags, - environment: this.env + environment: this.env, + compileGroup: this.compileGroup }) .should.equal(true) }) @@ -253,7 +255,8 @@ describe('CompileManager', function() { CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16', CHKTEX_EXIT_ON_ERROR: 1, CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000' - } + }, + compileGroup: this.compileGroup }) .should.equal(true) }) @@ -275,7 +278,8 @@ describe('CompileManager', function() { timeout: this.timeout, image: this.image, flags: this.flags, - environment: this.env + environment: this.env, + compileGroup: this.compileGroup }) .should.equal(true) }) @@ -384,7 +388,7 @@ describe('CompileManager', function() { this.stdout = `NODE\t${this.page}\t${this.h}\t${this.v}\t${this.width}\t${this.height}\n` this.CommandRunner.run = sinon .stub() - .callsArgWith(6, null, { stdout: this.stdout }) + .callsArgWith(7, null, { stdout: this.stdout }) return this.CompileManager.syncFromCode( this.project_id, this.user_id, @@ -443,7 +447,7 @@ describe('CompileManager', function() { this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}\t${this.line}\t${this.column}\n` this.CommandRunner.run = sinon .stub() - .callsArgWith(6, null, { stdout: this.stdout }) + .callsArgWith(7, null, { stdout: this.stdout }) return this.CompileManager.syncFromPdf( this.project_id, this.user_id, @@ -485,7 +489,7 @@ describe('CompileManager', function() { return describe('wordcount', function() { beforeEach(function() { - this.CommandRunner.run = sinon.stub().callsArg(6) + this.CommandRunner.run = sinon.stub().callsArg(7) this.fs.readFile = sinon .stub() .callsArgWith( diff --git a/services/clsi/test/unit/js/DockerRunnerTests.js b/services/clsi/test/unit/js/DockerRunnerTests.js index b761d20340..2e2ffec638 100644 --- a/services/clsi/test/unit/js/DockerRunnerTests.js +++ b/services/clsi/test/unit/js/DockerRunnerTests.js @@ -87,6 +87,7 @@ describe('DockerRunner', function() { this.project_id = 'project-id-123' this.volumes = { '/local/compile/directory': '/compile' } this.Settings.clsi.docker.image = this.defaultImage = 'default-image' + this.compileGroup = 'compile-group' return (this.Settings.clsi.docker.env = { PATH: 'mock-path' }) }) @@ -123,6 +124,7 @@ describe('DockerRunner', function() { this.image, this.timeout, this.env, + this.compileGroup, (err, output) => { this.callback(err, output) return done() @@ -172,6 +174,7 @@ describe('DockerRunner', function() { this.image, this.timeout, this.env, + this.compileGroup, this.callback ) }) @@ -220,6 +223,7 @@ describe('DockerRunner', function() { this.image, this.timeout, this.env, + this.compileGroup, this.callback ) }) @@ -253,6 +257,7 @@ describe('DockerRunner', function() { null, this.timeout, this.env, + this.compileGroup, this.callback ) }) @@ -282,6 +287,7 @@ describe('DockerRunner', function() { this.image, this.timeout, this.env, + this.compileGroup, this.callback ) }) @@ -293,6 +299,64 @@ describe('DockerRunner', function() { }) }) + describe('run with _getOptions', function() { + beforeEach(function(done) { + // this.DockerRunner._getContainerOptions = sinon + // .stub() + // .returns((this.options = { mockoptions: 'foo' })) + this.DockerRunner._fingerprintContainer = sinon + .stub() + .returns((this.fingerprint = 'fingerprint')) + + this.name = `project-${this.project_id}-${this.fingerprint}` + + this.command = ['mock', 'command', '--outdir=$COMPILE_DIR'] + this.command_with_dir = ['mock', 'command', '--outdir=/compile'] + this.timeout = 42000 + return done() + }) + + describe('when a compile group config is set', function() { + beforeEach(function() { + this.Settings.clsi.docker.compileGroupConfig = { + 'compile-group': { + 'HostConfig.newProperty': 'new-property' + }, + 'other-group': { otherProperty: 'other-property' } + } + this.DockerRunner._runAndWaitForContainer = sinon + .stub() + .callsArgWith(3, null, (this.output = 'mock-output')) + return this.DockerRunner.run( + this.project_id, + this.command, + this.directory, + this.image, + this.timeout, + this.env, + this.compileGroup, + this.callback + ) + }) + + it('should set the docker options for the compile group', function() { + const options = this.DockerRunner._runAndWaitForContainer.lastCall + .args[0] + return expect(options.HostConfig).to.deep.include({ + Binds: ['/local/compile/directory:/compile:rw'], + LogConfig: { Type: 'none', Config: {} }, + CapDrop: 'ALL', + SecurityOpt: ['no-new-privileges'], + newProperty: 'new-property' + }) + }) + + return it('should call the callback', function() { + return this.callback.calledWith(null, this.output).should.equal(true) + }) + }) + }) + describe('_runAndWaitForContainer', function() { beforeEach(function() { this.options = { mockoptions: 'foo', name: (this.name = 'mock-name') } diff --git a/services/clsi/test/unit/js/LatexRunnerTests.js b/services/clsi/test/unit/js/LatexRunnerTests.js index b112b17006..f480bc8245 100644 --- a/services/clsi/test/unit/js/LatexRunnerTests.js +++ b/services/clsi/test/unit/js/LatexRunnerTests.js @@ -48,6 +48,7 @@ describe('LatexRunner', function() { this.mainFile = 'main-file.tex' this.compiler = 'pdflatex' this.image = 'example.com/image' + this.compileGroup = 'compile-group' this.callback = sinon.stub() this.project_id = 'project-id-123' return (this.env = { foo: '123' }) @@ -55,7 +56,7 @@ describe('LatexRunner', function() { return describe('runLatex', function() { beforeEach(function() { - return (this.CommandRunner.run = sinon.stub().callsArgWith(6, null, { + return (this.CommandRunner.run = sinon.stub().callsArgWith(7, null, { stdout: 'this is stdout', stderr: 'this is stderr' })) @@ -71,7 +72,8 @@ describe('LatexRunner', function() { compiler: this.compiler, timeout: (this.timeout = 42000), image: this.image, - environment: this.env + environment: this.env, + compileGroup: this.compileGroup }, this.callback ) @@ -85,7 +87,8 @@ describe('LatexRunner', function() { this.directory, this.image, this.timeout, - this.env + this.env, + this.compileGroup ) .should.equal(true) })