github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-02-03 00:21:33 +00:00
Code Issues Projects Releases Packages Wiki Activity

add initial compileGroup support

Browse source
This commit is contained in:
Brian Gough 2020-06-11 16:01:44 +01:00
parent ede70b6f99
commit 2ce03f0554
9 changed files with 183 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
services/clsi/app/js/CompileManager.js
View file

@ -199,7 +199,8 @@ module.exports = CompileManager = {
timeout: request.timeout,
image: request.imageName,
flags: request.flags,
environment: env
environment: env,
compileGroup: request.compileGroup
},
function(error, output, stats, timings) {
// request was for validation only
@ -536,6 +537,7 @@ module.exports = CompileManager = {
const directory = getCompileDir(project_id, user_id)
const timeout = 60 * 1000 // increased to allow for large projects
const compileName = getCompileName(project_id, user_id)
const compileGroup = 'synctex'
return CommandRunner.run(
compileName,
command,
@ -543,6 +545,7 @@ module.exports = CompileManager = {
Settings.clsi != null ? Settings.clsi.docker.image : undefined,
timeout,
{},
compileGroup,
function(error, output) {
if (error != null) {
logger.err(
@ -606,6 +609,7 @@ module.exports = CompileManager = {
const compileDir = getCompileDir(project_id, user_id)
const timeout = 60 * 1000
const compileName = getCompileName(project_id, user_id)
const compileGroup = 'wordcount'
return fse.ensureDir(compileDir, function(error) {
if (error != null) {
logger.err(
@ -621,6 +625,7 @@ module.exports = CompileManager = {
image,
timeout,
{},
compileGroup,
function(error) {
if (error != null) {
return callback(error)

40
services/clsi/app/js/DockerRunner.js
View file

@ -45,7 +45,16 @@ module.exports = DockerRunner = {
ERR_EXITED: new Error('exited'),
ERR_TIMED_OUT: new Error('container timed out'),
run(project_id, command, directory, image, timeout, environment, callback) {
run(
project_id,
command,
directory,
image,
timeout,
environment,
compileGroup,
callback
) {
let name
if (callback == null) {
callback = function(error, output) {}
@ -88,7 +97,8 @@ module.exports = DockerRunner = {
image,
volumes,
timeout,
environment
environment,
compileGroup
)
const fingerprint = DockerRunner._fingerprintContainer(options)
options.name = name = `project-${project_id}-${fingerprint}`
@ -224,7 +234,14 @@ module.exports = DockerRunner = {
)
},
_getContainerOptions(command, image, volumes, timeout, environment) {
_getContainerOptions(
command,
image,
volumes,
timeout,
environment,
compileGroup
) {
let m, year
let key, value, hostVol, dockerVol
const timeoutInSeconds = timeout / 1000
@ -311,6 +328,23 @@ module.exports = DockerRunner = {
options.HostConfig.Runtime = Settings.clsi.docker.runtime
}
if (Settings.clsi.docker.Readonly) {
options.HostConfig.ReadonlyRootfs = true
options.HostConfig.Tmpfs = { '/tmp': 'rw,noexec,nosuid,size=65536k' }
}
// Allow per-compile group overriding of individual settings
if (
Settings.clsi.docker.compileGroupConfig &&
Settings.clsi.docker.compileGroupConfig[compileGroup]
) {
const override = Settings.clsi.docker.compileGroupConfig[compileGroup]
let key
for (key in override) {
_.set(options, key, override[key])
}
}
return options
},

14
services/clsi/app/js/LatexRunner.js
View file

@ -36,7 +36,8 @@ module.exports = LatexRunner = {
timeout,
image,
environment,
flags
flags,
compileGroup
} = options
if (!compiler) {
compiler = 'pdflatex'
@ -46,7 +47,15 @@ module.exports = LatexRunner = {
} // milliseconds
logger.log(
{ directory, compiler, timeout, mainFile, environment, flags },
{
directory,
compiler,
timeout,
mainFile,
environment,
flags,
compileGroup
},
'starting compile'
)
@ -79,6 +88,7 @@ module.exports = LatexRunner = {
image,
timeout,
environment,
compileGroup,
function(error, output) {
delete ProcessTable[id]
if (error != null) {

11
services/clsi/app/js/LocalCommandRunner.js
View file

@ -20,7 +20,16 @@ const logger = require('logger-sharelatex')
logger.info('using standard command runner')
module.exports = CommandRunner = {
run(project_id, command, directory, image, timeout, environment, callback) {
run(
project_id,
command,
directory,
image,
timeout,
environment,
compileGroup,
callback
) {
let key, value
if (callback == null) {
callback = function(error) {}

12
services/clsi/app/js/RequestParser.js
View file

@ -74,7 +74,17 @@ module.exports = RequestParser = {
default: [],
type: 'object'
})
if (settings.allowedCompileGroups) {
response.compileGroup = this._parseAttribute(
'compileGroup',
compile.options.compileGroup,
{
validValues: settings.allowedCompileGroups,
default: '',
type: 'string'
}
)
}
// The syncType specifies whether the request contains all
// resources (full) or only those resources to be updated
// in-place (incremental).

26
services/clsi/config/settings.defaults.js
View file

@ -63,6 +63,17 @@ module.exports = {
}
}
if (process.env.ALLOWED_COMPILE_GROUPS) {
try {
module.exports.allowedCompileGroups = process.env.ALLOWED_COMPILE_GROUPS.split(
' '
)
} catch (error) {
console.error(error, 'could not apply allowed compile group setting')
process.exit(1)
}
}
if (process.env.DOCKER_RUNNER) {
let seccompProfilePath
module.exports.clsi = {
@ -82,6 +93,21 @@ if (process.env.DOCKER_RUNNER) {
checkProjectsIntervalMs: 10 * 60 * 1000
}
try {
// Override individual docker settings using path-based keys, e.g.:
// compileGroupDockerConfigs = {
// priority: { 'HostConfig.CpuShares': 100 }
// beta: { 'dotted.path.here', 'value'}
// }
const compileGroupConfig = JSON.parse(
process.env.COMPILE_GROUP_DOCKER_CONFIGS || '{}'
)
module.exports.clsi.docker.compileGroupConfig = compileGroupConfig
} catch (error) {
console.error(error, 'could not apply compile group docker configs')
process.exit(1)
}
try {
seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
module.exports.clsi.docker.seccomp_profile = JSON.stringify(

18
services/clsi/test/unit/js/CompileManagerTests.js
View file

@ -160,7 +160,8 @@ describe('CompileManager', function() {
compiler: (this.compiler = 'pdflatex'),
timeout: (this.timeout = 42000),
imageName: (this.image = 'example.com/image'),
flags: (this.flags = ['-file-line-error'])
flags: (this.flags = ['-file-line-error']),
compileGroup: (this.compileGroup = 'compile-group')
}
this.env = {}
this.Settings.compileDir = 'compiles'
@ -199,7 +200,8 @@ describe('CompileManager', function() {
timeout: this.timeout,
image: this.image,
flags: this.flags,
environment: this.env
environment: this.env,
compileGroup: this.compileGroup
})
.should.equal(true)
})
@ -253,7 +255,8 @@ describe('CompileManager', function() {
CHKTEX_OPTIONS: '-nall -e9 -e10 -w15 -w16',
CHKTEX_EXIT_ON_ERROR: 1,
CHKTEX_ULIMIT_OPTIONS: '-t 5 -v 64000'
}
},
compileGroup: this.compileGroup
})
.should.equal(true)
})
@ -275,7 +278,8 @@ describe('CompileManager', function() {
timeout: this.timeout,
image: this.image,
flags: this.flags,
environment: this.env
environment: this.env,
compileGroup: this.compileGroup
})
.should.equal(true)
})
@ -384,7 +388,7 @@ describe('CompileManager', function() {
this.stdout = `NODE\t${this.page}\t${this.h}\t${this.v}\t${this.width}\t${this.height}\n`
this.CommandRunner.run = sinon
.stub()
.callsArgWith(6, null, { stdout: this.stdout })
.callsArgWith(7, null, { stdout: this.stdout })
return this.CompileManager.syncFromCode(
this.project_id,
this.user_id,
@ -443,7 +447,7 @@ describe('CompileManager', function() {
this.stdout = `NODE\t${this.Settings.path.compilesDir}/${this.project_id}-${this.user_id}/${this.file_name}\t${this.line}\t${this.column}\n`
this.CommandRunner.run = sinon
.stub()
.callsArgWith(6, null, { stdout: this.stdout })
.callsArgWith(7, null, { stdout: this.stdout })
return this.CompileManager.syncFromPdf(
this.project_id,
this.user_id,
@ -485,7 +489,7 @@ describe('CompileManager', function() {
return describe('wordcount', function() {
beforeEach(function() {
this.CommandRunner.run = sinon.stub().callsArg(6)
this.CommandRunner.run = sinon.stub().callsArg(7)
this.fs.readFile = sinon
.stub()
.callsArgWith(

64
services/clsi/test/unit/js/DockerRunnerTests.js
View file

@ -87,6 +87,7 @@ describe('DockerRunner', function() {
this.project_id = 'project-id-123'
this.volumes = { '/local/compile/directory': '/compile' }
this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
this.compileGroup = 'compile-group'
return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
})
@ -123,6 +124,7 @@ describe('DockerRunner', function() {
this.image,
this.timeout,
this.env,
this.compileGroup,
(err, output) => {
this.callback(err, output)
return done()
@ -172,6 +174,7 @@ describe('DockerRunner', function() {
this.image,
this.timeout,
this.env,
this.compileGroup,
this.callback
)
})
@ -220,6 +223,7 @@ describe('DockerRunner', function() {
this.image,
this.timeout,
this.env,
this.compileGroup,
this.callback
)
})
@ -253,6 +257,7 @@ describe('DockerRunner', function() {
null,
this.timeout,
this.env,
this.compileGroup,
this.callback
)
})
@ -282,6 +287,7 @@ describe('DockerRunner', function() {
this.image,
this.timeout,
this.env,
this.compileGroup,
this.callback
)
})
@ -293,6 +299,64 @@ describe('DockerRunner', function() {
})
})
describe('run with _getOptions', function() {
beforeEach(function(done) {
// this.DockerRunner._getContainerOptions = sinon
// .stub()
// .returns((this.options = { mockoptions: 'foo' }))
this.DockerRunner._fingerprintContainer = sinon
.stub()
.returns((this.fingerprint = 'fingerprint'))
this.name = `project-${this.project_id}-${this.fingerprint}`
this.command = ['mock', 'command', '--outdir=$COMPILE_DIR']
this.command_with_dir = ['mock', 'command', '--outdir=/compile']
this.timeout = 42000
return done()
})
describe('when a compile group config is set', function() {
beforeEach(function() {
this.Settings.clsi.docker.compileGroupConfig = {
'compile-group': {
'HostConfig.newProperty': 'new-property'
},
'other-group': { otherProperty: 'other-property' }
}
this.DockerRunner._runAndWaitForContainer = sinon
.stub()
.callsArgWith(3, null, (this.output = 'mock-output'))
return this.DockerRunner.run(
this.project_id,
this.command,
this.directory,
this.image,
this.timeout,
this.env,
this.compileGroup,
this.callback
)
})
it('should set the docker options for the compile group', function() {
const options = this.DockerRunner._runAndWaitForContainer.lastCall
.args[0]
return expect(options.HostConfig).to.deep.include({
Binds: ['/local/compile/directory:/compile:rw'],
LogConfig: { Type: 'none', Config: {} },
CapDrop: 'ALL',
SecurityOpt: ['no-new-privileges'],
newProperty: 'new-property'
})
})
return it('should call the callback', function() {
return this.callback.calledWith(null, this.output).should.equal(true)
})
})
})
describe('_runAndWaitForContainer', function() {
beforeEach(function() {
this.options = { mockoptions: 'foo', name: (this.name = 'mock-name') }

9
services/clsi/test/unit/js/LatexRunnerTests.js
View file

@ -48,6 +48,7 @@ describe('LatexRunner', function() {
this.mainFile = 'main-file.tex'
this.compiler = 'pdflatex'
this.image = 'example.com/image'
this.compileGroup = 'compile-group'
this.callback = sinon.stub()
this.project_id = 'project-id-123'
return (this.env = { foo: '123' })
@ -55,7 +56,7 @@ describe('LatexRunner', function() {
return describe('runLatex', function() {
beforeEach(function() {
return (this.CommandRunner.run = sinon.stub().callsArgWith(6, null, {
return (this.CommandRunner.run = sinon.stub().callsArgWith(7, null, {
stdout: 'this is stdout',
stderr: 'this is stderr'
}))
@ -71,7 +72,8 @@ describe('LatexRunner', function() {
compiler: this.compiler,
timeout: (this.timeout = 42000),
image: this.image,
environment: this.env
environment: this.env,
compileGroup: this.compileGroup
},
this.callback
)
@ -85,7 +87,8 @@ describe('LatexRunner', function() {
this.directory,
this.image,
this.timeout,
this.env
this.env,
this.compileGroup
)
.should.equal(true)
})