github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2024-11-21 20:47:08 -05:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #162 from overleaf/ta-jpa-epipe-retry

Browse source 
[DockerRunner] retry container inspect on EPIPE
This commit is contained in:
Henry Oswald 2020-05-19 11:15:25 +01:00 committed by GitHub
commit 5c3da905ba
2 changed files with 83 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
services/clsi/app/js/DockerRunner.js
View file

@ -26,6 +26,7 @@ const LockManager = require('./DockerLockManager')
const fs = require('fs')
const Path = require('path')
const _ = require('underscore')
const metrics = require('metrics-sharelatex')
logger.info('using docker runner')
@ -411,19 +412,28 @@ module.exports = DockerRunner = {
})
}
)
return container.inspect(function(error, stats) {
if ((error != null ? error.statusCode : undefined) === 404) {
return createAndStartContainer()
} else if (error != null) {
logger.err(
{ container_name: name, error },
'unable to inspect container to start'
)
return callback(error)
} else {
return startExistingContainer()
}
})
var inspectContainer = (isRetry) =>
container.inspect(function(error, stats) {
if ((error != null ? error.statusCode : undefined) === 404) {
return createAndStartContainer()
} else if (error != null) {
if (error.message.match(/EPIPE/)) {
if (!isRetry) {
metrics.inc('container-inspect-epipe-retry')
return inspectContainer(true)
}
metrics.inc('container-inspect-epipe-error')
}
logger.err(
{ container_name: name, error },
'unable to inspect container to start'
)
return callback(error)
} else {
return startExistingContainer()
}
})
inspectContainer(false)
},
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {

62
services/clsi/test/unit/js/DockerRunnerTests.js
View file

@ -36,6 +36,7 @@ describe('DockerRunner', function() {
'logger-sharelatex': (this.logger = {
log: sinon.stub(),
error: sinon.stub(),
err: sinon.stub(),
info: sinon.stub(),
warn: sinon.stub()
}),
@ -357,8 +358,8 @@ describe('DockerRunner', function() {
return this.DockerRunner.startContainer(
this.options,
this.volumes,
this.callback,
() => {}
() => {},
this.callback
)
})
@ -387,6 +388,63 @@ describe('DockerRunner', function() {
})
})
describe('when inspect always fails with EPIPE error', function() {
beforeEach(function() {
this.error = new Error('write EPIPE')
this.container.inspect = sinon.stub().yields(this.error)
this.container.start = sinon.stub().yields()
this.DockerRunner.startContainer(
this.options,
this.volumes,
() => {},
this.callback
)
})
it('should retry once', function() {
sinon.assert.callOrder(
this.container.inspect,
this.container.inspect,
this.callback
)
})
it('should call back with error', function() {
sinon.assert.calledWith(this.callback, this.error)
})
})
describe('when inspect fails once with EPIPE error', function() {
beforeEach(function() {
this.container.inspect = sinon.stub()
this.container.inspect.onFirstCall().yields(new Error('write EPIPE'))
this.container.inspect.onSecondCall().yields()
this.container.start = sinon.stub().yields()
this.DockerRunner.startContainer(
this.options,
this.volumes,
() => {},
this.callback
)
})
it('should retry once and start container', function() {
sinon.assert.callOrder(
this.container.inspect,
this.container.inspect,
this.DockerRunner.attachToContainer,
this.container.start,
this.callback
)
})
it('should call back without error', function() {
sinon.assert.calledWith(this.callback, null)
})
})
describe('when the container does not exist', function() {
beforeEach(function() {
const exists = false