retry once on EPIPE errors

Co-Authored-By: Jakob Ackermann <jakob.ackermann@overleaf.com>
This commit is contained in:
Tim Alby 2020-04-10 12:28:11 +02:00
parent 66447211d0
commit e3b1472c7f
2 changed files with 80 additions and 12 deletions

View file

@ -407,7 +407,16 @@ module.exports = DockerRunner = {
})
}
)
return container.inspect(function(error, stats) {
const callbackWithRetry = error => {
if (error.message.match(/EPIPE/)) {
return inspectContainer(container, callback)
}
callback(error)
}
var inspectContainer = (container, innerCallback) =>
container.inspect(function(error, stats) {
if ((error != null ? error.statusCode : undefined) === 404) {
return createAndStartContainer()
} else if (error != null) {
@ -415,11 +424,12 @@ module.exports = DockerRunner = {
{ container_name: name, error },
'unable to inspect container to start'
)
return callback(error)
return innerCallback(error)
} else {
return startExistingContainer()
}
})
inspectContainer(container, callbackWithRetry)
},
attachToContainer(containerId, attachStreamHandler, attachStartCallback) {

View file

@ -36,6 +36,7 @@ describe('DockerRunner', function() {
'logger-sharelatex': (this.logger = {
log: sinon.stub(),
error: sinon.stub(),
err: sinon.stub(),
info: sinon.stub(),
warn: sinon.stub()
}),
@ -387,6 +388,63 @@ describe('DockerRunner', function() {
})
})
describe('when inspect always fails with EPIPE error', function() {
beforeEach(function() {
this.error = new Error('write EPIPE')
this.container.inspect = sinon.stub().yields(this.error)
this.container.start = sinon.stub().yields()
this.DockerRunner.startContainer(
this.options,
this.volumes,
() => {},
this.callback
)
})
it('should retry once', function() {
sinon.assert.callOrder(
this.container.inspect,
this.container.inspect,
this.callback
)
})
it('should call back with error', function() {
sinon.assert.calledWith(this.callback, this.error)
})
})
describe('when inspect fails once with EPIPE error', function() {
beforeEach(function() {
this.container.inspect = sinon.stub()
this.container.inspect.onFirstCall().yields(new Error('write EPIPE'))
this.container.inspect.onSecondCall().yields()
this.container.start = sinon.stub().yields()
this.DockerRunner.startContainer(
this.options,
this.volumes,
() => {},
this.callback
)
})
it('should retry once and start container', function() {
sinon.assert.callOrder(
this.container.inspect,
this.container.inspect,
this.DockerRunner.attachToContainer,
this.container.start,
this.callback
)
})
it('should call back without error', function() {
sinon.assert.calledWith(this.callback, null)
})
})
describe('when the container does not exist', function() {
beforeEach(function() {
const exists = false