retry once on EPIPE errors

Co-Authored-By: Jakob Ackermann <jakob.ackermann@overleaf.com>

services/clsi/app/js/DockerRunner.js

@@ -407,7 +407,16 @@ module.exports = DockerRunner = {
           })
         }
       )
-    return container.inspect(function(error, stats) {
+
+    const callbackWithRetry = error => {
+      if (error.message.match(/EPIPE/)) {
+        return inspectContainer(container, callback)
+      }
+      callback(error)
+    }
+
+    var inspectContainer = (container, innerCallback) =>
+      container.inspect(function(error, stats) {
         if ((error != null ? error.statusCode : undefined) === 404) {
           return createAndStartContainer()
         } else if (error != null) {
@@ -415,11 +424,12 @@ module.exports = DockerRunner = {
             { container_name: name, error },
             'unable to inspect container to start'
           )
-        return callback(error)
+          return innerCallback(error)
         } else {
           return startExistingContainer()
         }
       })
+    inspectContainer(container, callbackWithRetry)
   },
 
   attachToContainer(containerId, attachStreamHandler, attachStartCallback) {

services/clsi/test/unit/js/DockerRunnerTests.js

@@ -36,6 +36,7 @@ describe('DockerRunner', function() {
         'logger-sharelatex': (this.logger = {
           log: sinon.stub(),
           error: sinon.stub(),
+          err: sinon.stub(),
           info: sinon.stub(),
           warn: sinon.stub()
         }),
@@ -387,6 +388,63 @@ describe('DockerRunner', function() {
       })
     })
 
+    describe('when inspect always fails with EPIPE error', function() {
+      beforeEach(function() {
+        this.error = new Error('write EPIPE')
+        this.container.inspect = sinon.stub().yields(this.error)
+        this.container.start = sinon.stub().yields()
+
+        this.DockerRunner.startContainer(
+          this.options,
+          this.volumes,
+          () => {},
+          this.callback
+        )
+      })
+
+      it('should retry once', function() {
+        sinon.assert.callOrder(
+          this.container.inspect,
+          this.container.inspect,
+          this.callback
+        )
+      })
+
+      it('should call back with error', function() {
+        sinon.assert.calledWith(this.callback, this.error)
+      })
+    })
+
+    describe('when inspect fails once with EPIPE error', function() {
+      beforeEach(function() {
+        this.container.inspect = sinon.stub()
+        this.container.inspect.onFirstCall().yields(new Error('write EPIPE'))
+        this.container.inspect.onSecondCall().yields()
+        this.container.start = sinon.stub().yields()
+
+        this.DockerRunner.startContainer(
+          this.options,
+          this.volumes,
+          () => {},
+          this.callback
+        )
+      })
+
+      it('should retry once and start container', function() {
+        sinon.assert.callOrder(
+          this.container.inspect,
+          this.container.inspect,
+          this.DockerRunner.attachToContainer,
+          this.container.start,
+          this.callback
+        )
+      })
+
+      it('should call back without error', function() {
+        sinon.assert.calledWith(this.callback, null)
+      })
+    })
+
     describe('when the container does not exist', function() {
       beforeEach(function() {
         const exists = false