retry once on EPIPE errors

Co-Authored-By: Jakob Ackermann <jakob.ackermann@overleaf.com>

services/clsi/app/js/DockerRunner.js

@@ -407,19 +407,29 @@ module.exports = DockerRunner = {
           })
         }
       )
-    return container.inspect(function(error, stats) {
-      if ((error != null ? error.statusCode : undefined) === 404) {
-        return createAndStartContainer()
-      } else if (error != null) {
-        logger.err(
-          { container_name: name, error },
-          'unable to inspect container to start'
-        )
-        return callback(error)
-      } else {
-        return startExistingContainer()
+
+    const callbackWithRetry = error => {
+      if (error.message.match(/EPIPE/)) {
+        return inspectContainer(container, callback)
       }
-    })
+      callback(error)
+    }
+
+    var inspectContainer = (container, innerCallback) =>
+      container.inspect(function(error, stats) {
+        if ((error != null ? error.statusCode : undefined) === 404) {
+          return createAndStartContainer()
+        } else if (error != null) {
+          logger.err(
+            { container_name: name, error },
+            'unable to inspect container to start'
+          )
+          return innerCallback(error)
+        } else {
+          return startExistingContainer()
+        }
+      })
+    inspectContainer(container, callbackWithRetry)
   },
 
   attachToContainer(containerId, attachStreamHandler, attachStartCallback) {

services/clsi/test/unit/js/DockerRunnerTests.js

@@ -36,6 +36,7 @@ describe('DockerRunner', function() {
         'logger-sharelatex': (this.logger = {
           log: sinon.stub(),
           error: sinon.stub(),
+          err: sinon.stub(),
           info: sinon.stub(),
           warn: sinon.stub()
         }),
@@ -387,6 +388,63 @@ describe('DockerRunner', function() {
       })
     })
 
+    describe('when inspect always fails with EPIPE error', function() {
+      beforeEach(function() {
+        this.error = new Error('write EPIPE')
+        this.container.inspect = sinon.stub().yields(this.error)
+        this.container.start = sinon.stub().yields()
+
+        this.DockerRunner.startContainer(
+          this.options,
+          this.volumes,
+          () => {},
+          this.callback
+        )
+      })
+
+      it('should retry once', function() {
+        sinon.assert.callOrder(
+          this.container.inspect,
+          this.container.inspect,
+          this.callback
+        )
+      })
+
+      it('should call back with error', function() {
+        sinon.assert.calledWith(this.callback, this.error)
+      })
+    })
+
+    describe('when inspect fails once with EPIPE error', function() {
+      beforeEach(function() {
+        this.container.inspect = sinon.stub()
+        this.container.inspect.onFirstCall().yields(new Error('write EPIPE'))
+        this.container.inspect.onSecondCall().yields()
+        this.container.start = sinon.stub().yields()
+
+        this.DockerRunner.startContainer(
+          this.options,
+          this.volumes,
+          () => {},
+          this.callback
+        )
+      })
+
+      it('should retry once and start container', function() {
+        sinon.assert.callOrder(
+          this.container.inspect,
+          this.container.inspect,
+          this.DockerRunner.attachToContainer,
+          this.container.start,
+          this.callback
+        )
+      })
+
+      it('should call back without error', function() {
+        sinon.assert.calledWith(this.callback, null)
+      })
+    })
+
     describe('when the container does not exist', function() {
       beforeEach(function() {
         const exists = false