Merge pull request #162 from overleaf/ta-jpa-epipe-retry

[DockerRunner] retry container inspect on EPIPE

services/clsi/app/js/DockerRunner.js

@@ -26,6 +26,7 @@ const LockManager = require('./DockerLockManager')
 const fs = require('fs')
 const Path = require('path')
 const _ = require('underscore')
+const metrics = require('metrics-sharelatex')
 
 logger.info('using docker runner')
 
@@ -411,19 +412,28 @@ module.exports = DockerRunner = {
           })
         }
       )
-    return container.inspect(function(error, stats) {
-      if ((error != null ? error.statusCode : undefined) === 404) {
-        return createAndStartContainer()
-      } else if (error != null) {
-        logger.err(
-          { container_name: name, error },
-          'unable to inspect container to start'
-        )
-        return callback(error)
-      } else {
-        return startExistingContainer()
-      }
-    })
+    var inspectContainer = (isRetry) =>
+      container.inspect(function(error, stats) {
+        if ((error != null ? error.statusCode : undefined) === 404) {
+          return createAndStartContainer()
+        } else if (error != null) {
+          if (error.message.match(/EPIPE/)) {
+            if (!isRetry) {
+              metrics.inc('container-inspect-epipe-retry')
+              return inspectContainer(true)
+            }
+            metrics.inc('container-inspect-epipe-error')
+          }
+          logger.err(
+            { container_name: name, error },
+            'unable to inspect container to start'
+          )
+          return callback(error)
+        } else {
+          return startExistingContainer()
+        }
+      })
+    inspectContainer(false)
   },
 
   attachToContainer(containerId, attachStreamHandler, attachStartCallback) {

services/clsi/test/unit/js/DockerRunnerTests.js

@@ -36,6 +36,7 @@ describe('DockerRunner', function() {
         'logger-sharelatex': (this.logger = {
           log: sinon.stub(),
           error: sinon.stub(),
+          err: sinon.stub(),
           info: sinon.stub(),
           warn: sinon.stub()
         }),
@@ -357,8 +358,8 @@ describe('DockerRunner', function() {
         return this.DockerRunner.startContainer(
           this.options,
           this.volumes,
-          this.callback,
-          () => {}
+          () => {},
+          this.callback
         )
       })
 
@@ -387,6 +388,63 @@ describe('DockerRunner', function() {
       })
     })
 
+    describe('when inspect always fails with EPIPE error', function() {
+      beforeEach(function() {
+        this.error = new Error('write EPIPE')
+        this.container.inspect = sinon.stub().yields(this.error)
+        this.container.start = sinon.stub().yields()
+
+        this.DockerRunner.startContainer(
+          this.options,
+          this.volumes,
+          () => {},
+          this.callback
+        )
+      })
+
+      it('should retry once', function() {
+        sinon.assert.callOrder(
+          this.container.inspect,
+          this.container.inspect,
+          this.callback
+        )
+      })
+
+      it('should call back with error', function() {
+        sinon.assert.calledWith(this.callback, this.error)
+      })
+    })
+
+    describe('when inspect fails once with EPIPE error', function() {
+      beforeEach(function() {
+        this.container.inspect = sinon.stub()
+        this.container.inspect.onFirstCall().yields(new Error('write EPIPE'))
+        this.container.inspect.onSecondCall().yields()
+        this.container.start = sinon.stub().yields()
+
+        this.DockerRunner.startContainer(
+          this.options,
+          this.volumes,
+          () => {},
+          this.callback
+        )
+      })
+
+      it('should retry once and start container', function() {
+        sinon.assert.callOrder(
+          this.container.inspect,
+          this.container.inspect,
+          this.DockerRunner.attachToContainer,
+          this.container.start,
+          this.callback
+        )
+      })
+
+      it('should call back without error', function() {
+        sinon.assert.calledWith(this.callback, null)
+      })
+    })
+
     describe('when the container does not exist', function() {
       beforeEach(function() {
         const exists = false