use requestretry to work around web api timeouts

services/track-changes/app/coffee/WebApiManager.coffee

@@ -1,4 +1,4 @@
-request = require "request"
+request = require "requestretry"  # allow retry on error https://github.com/FGRibreau/node-request-retry
 logger = require "logger-sharelatex"
 Settings = require "settings-sharelatex"
 
@@ -15,6 +15,7 @@ module.exports = WebApiManager =
 		request.get {
 			url: "#{Settings.apis.web.url}#{url}"
 			timeout: MAX_HTTP_REQUEST_LENGTH
+			maxAttempts: 2 # for node-request-retry
 			auth:
 				user: Settings.apis.web.user
 				pass: Settings.apis.web.pass
@@ -28,7 +29,7 @@ module.exports = WebApiManager =
 			if res.statusCode >= 200 and res.statusCode < 300
 				return callback null, body
 			else
-				error = new Error("web returned a non-success status code: #{res.statusCode}")
+				error = new Error("web returned a non-success status code: #{res.statusCode} (attempts: #{res.attempts})")
 				callback error
 
 	getUserInfo: (user_id, callback = (error, userInfo) ->) ->

services/track-changes/npm-shrinkwrap.json

@@ -1360,6 +1360,307 @@
         }
       }
     },
+    "requestretry": {
+      "version": "1.12.0",
+      "from": "requestretry@latest",
+      "resolved": "https://registry.npmjs.org/requestretry/-/requestretry-1.12.0.tgz",
+      "dependencies": {
+        "extend": {
+          "version": "3.0.0",
+          "from": "extend@>=3.0.0 <4.0.0",
+          "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz"
+        },
+        "lodash": {
+          "version": "4.17.4",
+          "from": "lodash@>=4.15.0 <5.0.0",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz"
+        },
+        "request": {
+          "version": "2.81.0",
+          "from": "request@>=2.74.0 <3.0.0",
+          "resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz",
+          "dependencies": {
+            "aws-sign2": {
+              "version": "0.6.0",
+              "from": "aws-sign2@>=0.6.0 <0.7.0",
+              "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
+            },
+            "aws4": {
+              "version": "1.6.0",
+              "from": "aws4@>=1.2.1 <2.0.0",
+              "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz"
+            },
+            "caseless": {
+              "version": "0.12.0",
+              "from": "caseless@>=0.12.0 <0.13.0",
+              "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz"
+            },
+            "combined-stream": {
+              "version": "1.0.5",
+              "from": "combined-stream@>=1.0.5 <1.1.0",
+              "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
+              "dependencies": {
+                "delayed-stream": {
+                  "version": "1.0.0",
+                  "from": "delayed-stream@>=1.0.0 <1.1.0",
+                  "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz"
+                }
+              }
+            },
+            "forever-agent": {
+              "version": "0.6.1",
+              "from": "forever-agent@>=0.6.1 <0.7.0",
+              "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz"
+            },
+            "form-data": {
+              "version": "2.1.2",
+              "from": "form-data@>=2.1.1 <2.2.0",
+              "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.2.tgz",
+              "dependencies": {
+                "asynckit": {
+                  "version": "0.4.0",
+                  "from": "asynckit@>=0.4.0 <0.5.0",
+                  "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz"
+                }
+              }
+            },
+            "har-validator": {
+              "version": "4.2.1",
+              "from": "har-validator@>=4.2.1 <4.3.0",
+              "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz",
+              "dependencies": {
+                "ajv": {
+                  "version": "4.11.5",
+                  "from": "ajv@>=4.9.1 <5.0.0",
+                  "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.5.tgz",
+                  "dependencies": {
+                    "co": {
+                      "version": "4.6.0",
+                      "from": "co@>=4.6.0 <5.0.0",
+                      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz"
+                    },
+                    "json-stable-stringify": {
+                      "version": "1.0.1",
+                      "from": "json-stable-stringify@>=1.0.1 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
+                      "dependencies": {
+                        "jsonify": {
+                          "version": "0.0.0",
+                          "from": "jsonify@>=0.0.0 <0.1.0",
+                          "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz"
+                        }
+                      }
+                    }
+                  }
+                },
+                "har-schema": {
+                  "version": "1.0.5",
+                  "from": "har-schema@>=1.0.5 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz"
+                }
+              }
+            },
+            "hawk": {
+              "version": "3.1.3",
+              "from": "hawk@>=3.1.3 <3.2.0",
+              "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
+              "dependencies": {
+                "hoek": {
+                  "version": "2.16.3",
+                  "from": "hoek@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz"
+                },
+                "boom": {
+                  "version": "2.10.1",
+                  "from": "boom@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz"
+                },
+                "cryptiles": {
+                  "version": "2.0.5",
+                  "from": "cryptiles@>=2.0.0 <3.0.0",
+                  "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz"
+                },
+                "sntp": {
+                  "version": "1.0.9",
+                  "from": "sntp@>=1.0.0 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz"
+                }
+              }
+            },
+            "http-signature": {
+              "version": "1.1.1",
+              "from": "http-signature@>=1.1.0 <1.2.0",
+              "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
+              "dependencies": {
+                "assert-plus": {
+                  "version": "0.2.0",
+                  "from": "assert-plus@>=0.2.0 <0.3.0",
+                  "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz"
+                },
+                "jsprim": {
+                  "version": "1.4.0",
+                  "from": "jsprim@>=1.2.2 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.0.tgz",
+                  "dependencies": {
+                    "assert-plus": {
+                      "version": "1.0.0",
+                      "from": "assert-plus@1.0.0",
+                      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+                    },
+                    "extsprintf": {
+                      "version": "1.0.2",
+                      "from": "extsprintf@1.0.2",
+                      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz"
+                    },
+                    "json-schema": {
+                      "version": "0.2.3",
+                      "from": "json-schema@0.2.3",
+                      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz"
+                    },
+                    "verror": {
+                      "version": "1.3.6",
+                      "from": "verror@1.3.6",
+                      "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz"
+                    }
+                  }
+                },
+                "sshpk": {
+                  "version": "1.11.0",
+                  "from": "sshpk@>=1.7.0 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.11.0.tgz",
+                  "dependencies": {
+                    "asn1": {
+                      "version": "0.2.3",
+                      "from": "asn1@>=0.2.3 <0.3.0",
+                      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz"
+                    },
+                    "assert-plus": {
+                      "version": "1.0.0",
+                      "from": "assert-plus@>=1.0.0 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz"
+                    },
+                    "dashdash": {
+                      "version": "1.14.1",
+                      "from": "dashdash@>=1.12.0 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz"
+                    },
+                    "getpass": {
+                      "version": "0.1.6",
+                      "from": "getpass@>=0.1.1 <0.2.0",
+                      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz"
+                    },
+                    "jsbn": {
+                      "version": "0.1.1",
+                      "from": "jsbn@>=0.1.0 <0.2.0",
+                      "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz"
+                    },
+                    "tweetnacl": {
+                      "version": "0.14.5",
+                      "from": "tweetnacl@>=0.14.0 <0.15.0",
+                      "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz"
+                    },
+                    "jodid25519": {
+                      "version": "1.0.2",
+                      "from": "jodid25519@>=1.0.0 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz"
+                    },
+                    "ecc-jsbn": {
+                      "version": "0.1.1",
+                      "from": "ecc-jsbn@>=0.1.1 <0.2.0",
+                      "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz"
+                    },
+                    "bcrypt-pbkdf": {
+                      "version": "1.0.1",
+                      "from": "bcrypt-pbkdf@>=1.0.0 <2.0.0",
+                      "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz"
+                    }
+                  }
+                }
+              }
+            },
+            "is-typedarray": {
+              "version": "1.0.0",
+              "from": "is-typedarray@>=1.0.0 <1.1.0",
+              "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz"
+            },
+            "isstream": {
+              "version": "0.1.2",
+              "from": "isstream@>=0.1.2 <0.2.0",
+              "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz"
+            },
+            "json-stringify-safe": {
+              "version": "5.0.1",
+              "from": "json-stringify-safe@>=5.0.1 <5.1.0",
+              "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
+            },
+            "mime-types": {
+              "version": "2.1.15",
+              "from": "mime-types@>=2.1.7 <2.2.0",
+              "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz",
+              "dependencies": {
+                "mime-db": {
+                  "version": "1.27.0",
+                  "from": "mime-db@>=1.27.0 <1.28.0",
+                  "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.27.0.tgz"
+                }
+              }
+            },
+            "oauth-sign": {
+              "version": "0.8.2",
+              "from": "oauth-sign@>=0.8.1 <0.9.0",
+              "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz"
+            },
+            "performance-now": {
+              "version": "0.2.0",
+              "from": "performance-now@>=0.2.0 <0.3.0",
+              "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz"
+            },
+            "qs": {
+              "version": "6.4.0",
+              "from": "qs@>=6.4.0 <6.5.0",
+              "resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz"
+            },
+            "safe-buffer": {
+              "version": "5.0.1",
+              "from": "safe-buffer@>=5.0.1 <6.0.0",
+              "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.0.1.tgz"
+            },
+            "stringstream": {
+              "version": "0.0.5",
+              "from": "stringstream@>=0.0.4 <0.1.0",
+              "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz"
+            },
+            "tough-cookie": {
+              "version": "2.3.2",
+              "from": "tough-cookie@>=2.3.0 <2.4.0",
+              "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz",
+              "dependencies": {
+                "punycode": {
+                  "version": "1.4.1",
+                  "from": "punycode@>=1.4.1 <2.0.0",
+                  "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz"
+                }
+              }
+            },
+            "tunnel-agent": {
+              "version": "0.6.0",
+              "from": "tunnel-agent@>=0.6.0 <0.7.0",
+              "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz"
+            },
+            "uuid": {
+              "version": "3.0.1",
+              "from": "uuid@>=3.0.0 <4.0.0",
+              "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.0.1.tgz"
+            }
+          }
+        },
+        "when": {
+          "version": "3.7.8",
+          "from": "when@>=3.7.7 <4.0.0",
+          "resolved": "https://registry.npmjs.org/when/-/when-3.7.8.tgz"
+        }
+      }
+    },
     "s3-streams": {
       "version": "0.3.0",
       "from": "https://registry.npmjs.org/s3-streams/-/s3-streams-0.3.0.tgz",

services/track-changes/package.json

@@ -18,6 +18,7 @@
     "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.5.6",
     "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex.git#v1.5.0",
     "request": "~2.33.0",
+    "requestretry": "^1.12.0",
     "redis-sharelatex": "~0.0.9",
     "redis": "~0.10.1",
     "underscore": "~1.7.0",

services/track-changes/test/unit/coffee/WebApiManager/WebApiManagerTests.coffee

@@ -8,7 +8,7 @@ SandboxedModule = require('sandboxed-module')
 describe "WebApiManager", ->
 	beforeEach ->
 		@WebApiManager = SandboxedModule.require modulePath, requires:
-			"request": @request = {}
+			"requestretry": @request = {}
 			"logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() }
 			'settings-sharelatex': @settings =
 				apis: