Delete files from a directory in parallel

services/filestore/app/js/GcsPersistor.js

@@ -5,6 +5,7 @@ const Stream = require('stream')
 const { Storage } = require('@google-cloud/storage')
 const { callbackify } = require('util')
 const { WriteError, ReadError, NotFoundError } = require('./Errors')
+const asyncPool = require('tiny-async-pool')
 const PersistorHelper = require('./PersistorHelper')
 
 const pipeline = promisify(Stream.pipeline)
@@ -215,9 +216,9 @@ async function deleteDirectory(bucketName, key) {
       .bucket(bucketName)
       .getFiles({ directory: key })
 
-    for (const file of files) {
+    await asyncPool(50, files, async file => {
       await deleteFile(bucketName, file.name)
-    }
+    })
   } catch (err) {
     const error = PersistorHelper.wrapError(
       err,

services/filestore/package-lock.json

@@ -5582,6 +5582,22 @@
         "readable-stream": "2 || 3"
       }
     },
+    "tiny-async-pool": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/tiny-async-pool/-/tiny-async-pool-1.1.0.tgz",
+      "integrity": "sha512-jIglyHF/9QdCC3662m/UMVADE6SlocBDpXdFLMZyiAfrw8MSG1pml7lwRtBMT6L/z4dddAxfzw2lpW2Vm42fyQ==",
+      "requires": {
+        "semver": "^5.5.0",
+        "yaassertion": "^1.0.0"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
+        }
+      }
+    },
     "tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@@ -5980,6 +5996,11 @@
       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
       "dev": true
     },
+    "yaassertion": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/yaassertion/-/yaassertion-1.0.2.tgz",
+      "integrity": "sha512-sBoJBg5vTr3lOpRX0yFD+tz7wv/l2UPMFthag4HGTMPrypBRKerjjS8jiEnNMjcAEtPXjbHiKE0UwRR1W1GXBg=="
+    },
     "yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",

services/filestore/package.json

@@ -36,7 +36,8 @@
     "rimraf": "2.2.8",
     "settings-sharelatex": "^1.1.0",
     "stream-buffers": "~0.2.5",
-    "stream-meter": "^1.0.4"
+    "stream-meter": "^1.0.4",
+    "tiny-async-pool": "^1.1.0"
   },
   "devDependencies": {
     "babel-eslint": "^10.0.3",

services/filestore/test/unit/js/GcsPersistorTests.js

@@ -4,6 +4,7 @@ const { expect } = chai
 const modulePath = '../../../app/js/GcsPersistor.js'
 const SandboxedModule = require('sandboxed-module')
 const { ObjectId } = require('mongodb')
+const asyncPool = require('tiny-async-pool')
 
 const Errors = require('../../../app/js/Errors')
 
@@ -135,6 +136,7 @@ describe('GcsPersistorTests', function() {
         '@google-cloud/storage': { Storage },
         'settings-sharelatex': settings,
         'logger-sharelatex': Logger,
+        'tiny-async-pool': asyncPool,
         './Errors': Errors,
         fs: Fs,
         'stream-meter': Meter,