Merge pull request #15518 from overleaf/jpa-webpack-thread-loader

[web] compile less via thread-loader off the main event loop

GitOrigin-RevId: 44cde525dcbb7caa2db7eff724ba5785c459a013

package-lock.json

@@ -25849,6 +25849,12 @@
         "bignumber.js": "^9.0.0"
       }
     },
+    "node_modules/json-parse-better-errors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz",
+      "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==",
+      "dev": true
+    },
     "node_modules/json-parse-even-better-errors": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
@@ -35116,6 +35122,81 @@
         "node": ">=0.2.6"
       }
     },
+    "node_modules/thread-loader": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/thread-loader/-/thread-loader-4.0.2.tgz",
+      "integrity": "sha512-UOk/KBydsQjh4Ja5kocxDUzhv11KYptHN/h8gdSwo6/MBkYrWqQua6K2qwlpXnCXS9c/uLs8F/JF8rpveF0+fA==",
+      "dev": true,
+      "dependencies": {
+        "json-parse-better-errors": "^1.0.2",
+        "loader-runner": "^4.1.0",
+        "neo-async": "^2.6.2",
+        "schema-utils": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 16.10.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.0.0"
+      }
+    },
+    "node_modules/thread-loader/node_modules/ajv": {
+      "version": "8.12.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
+      "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
+      "dev": true,
+      "dependencies": {
+        "fast-deep-equal": "^3.1.1",
+        "json-schema-traverse": "^1.0.0",
+        "require-from-string": "^2.0.2",
+        "uri-js": "^4.2.2"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/epoberezkin"
+      }
+    },
+    "node_modules/thread-loader/node_modules/ajv-keywords": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz",
+      "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==",
+      "dev": true,
+      "dependencies": {
+        "fast-deep-equal": "^3.1.3"
+      },
+      "peerDependencies": {
+        "ajv": "^8.8.2"
+      }
+    },
+    "node_modules/thread-loader/node_modules/json-schema-traverse": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+      "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+      "dev": true
+    },
+    "node_modules/thread-loader/node_modules/schema-utils": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.2.0.tgz",
+      "integrity": "sha512-L0jRsrPpjdckP3oPug3/VxNKt2trR8TcabrM6FOAAlvC/9Phcmm+cuAgTlxBqdBR1WJx7Naj9WHw+aOmheSVbw==",
+      "dev": true,
+      "dependencies": {
+        "@types/json-schema": "^7.0.9",
+        "ajv": "^8.9.0",
+        "ajv-formats": "^2.1.1",
+        "ajv-keywords": "^5.1.0"
+      },
+      "engines": {
+        "node": ">= 12.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      }
+    },
     "node_modules/thriftrw": {
       "version": "3.12.0",
       "resolved": "https://registry.npmjs.org/thriftrw/-/thriftrw-3.12.0.tgz",
@@ -39698,6 +39779,7 @@
         "socket.io-mock": "^1.3.1",
         "strict-event-emitter": "^0.5.1",
         "terser-webpack-plugin": "^5.3.9",
+        "thread-loader": "^4.0.2",
         "timekeeper": "^2.2.0",
         "to-string-loader": "^1.2.0",
         "typescript": "^5.0.4",
@@ -47921,6 +48003,7 @@
         "socket.io-mock": "^1.3.1",
         "strict-event-emitter": "^0.5.1",
         "terser-webpack-plugin": "^5.3.9",
+        "thread-loader": "^4.0.2",
         "timekeeper": "^2.2.0",
         "to-string-loader": "^1.2.0",
         "tough-cookie": "^4.0.0",
@@ -62374,6 +62457,12 @@
         "bignumber.js": "^9.0.0"
       }
     },
+    "json-parse-better-errors": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz",
+      "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==",
+      "dev": true
+    },
     "json-parse-even-better-errors": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz",
@@ -69720,6 +69809,59 @@
       "resolved": "https://registry.npmjs.org/thirty-two/-/thirty-two-1.0.2.tgz",
       "integrity": "sha1-TKL//AKlEpDSdEueP1V2k8prYno="
     },
+    "thread-loader": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/thread-loader/-/thread-loader-4.0.2.tgz",
+      "integrity": "sha512-UOk/KBydsQjh4Ja5kocxDUzhv11KYptHN/h8gdSwo6/MBkYrWqQua6K2qwlpXnCXS9c/uLs8F/JF8rpveF0+fA==",
+      "dev": true,
+      "requires": {
+        "json-parse-better-errors": "^1.0.2",
+        "loader-runner": "^4.1.0",
+        "neo-async": "^2.6.2",
+        "schema-utils": "^4.0.1"
+      },
+      "dependencies": {
+        "ajv": {
+          "version": "8.12.0",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
+          "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
+          "dev": true,
+          "requires": {
+            "fast-deep-equal": "^3.1.1",
+            "json-schema-traverse": "^1.0.0",
+            "require-from-string": "^2.0.2",
+            "uri-js": "^4.2.2"
+          }
+        },
+        "ajv-keywords": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-5.1.0.tgz",
+          "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==",
+          "dev": true,
+          "requires": {
+            "fast-deep-equal": "^3.1.3"
+          }
+        },
+        "json-schema-traverse": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+          "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+          "dev": true
+        },
+        "schema-utils": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.2.0.tgz",
+          "integrity": "sha512-L0jRsrPpjdckP3oPug3/VxNKt2trR8TcabrM6FOAAlvC/9Phcmm+cuAgTlxBqdBR1WJx7Naj9WHw+aOmheSVbw==",
+          "dev": true,
+          "requires": {
+            "@types/json-schema": "^7.0.9",
+            "ajv": "^8.9.0",
+            "ajv-formats": "^2.1.1",
+            "ajv-keywords": "^5.1.0"
+          }
+        }
+      }
+    },
     "thriftrw": {
       "version": "3.12.0",
       "resolved": "https://registry.npmjs.org/thriftrw/-/thriftrw-3.12.0.tgz",

patches/thread-loader+4.0.2.patch

@@ -0,0 +1,81 @@
+diff --git a/node_modules/thread-loader/dist/WorkerPool.js b/node_modules/thread-loader/dist/WorkerPool.js
+index 4145779..f0ff068 100644
+--- a/node_modules/thread-loader/dist/WorkerPool.js
++++ b/node_modules/thread-loader/dist/WorkerPool.js
+@@ -258,6 +258,19 @@ class PoolWorker {
+           finalCallback();
+           break;
+         }
++      case 'logMessage':
++        {
++          const {
++            data: { loggerName, methodName, args }
++          } = message;
++          const {
++            data: jobData
++          } = this.jobs[id];
++          const logger = jobData.getLogger(loggerName);
++          logger[methodName].apply(logger, args);
++          finalCallback();
++          break;
++        }
+       case 'emitWarning':
+         {
+           const {
+diff --git a/node_modules/thread-loader/dist/index.js b/node_modules/thread-loader/dist/index.js
+index 75cd30f..d834af6 100644
+--- a/node_modules/thread-loader/dist/index.js
++++ b/node_modules/thread-loader/dist/index.js
+@@ -43,6 +43,7 @@ function pitch() {
+     sourceMap: this.sourceMap,
+     emitError: this.emitError,
+     emitWarning: this.emitWarning,
++    getLogger: this.getLogger,
+     loadModule: this.loadModule,
+     resolve: this.resolve,
+     getResolve: this.getResolve,
+diff --git a/node_modules/thread-loader/dist/worker.js b/node_modules/thread-loader/dist/worker.js
+index 8e67959..aca94f1 100644
+--- a/node_modules/thread-loader/dist/worker.js
++++ b/node_modules/thread-loader/dist/worker.js
+@@ -90,6 +90,22 @@ function writeJson(data) {
+   writePipeWrite(lengthBuffer);
+   writePipeWrite(messageBuffer);
+ }
++const LOGGER_METHODS = ['error', 'warn', 'info', 'log', 'debug', 'trace', 'group', 'groupEnd', 'groupCollapsed', 'status', 'clear', 'profile', 'profileEnd'];
++class Logger {
++  constructor(id, loggerName) {
++    this.id = id
++    this.loggerName = loggerName
++    for (const methodName of LOGGER_METHODS) {
++      this[methodName] = (...args) => {
++        writeJson({
++          type: 'logMessage',
++          id: this.id,
++          data: { loggerName, methodName, args }
++        })
++      }
++    }
++  }
++}
+ const queue = (0, _queue.default)(({
+   id,
+   data
+@@ -190,6 +206,7 @@ const queue = (0, _queue.default)(({
+           }
+           return options;
+         },
++        getLogger: (name) => new Logger(id, name),
+         emitWarning: warning => {
+           writeJson({
+             type: 'emitWarning',
+@@ -211,6 +228,9 @@ const queue = (0, _queue.default)(({
+           module._compile(code, filename); // eslint-disable-line no-underscore-dangle
+           return module.exports;
+         },
++        addDependency: filename => {
++          buildDependencies.push(filename);
++        },
+         addBuildDependency: filename => {
+           buildDependencies.push(filename);
+         },

services/web/package.json

@@ -332,6 +332,7 @@
     "socket.io-mock": "^1.3.1",
     "strict-event-emitter": "^0.5.1",
     "terser-webpack-plugin": "^5.3.9",
+    "thread-loader": "^4.0.2",
     "timekeeper": "^2.2.0",
     "to-string-loader": "^1.2.0",
     "typescript": "^5.0.4",

services/web/webpack.config.js

@@ -163,6 +163,19 @@ module.exports = {
               },
             },
           },
+          // Compile Less off the main event loop
+          {
+            loader: 'thread-loader',
+            options: {
+              // keep workers alive for dev-server, and shut them down when not needed
+              poolTimeout:
+                process.env.NODE_ENV === 'development' ? 10 * 60 * 1000 : 500,
+              // bring up more workers after they timed out
+              poolRespawn: true,
+              // limit concurrency (one per entrypoint and let the small includes queue up)
+              workers: 6,
+            },
+          },
           // Compiles the Less syntax to CSS
           { loader: 'less-loader' },
         ],