Merge pull request #7880 from overleaf/jk-cm6-parser-migrate-to-web

[web] CM6: Migrate lezer parser to web

GitOrigin-RevId: d6f63ab2f3440292373bcc814c3769285be47c2b

package-lock.json

@@ -4067,6 +4067,19 @@
         "@lezer/lr": "^0.16.0"
       }
     },
+    "node_modules/@lezer/generator": {
+      "version": "0.16.0",
+      "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-0.16.0.tgz",
+      "integrity": "sha512-dhkChgTGXdgj++0VV/Av1ARNe35Fj5uYuVt7W6g/7cDtHToS1dNn73rUtRb1cRO6kQXaISE/ne1cr3VofyBcng==",
+      "dev": true,
+      "dependencies": {
+        "@lezer/common": "^0.16.0",
+        "@lezer/lr": "^0.16.0"
+      },
+      "bin": {
+        "lezer-generator": "dist/lezer-generator.cjs"
+      }
+    },
     "node_modules/@lezer/highlight": {
       "version": "0.16.0",
       "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-0.16.0.tgz",
@@ -35168,6 +35181,7 @@
         "@babel/register": "^7.14.5",
         "@cypress/react": "^5.12.5",
         "@juggle/resize-observer": "^3.3.1",
+        "@lezer/generator": "^0.16.0",
         "@testing-library/cypress": "^8.0.3",
         "@testing-library/dom": "^8.13.0",
         "@testing-library/react": "^12.1.5",
@@ -39606,6 +39620,16 @@
         "@lezer/lr": "^0.16.0"
       }
     },
+    "@lezer/generator": {
+      "version": "0.16.0",
+      "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-0.16.0.tgz",
+      "integrity": "sha512-dhkChgTGXdgj++0VV/Av1ARNe35Fj5uYuVt7W6g/7cDtHToS1dNn73rUtRb1cRO6kQXaISE/ne1cr3VofyBcng==",
+      "dev": true,
+      "requires": {
+        "@lezer/common": "^0.16.0",
+        "@lezer/lr": "^0.16.0"
+      }
+    },
     "@lezer/highlight": {
       "version": "0.16.0",
       "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-0.16.0.tgz",
@@ -42588,6 +42612,7 @@
         "@cypress/react": "^5.12.5",
         "@juggle/resize-observer": "^3.3.1",
         "@lezer/common": "^0.16.0",
+        "@lezer/generator": "^0.16.0",
         "@lezer/highlight": "^0.16.0",
         "@lezer/lr": "^0.16.3",
         "@overleaf/logger": "^3.1.0",

services/web/.eslintignore

@@ -4,3 +4,5 @@ modules/**/scripts
 frontend/js/vendor
 modules/**/frontend/js/vendor
 /public/
+modules/source-editor/frontend/js/lezer-latex/latex.mjs
+modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs

services/web/.gitignore

@@ -92,4 +92,8 @@ cypress/downloads/
 # Ace themes for conversion
 modules/source-editor/frontend/js/themes/ace/
 
+# Compiled parser files
+modules/source-editor/frontend/js/lezer-latex/latex.mjs
+modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs
+
 !**/fixtures/**/*.log

services/web/.prettierignore

@@ -6,3 +6,5 @@ modules/**/frontend/js/vendor
 public/js
 public/minjs
 frontend/stylesheets/components/nvd3.less
+modules/source-editor/frontend/js/lezer-latex/latex.mjs
+modules/source-editor/frontend/js/lezer-latex/latex.terms.mjs

services/web/Dockerfile

@@ -29,6 +29,9 @@ FROM deps as dev
 
 COPY services/web /overleaf/services/web
 
+# Build the latex parser
+RUN cd /overleaf/services/web && npm run 'lezer-latex:generate'
+
 RUN mkdir -p /overleaf/services/web/data/dumpFolder \
 &&  mkdir -p /overleaf/services/web/data/logs \
 &&  mkdir -p /overleaf/services/web/data/pdf \

services/web/package.json

@@ -13,7 +13,7 @@
     "test:unit:all": "npm run test:unit:run_dir -- test/unit/src modules/*/test/unit/src",
     "test:unit:all:silent": "npm run test:unit:all -- --reporter dot",
     "test:unit:app": "npm run test:unit:run_dir -- test/unit/src",
-    "test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,ts,tsx}' test/frontend modules/*/test/frontend",
+    "test:frontend": "NODE_ENV=test TZ=GMT mocha --recursive --timeout 5000 --exit --extension js,mjs,ts,tsx --grep=$MOCHA_GREP --require test/frontend/bootstrap.js --ignore '**/*.spec.{js,ts,tsx}' test/frontend modules/*/test/frontend",
     "test:frontend:coverage": "c8 --all --include 'frontend/js' --include 'modules/*/frontend/js' --exclude 'frontend/js/vendor' --reporter=lcov --reporter=text-summary npm run test:frontend",
     "test:karma": "karma start",
     "test:karma:single": "karma start --no-auto-watch --single-run",
@@ -34,6 +34,8 @@
     "convert-themes": "node modules/source-editor/frontend/js/themes/convert.js",
     "cypress:open-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress open --component",
     "cypress:run-ct": "SHARELATEX_CONFIG=$PWD/config/settings.webpack.js cypress run --component",
+    "lezer-latex:generate": "node modules/source-editor/scripts/lezer-latex/generate.js",
+    "lezer-latex:run": "node modules/source-editor/scripts/lezer-latex/run.mjs",
     "routes": "bin/routes"
   },
   "browserslist": [
@@ -205,6 +207,7 @@
     "@babel/register": "^7.14.5",
     "@cypress/react": "^5.12.5",
     "@juggle/resize-observer": "^3.3.1",
+    "@lezer/generator": "^0.16.0",
     "@testing-library/cypress": "^8.0.3",
     "@testing-library/dom": "^8.13.0",
     "@testing-library/react": "^12.1.5",

services/web/webpack-plugins/lezer-grammar-compiler.js

@@ -0,0 +1,49 @@
+const {
+  compile,
+  options,
+} = require('../modules/source-editor/scripts/lezer-latex/generate')
+const fs = require('fs')
+
+const PLUGIN_NAME = 'lezer-grammar-compiler'
+
+class LezerGrammarCompilerPlugin {
+  apply(compiler) {
+    compiler.hooks.make.tap(PLUGIN_NAME, compilation => {
+      // Add the grammar file to the file paths watched by webpack
+      compilation.fileDependencies.add(options.grammarPath)
+    })
+    compiler.hooks.beforeCompile.tapAsync(
+      PLUGIN_NAME,
+      (_compilation, callback) => {
+        // Check timestamps on grammar and parser files, and re-compile if needed.
+        // (Note: the compiled parser file is watched by webpack, and so will trigger
+        //  a second compilation immediately after. This seems harmless.)
+        if (
+          !fs.existsSync(options.parserOutputPath) ||
+          !fs.existsSync(options.termsOutputPath)
+        ) {
+          console.log('Parser does not exist, compiling')
+          compile()
+          return callback()
+        }
+        fs.stat(options.grammarPath, (err, grammarStat) => {
+          if (err) {
+            return callback(err)
+          }
+          fs.stat(options.parserOutputPath, (err, parserStat) => {
+            if (err) {
+              return callback(err)
+            }
+            callback()
+            if (grammarStat.mtime > parserStat.mtime) {
+              console.log('Grammar file newer than parser file, re-compiling')
+              compile()
+            }
+          })
+        })
+      }
+    )
+  }
+}
+
+module.exports = { LezerGrammarCompilerPlugin }

services/web/webpack.config.js

@@ -4,6 +4,9 @@ const webpack = require('webpack')
 const CopyPlugin = require('copy-webpack-plugin')
 const WebpackAssetsManifest = require('webpack-assets-manifest')
 const MiniCssExtractPlugin = require('mini-css-extract-plugin')
+const {
+  LezerGrammarCompilerPlugin,
+} = require('./webpack-plugins/lezer-grammar-compiler')
 
 const PackageVersions = require('./app/src/infrastructure/PackageVersions')
 
@@ -226,6 +229,8 @@ module.exports = {
   },
 
   plugins: [
+    new LezerGrammarCompilerPlugin(),
+
     // Generate a manifest.json file which is used by the backend to map the
     // base filenames to the generated output filenames
     new WebpackAssetsManifest({