Merge pull request #21105 from overleaf/ar-es-module-helper-scripts

[web] ES module helper scripts

GitOrigin-RevId: ebbb123b6e7336215038156b83e96ad42869bb07

services/web/package.json

@@ -258,6 +258,7 @@
     "@uppy/utils": "^5.7.0",
     "@uppy/xhr-upload": "^3.6.0",
     "abort-controller": "^3.0.0",
+    "5to6-codemod": "^1.8.0",
     "acorn": "^7.1.1",
     "acorn-walk": "^7.1.1",
     "algoliasearch": "^3.35.1",
@@ -308,6 +309,7 @@
     "html-webpack-plugin": "^5.5.3",
     "i18next-scanner": "^4.4.0",
     "jquery": "^3.7.1",
+    "jscodeshift": "^17.0.0",
     "jsdom": "^19.0.0",
     "jsdom-global": "^3.0.2",
     "less": "^3.13.1",

services/web/transform/cjs-to-esm/cjs-to-esm.mjs

@@ -0,0 +1,74 @@
+import minimist from 'minimist'
+import { exec } from 'node:child_process'
+import { promisify } from 'node:util'
+import Runner from 'jscodeshift/src/Runner.js'
+import { fileURLToPath } from 'node:url'
+import path from 'node:path'
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url))
+// use minimist to get a list of files from the argv
+const argv = minimist(process.argv.slice(2), {
+  boolean: ['usage'],
+})
+
+function printUsage() {
+  console.log(
+    'node scripts/esm-migration/cjs-to-esm.mjs [files] [--format] [--lint] [--usage]'
+  )
+  console.log(
+    'WARNING: this will only work in local development as important dependencies will be missing in production'
+  )
+  console.log('Options:')
+  console.log('   files: a list of files to convert')
+  console.log('--format: run prettier to fix formatting')
+  console.log('  --lint: run eslint to fix linting')
+  console.log(' --usage: show this help message')
+  process.exit(0)
+}
+
+const files = argv._
+
+if (argv.usage) {
+  printUsage()
+}
+
+if (!Array.isArray(files) || files.length === 0) {
+  console.error('You must provide a list of files to convert')
+  printUsage()
+  process.exit(1)
+}
+
+const promisifiedExec = promisify(exec)
+
+const cjsTransform = fileURLToPath(
+  import.meta.resolve('5to6-codemod/transforms/cjs.js')
+)
+const exportsTransform = fileURLToPath(
+  import.meta.resolve('5to6-codemod/transforms/exports.js')
+)
+const overleafTransform = fileURLToPath(
+  import.meta.resolve('./overleaf-es-codemod.js')
+)
+
+const config = {
+  output: __dirname,
+  silent: true,
+  print: false,
+  verbose: 0,
+  hoist: true,
+}
+
+await Runner.run(cjsTransform, files, config)
+await Runner.run(exportsTransform, files, config)
+await Runner.run(overleafTransform, files, config)
+
+const webRoot = fileURLToPath(new URL('../../', import.meta.url))
+
+for (const file of files) {
+  // move files with git mv
+  await promisifiedExec(`git mv ${file} ${file.replace('.js', '.mjs')}`)
+  const relativePath = path.relative(webRoot, file)
+  console.log(
+    `transformed ${relativePath} and renamed it to have a .mjs extension`
+  )
+}

services/web/transform/cjs-to-esm/overleaf-es-codemod.js

@@ -0,0 +1,230 @@
+// Performs a few useful codemod transformations for Overleaf's esm migration.
+// The transformations mostly address specific issues faced commonly in Overleaf's `web` service.
+// * Replaces `sandboxed-module` imports with `esmock` imports.
+// * Replaces `sandboxed-module` invocation with `esmock` invocation (Assumes `SandboxedModule.require` is used for the invocation).
+// * Fixes `mongodb-legacy` import to use `mongodb` import and extract `ObjectId` from the import.
+// * Replaces `require('path').join` with `path.join` (importing the path module if not already imported).
+// * Adds `const __dirname = fileURLToPath(new URL('.', import.meta.url))` if `__dirname` is used in the file.
+// * Adds `.js` or `.mjs` extension (as appropriate) to relative path imports.
+// call this with `jscodeshift -t overleaf-es-codemod.js <file>` or using the `cjs-to-esm.js` script (which does this as the final step before formatting).
+
+const fs = require('node:fs')
+const Path = require('node:path')
+
+module.exports = function (fileInfo, api) {
+  const j = api.jscodeshift
+  const root = j(fileInfo.source)
+  const body = root.get().value.program.body
+
+  /**
+   * Conditionally adds an import statement to the top of the file if it doesn't already exist.
+   * @param moduleName A plain text name for the module to import (e.g. 'node:path').
+   * @param specifier A jscodeshift specifier for the import statement (provides e.g. `{ promises }` from `import { promises } from 'fs'`.
+   * @param existingImportCheck A function that checks if a specific import statement is the one we're looking for.
+   */
+  function addImport(moduleName, specifier, existingImportCheck) {
+    // Add import path from 'path' at the top if not already present
+    const importDeclaration = j.importDeclaration(
+      specifier,
+      j.literal(moduleName)
+    )
+
+    if (!existingImportCheck) {
+      existingImportCheck = node => node.source.value === moduleName
+    }
+
+    const existingImport = body.find(
+      node => node.type === 'ImportDeclaration' && existingImportCheck(node)
+    )
+
+    if (!existingImport) {
+      const lastImportIndex = body.reduce((lastIndex, node, index) => {
+        return node.type === 'ImportDeclaration' ? index : lastIndex
+      }, -1)
+      body.splice(lastImportIndex, 0, importDeclaration)
+    }
+  }
+
+  // Replace sandboxed-module imports
+  root
+    .find(j.ImportDeclaration, {
+      source: { value: 'sandboxed-module' },
+    })
+    .forEach(path => {
+      path.node.source.value = 'esmock'
+      if (path.node.specifiers.length > 0 && path.node.specifiers[0].local) {
+        path.node.specifiers[0].local.name = 'esmock'
+      }
+    })
+
+  // Replace sandboxedModule.require calls with awaited esmock calls
+  root
+    .find(j.CallExpression, {
+      callee: {
+        object: { name: 'SandboxedModule' },
+        property: { name: 'require' },
+      },
+    })
+    .forEach(path => {
+      const args = path.node.arguments
+      if (args.length > 0) {
+        const firstArg = args[0]
+        const esmockArgs = [firstArg]
+
+        // Check if there's a second argument with a 'requires' property
+        if (args.length > 1 && args[1].type === 'ObjectExpression') {
+          const requiresProp = args[1].properties.find(
+            prop =>
+              prop.key.name === 'requires' || prop.key.value === 'requires'
+          )
+
+          if (requiresProp) {
+            // Move contents of 'requires' to top level
+            esmockArgs.push(requiresProp.value)
+          }
+        }
+
+        // Create the await expression with restructured arguments
+        const awaitExpression = j.awaitExpression(
+          j.callExpression(
+            j.memberExpression(j.identifier('esmock'), j.identifier('strict')),
+            esmockArgs
+          )
+        )
+
+        // Replace the original call with the await expression
+        j(path).replaceWith(awaitExpression)
+
+        // Find the closest function and make it async
+        let functionPath = path
+        while ((functionPath = functionPath.parent)) {
+          if (
+            functionPath.node.type === 'FunctionDeclaration' ||
+            functionPath.node.type === 'FunctionExpression' ||
+            functionPath.node.type === 'ArrowFunctionExpression'
+          ) {
+            functionPath.node.async = true
+            break
+          }
+        }
+      }
+    })
+
+  // Fix mongodb-legacy import
+  root
+    .find(j.ImportDeclaration, {
+      source: { value: 'mongodb-legacy' },
+      specifiers: [{ imported: { name: 'ObjectId' } }],
+    })
+    .forEach(path => {
+      // Create new import declaration
+      const newImport = j.importDeclaration(
+        [j.importDefaultSpecifier(j.identifier('mongodb'))],
+        j.literal('mongodb-legacy')
+      )
+
+      // Create new constant declaration
+      const newConst = j.variableDeclaration('const', [
+        j.variableDeclarator(
+          j.objectPattern([
+            j.property(
+              'init',
+              j.identifier('ObjectId'),
+              j.identifier('ObjectId')
+            ),
+          ]),
+          j.identifier('mongodb')
+        ),
+      ])
+
+      // Replace the old import with the new import and constant declaration
+      j(path).replaceWith(newImport)
+      path.insertAfter(newConst)
+    })
+
+  root
+    .find(j.CallExpression, {
+      callee: {
+        object: { callee: { name: 'require' }, arguments: [{ value: 'path' }] },
+        property: { name: 'join' },
+      },
+    })
+    .forEach(path => {
+      // Replace with path.join
+      j(path).replaceWith(
+        j.callExpression(
+          j.memberExpression(j.identifier('path'), j.identifier('join')),
+          path.node.arguments
+        )
+      )
+
+      // Add import path from 'path' at the top if not already presen
+      addImport(
+        'node:path',
+        [j.importDefaultSpecifier(j.identifier('path'))],
+        node =>
+          node.source.value === 'path' || node.source.value === 'node:path'
+      )
+    })
+
+  // Add const __dirname = fileURLToPath(new URL('.', import.meta.url)) if there is a usage of __dirname
+  const dirnameDeclaration = j.variableDeclaration('const', [
+    j.variableDeclarator(
+      j.identifier('__dirname'),
+      j.callExpression(j.identifier('fileURLToPath'), [
+        j.newExpression(j.identifier('URL'), [
+          j.literal('.'),
+          j.memberExpression(j.identifier('import'), j.identifier('meta.url')),
+        ]),
+      ])
+    ),
+  ])
+
+  const existingDirnameDeclaration = body.find(
+    node =>
+      node.type === 'VariableDeclaration' &&
+      node.declarations[0].id.name === '__dirname'
+  )
+  const firstDirnameUsage = root.find(j.Identifier, { name: '__dirname' }).at(0)
+
+  if (firstDirnameUsage.size() > 0 && !existingDirnameDeclaration) {
+    // Add import path from 'path' at the top if not already present
+    addImport(
+      'node:url',
+      [j.importSpecifier(j.identifier('fileURLToPath'))],
+      node => node.source.value === 'url' || node.source.value === 'node:url'
+    )
+
+    const lastImportIndex = body.reduce((lastIndex, node, index) => {
+      return node.type === 'ImportDeclaration' ? index : lastIndex
+    }, -1)
+
+    body.splice(lastImportIndex + 1, 0, dirnameDeclaration)
+  }
+
+  // Add extension to relative path imports
+  root
+    .find(j.ImportDeclaration)
+    .filter(path => path.node.source.value.startsWith('.'))
+    .forEach(path => {
+      const importPath = path.node.source.value
+      const fullPathJs = Path.resolve(
+        Path.dirname(fileInfo.path),
+        `${importPath}.js`
+      )
+      const fullPathMjs = Path.resolve(
+        Path.dirname(fileInfo.path),
+        `${importPath}.mjs`
+      )
+
+      if (fs.existsSync(fullPathJs)) {
+        path.node.source.value = `${importPath}.js`
+      } else if (fs.existsSync(fullPathMjs)) {
+        path.node.source.value = `${importPath}.mjs`
+      }
+    })
+
+  return root.toSource({
+    quote: 'single',
+  })
+}

services/web/transform/cjs-to-esm/transform-dir.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ -z "$1" ]; then
+  echo "Usage: transform-dir.sh <module_path>"
+  exit 1
+fi
+
+MODULE_PATH=$1
+
+while true; do
+  FILES=$(node scripts/esm-check-migration.mjs -f "$MODULE_PATH" -j | jq -r '.filesNotImportedViaCjs | join(" ")')
+  if [ -z "$FILES" ]; then
+    break
+  fi
+  node transform/cjs-to-esm/cjs-to-esm.mjs $FILES
+done
+
+make format_fix > /dev/null
+
+echo "All files processed."