Merge pull request #34817 from overleaf/em-library-backend-search-2

Library search: make backend search tokens consistent with frontend (re-apply #34643)

GitOrigin-RevId: e5a143edf3226c12eaff77b2bc4cdb68e139bb10

services/web/.storybook/main.ts

@@ -121,6 +121,7 @@ export default defineMain({
           ...storybookConfig.resolve?.alias,
           // custom prefixes for import paths
           '@': path.join(rootDir, 'frontend/js/'),
+          '@modules': path.join(rootDir, 'modules/'),
           '@ol-types': path.join(rootDir, 'types/'),
           '@ol-storybook': path.join(rootDir, '.storybook/'),
           '@wf': path.join(

services/web/scripts/backfill_library_references_search.mjs

@@ -2,13 +2,17 @@
 import minimist from 'minimist'
 import logger from '@overleaf/logger'
 import { db } from '../app/src/infrastructure/mongodb.mjs'
-import { buildSearchFields } from '../modules/library/app/src/LibraryReferenceRepository.mts'
+import {
+  buildSearchTokens,
+  docSchema,
+} from '../modules/library/app/src/LibraryReferenceRepository.mts'
+import { tokenize } from '../modules/library/app/src/bibtex-search-tokens.mts'
 import { scriptRunner } from './lib/ScriptRunner.mjs'
 
 /** @typedef {import('mongodb').AnyBulkWriteOperation} AnyBulkWriteOperation */
 
 const argv = minimist(process.argv.slice(2), {
-  boolean: ['commit', 'rollback', 'all', 'help'],
+  boolean: ['commit', 'all', 'help'],
   default: { 'batch-size': 1000 },
 })
 
@@ -17,16 +21,15 @@ function usage() {
     {},
     `Usage: node backfill_library_references_search.mjs [options]
 
-Populates searchKey and fields.searchValue on libraryReferences so the
-account-level library search can index them. Safe to rerun; picks up only
-un-indexed rows by default.
+Populates searchKey and searchTokens on libraryReferences so the
+account-level library search can index them. Also unsets the obsolete
+fields.$[].searchValue. Safe to rerun; picks up only un-indexed rows
+by default.
 
 Options:
   --commit          Apply changes. Without this, runs as a dry run.
-  --rollback        Unset searchKey and fields.searchValue on all rows that
-                    have them. Mirrors the original migration's rollback.
-  --all             Re-index every row, not just rows where searchKey is null.
-                    Use when the tokenization format has changed.
+  --all             Re-index every row, not just rows where searchTokens
+                    is null. Use when the tokenization format has changed.
   --batch-size <n>  bulkWrite batch size (default 1000).
 `
   )
@@ -41,11 +44,10 @@ const BATCH_SIZE = Number(argv['batch-size'])
 
 /** @param {(message: string) => Promise<void>} trackProgress */
 async function backfill(trackProgress) {
-  const filter = argv.all ? {} : { searchKey: null }
+  const filter = argv.all ? {} : { searchTokens: null }
   const cursor = db.libraryReferences
     .find(filter)
-    .hint({ userId: 1, searchKey: 1 })
-    .project({ key: 1, fields: 1 })
+    .project({ key: 1, type: 1, fields: 1, updatedAt: 1 })
 
   let processed = 0
   /** @type {AnyBulkWriteOperation[]} */
@@ -64,19 +66,20 @@ async function backfill(trackProgress) {
   }
 
   for await (const doc of cursor) {
-    const { searchKey, fields } = buildSearchFields({
-      key: doc.key,
-      fields: (doc.fields ?? []).map(
-        (/** @type {{ name: string; editableValue?: string }} */ f) => ({
-          name: f.name,
-          editableValue: f.editableValue ?? '',
-        })
-      ),
+    const entry = docSchema.parse({
+      ...doc,
+      type: doc.type ?? 'misc',
+      updatedAt: doc.updatedAt ?? new Date(0),
     })
+    const searchKey = tokenize(doc.key)
+    const searchTokens = buildSearchTokens(entry)
     ops.push({
       updateOne: {
         filter: { _id: doc._id },
-        update: { $set: { searchKey, fields } },
+        update: {
+          $set: { searchKey, searchTokens },
+          $unset: { 'fields.$[].searchValue': 1 },
+        },
       },
     })
     if (ops.length >= BATCH_SIZE) {
@@ -87,39 +90,17 @@ async function backfill(trackProgress) {
   await trackProgress(`done; processed ${processed} docs`)
 }
 
-/** @param {(message: string) => Promise<void>} trackProgress */
-async function rollback(trackProgress) {
-  if (!argv.commit) {
-    const count = await db.libraryReferences.countDocuments({
-      searchKey: { $ne: null },
-    })
-    await trackProgress(`[dry-run] would unset search fields on ${count} docs`)
-    return
-  }
-  const result = await db.libraryReferences.updateMany(
-    { searchKey: { $ne: null } },
-    { $unset: { searchKey: 1, 'fields.$[].searchValue': 1 } },
-    { hint: { userId: 1, searchKey: 1 } }
-  )
-  await trackProgress(`unset search fields on ${result.modifiedCount} docs`)
-}
-
 /** @param {(message: string) => Promise<void>} trackProgress */
 async function main(trackProgress) {
   if (!argv.commit) {
     await trackProgress('DRY RUN. Pass --commit to apply changes.')
   }
-  if (argv.rollback) {
-    await rollback(trackProgress)
-  } else {
-    await backfill(trackProgress)
-  }
+  await backfill(trackProgress)
 }
 
 try {
   await scriptRunner(main, {
     commit: Boolean(argv.commit),
-    rollback: Boolean(argv.rollback),
     all: Boolean(argv.all),
     batchSize: BATCH_SIZE,
   })

services/web/tsconfig.backend.json

@@ -6,6 +6,7 @@
   "include": [
     "app/src/**/*",
     "modules/*/app/src/**/*",
+    "modules/*/shared/**/*",
     "modules/*/test/acceptance/**/*",
     "modules/*/test/unit/**/*",
     "scripts/**/*",

services/web/tsconfig.json

@@ -14,6 +14,7 @@
     "forceConsistentCasingInFileNames": true,
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
+    "allowImportingTsExtensions": true,
     "baseUrl": ".",
     "paths": {
       "@/*": ["./frontend/js/*"],
@@ -38,6 +39,7 @@
   "include": [
     "frontend/js/**/*.*",
     "modules/**/frontend/js/**/*.*",
+    "modules/*/shared/**/*.*",
     "test/frontend/**/*.*",
     "modules/**/test/frontend/**/*.*",
     "frontend/stories/**/*.*",

services/web/webpack.config.js

@@ -142,7 +142,7 @@ module.exports = {
       {
         // Pass application JS/TS files through babel-loader,
         // transpiling to targets defined in browserslist
-        test: /\.([jt]sx?|[cm]js)$/,
+        test: /\.([jt]sx?|[cm][jt]s)$/,
         // Only compile application files and specific dependencies
         // (other npm and vendored dependencies must be in ES5 already)
         exclude: [

tools/migrations/20260612120000_replace_libraryReferences_searchTokens_index.mjs

@@ -0,0 +1,26 @@
+import Helpers from './lib/helpers.mjs'
+
+const tags = ['saas']
+
+const newIndexes = [
+  {
+    key: { userId: 1, searchTokens: 1 },
+    name: 'userId_1_searchTokens_1',
+  },
+]
+
+const migrate = async client => {
+  const { db } = client
+  await Helpers.addIndexesToCollection(db.libraryReferences, newIndexes)
+}
+
+const rollback = async client => {
+  const { db } = client
+  await Helpers.dropIndexesFromCollection(db.libraryReferences, newIndexes)
+}
+
+export default {
+  tags,
+  migrate,
+  rollback,
+}