[web+ops+references] Remove references service code (#29699)

GitOrigin-RevId: 76656abe2e7b012355438d053a05e157bb719b0a

package-lock.json

@@ -27,7 +27,6 @@
         "services/notifications",
         "services/project-history",
         "services/real-time",
-        "services/references",
         "services/templates",
         "services/third-party-datastore",
         "services/third-party-references",
@@ -16084,10 +16083,6 @@
       "resolved": "libraries/redis-wrapper",
       "link": true
     },
-    "node_modules/@overleaf/references": {
-      "resolved": "services/references",
-      "link": true
-    },
     "node_modules/@overleaf/saas-e2e": {
       "resolved": "tools/saas-e2e",
       "link": true
@@ -59877,33 +59872,6 @@
         "url": "https://github.com/sponsors/eemeli"
       }
     },
-    "services/references": {
-      "name": "@overleaf/references",
-      "dependencies": {
-        "@overleaf/fetch-utils": "*",
-        "@overleaf/logger": "*",
-        "@overleaf/metrics": "*",
-        "@overleaf/o-error": "*",
-        "@overleaf/redis-wrapper": "*",
-        "@overleaf/settings": "*",
-        "async": "^3.2.5",
-        "bunyan": "^1.8.15",
-        "express": "4.22.1",
-        "ioredis": "^4.16.1",
-        "lodash": "^4.17.19"
-      },
-      "devDependencies": {
-        "chai": "^4.3.6",
-        "chai-as-promised": "^7.1.1",
-        "esmock": "^2.6.9",
-        "mocha": "^11.1.0",
-        "mocha-junit-reporter": "^2.2.1",
-        "mocha-multi-reporters": "^1.5.1",
-        "mongodb": "6.12.0",
-        "sinon": "^9.2.4",
-        "typescript": "^5.0.4"
-      }
-    },
     "services/templates": {
       "name": "@overleaf/templates",
       "dependencies": {

package.json

@@ -70,7 +70,6 @@
     "services/notifications",
     "services/project-history",
     "services/real-time",
-    "services/references",
     "services/templates",
     "services/third-party-datastore",
     "services/third-party-references",

server-ce/config/settings.js

@@ -140,7 +140,6 @@ const settings = {
     api: redisConfig,
     pubsub: redisConfig,
     project_history: redisConfig,
-    references: redisConfig,
 
     project_history_migration: {
       host: redisConfig.host,
@@ -293,7 +292,6 @@ const settings = {
       ),
     },
   },
-  references: {},
   notifications: undefined,
 
   defaultFeatures: {
@@ -426,14 +424,6 @@ if (
   }
 }
 
-// /References
-// -----------
-if (process.env.OVERLEAF_ELASTICSEARCH_URL != null) {
-  settings.references.elasticsearch = {
-    host: process.env.OVERLEAF_ELASTICSEARCH_URL,
-  }
-}
-
 // filestore
 switch (process.env.OVERLEAF_FILESTORE_BACKEND) {
   case 's3':

services/web/app/src/infrastructure/Features.mjs

@@ -72,8 +72,6 @@ const Features = {
       case 'affiliations':
       case 'analytics':
         return Boolean(_.get(Settings, ['apis', 'v1', 'url']))
-      case 'references':
-        return Boolean(_.get(Settings, ['apis', 'references', 'url']))
       case 'saml':
         return Boolean(Settings.enableSaml)
       case 'linked-project-file':

services/web/frontend/js/features/ide-react/references/bib2json.js

@@ -59,6 +59,9 @@ function BibtexParser(arg0, allowedKeys) {
   this.ALLOWEDKEYS_ = allowedKeys || []
   this.reset_(arg0)
   this.initMacros_()
+  // Force parsing of all entries, even if they arent in our list of ENTRY_TYPES since new types are being created,
+  // and users can create their own custom type anyways
+  this.FORCE_PARSING = true
   return this
 }
 
@@ -131,6 +134,9 @@ BibtexParser.prototype.reset_ = function (arg0) {
   mvproceedings: 39,
   mvreference: 40,
   online: 41,
+  webpage: 41,
+  electronic: 41,
+  www: 41,
   patent: 42,
   performance: 43,
   reference: 44,
@@ -335,7 +341,7 @@ BibtexParser.prototype.processCharacter_ = function (c) {
               if (ot == 'preamble') {
                 this.STATE_ = this.STATES_.ENTRY_OR_JUNK
               } else {
-                if (ot in this.ENTRY_TYPES_) {
+                if (ot in this.ENTRY_TYPES_ || this.FORCE_PARSING) {
                   // SUCCESS:     Parsed a valid object type.
                   // NEXT_STATE:  ENTRY_KEY
                   this.DATA_.ObjectType = 'entry'

services/web/frontend/js/features/ide-react/references/types.ts

@@ -6,7 +6,7 @@ export type Bib2JsonEntry = {
     journal: string
     title: string
     year: string
-  }
+  } & Record<string, string>
 }
 
 export type AdvancedReferenceSearchResult = {

services/web/test/frontend/features/ide-react/unit/references/bib2json.spec.ts

@@ -0,0 +1,201 @@
+import { Bib2JsonEntry } from '@/features/ide-react/references/types'
+import BibtexParserImport from '../../../../../../frontend/js/features/ide-react/references/bib2json'
+import { expect } from 'chai'
+
+const BibtexParser = BibtexParserImport as unknown as (
+  content: string,
+  allowedKeys?: string[]
+) => {
+  entries: Bib2JsonEntry[]
+  errors: any[]
+}
+
+describe('Bib2JsonTests', function () {
+  describe('Upstream', function () {
+    // pulled in from the bib2json repository
+    // https://github.com/mayanklahiri/bib2json/blob/3d7c1f0d738c07d0e1c9a59f4cb9b96c74deb744/test/spec/ParserSpec.js
+    // Author: Mayank Lahiri <mlahiri@gmail.com>
+    // License: BSD-2-Clause
+    it('Parse an empty string without errors or results', function () {
+      const result = BibtexParser('  \t\t\n   \n\n')
+      expect(result.entries.length).to.equal(0)
+      expect(result.errors.length).to.equal(0)
+    })
+
+    it('Parse braces within braces', function () {
+      const result = BibtexParser(
+        '@book { pollock, title={{{A}} very {Big} Book.} }'
+      )
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('A very Big Book.')
+    })
+
+    it('Parse braces within quotes', function () {
+      const result = BibtexParser(
+        '@book { pollock, title="{A} very {Big} Book."}'
+      )
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('A very Big Book.')
+    })
+
+    it('Parse quotes within braces', function () {
+      const result = BibtexParser(
+        '@book { pollock, title="{A} very {"Big"} Book."}'
+      )
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('A very "Big" Book.')
+    })
+
+    it('Respect backslashes', function () {
+      const text = '@book { pollock, title="{A} \\\\very \\{{Big} \\"Book\\"."}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('A \\very {Big "Book".')
+    })
+
+    it('Convert some Latex characters to UTF-8', function () {
+      const result = BibtexParser(
+        '@book { pollock, title="\\"{o}\\AA \\^{I}\\alpha " }'
+      )
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal(
+        '\u00f6\u00c5\u00ce\u03b1'
+      )
+    })
+
+    it('Expand a predefined macro in the middle an entry', function () {
+      const text = '@book{ pollock, month   = jan, title="A title!" }'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.month).to.equal('January')
+    })
+
+    return it('Expand a macro at the end of an entry', function () {
+      const text =
+        '@string  \n{ howdy = "well, hello!" }@book{ pollock, title=howdy }'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('well, hello!')
+    })
+  })
+
+  describe('normalize keys to lower case', function () {
+    it('should lower the case of RaNDoM', function () {
+      const text = '@book{ id, RaNDoM = "VALUE" }'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields).to.deep.equal({ random: 'VALUE' })
+    })
+
+    return it('should preserve the marco usage of RaNDoM', function () {
+      const text = '@string{ RaNDoM="MACRO"}@book{ id, Title = RaNDoM }'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields).to.deep.equal({ title: 'MACRO' })
+    })
+  })
+
+  // sometimes imported bib files contain {\\}, {\\\}, {\\\\} etc to replace whitespace in values
+  describe('handles backslash braces from imported bibs', function () {
+    it('handles one backslash {\\}', function () {
+      const text =
+        '@book { pollock, title="{\\}{A} \\\\very \\{{Big} \\"Book\\"."}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('}A \\very {Big "Book".')
+    })
+    it('handles two backslashes {\\\\}', function () {
+      const text =
+        '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('\\A \\very {Big "Book".')
+    })
+    it('handles three backslashes {\\\\\\}', function () {
+      const text =
+        '@book { pollock, title="{\\\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal(
+        '\\}A \\very {Big "Book".'
+      )
+    })
+    it('handles four backslashes {\\\\\\\\}', function () {
+      const text =
+        '@book { pollock, title="{\\\\\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal(
+        '\\\\A \\very {Big "Book".'
+      )
+    })
+    it('keeps parsing future entries after an even amount of backslashes {\\\\}', function () {
+      const text =
+        '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' +
+        '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} secondary \\"Book\\".", author="Muräkämi, Häruki"}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(2)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal('\\A \\very {Big "Book".')
+      expect(result.entries[1].Fields.title).to.equal(
+        '\\A \\very {Big secondary "Book".'
+      )
+      expect(result.entries[1].Fields.author).to.equal('Muräkämi, Häruki')
+    })
+    return it('keeps parsing future entries after an odd amount of backslashes {\\\\\\}', function () {
+      const text =
+        '@book { pollock, title="{\\\\\\}{A} \\\\very \\{{Big} \\"Book\\"."}' +
+        '@book { pollock, title="{\\\\}{A} \\\\very \\{{Big} secondary \\"Book\\".", author="Muräkämi, Häruki"}'
+      const result = BibtexParser(text)
+      expect(result.entries.length).to.equal(2)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields.title).to.equal(
+        '\\}A \\very {Big "Book".'
+      )
+      expect(result.entries[1].Fields.title).to.equal(
+        '\\A \\very {Big secondary "Book".'
+      )
+      expect(result.entries[1].Fields.author).to.equal('Muräkämi, Häruki')
+    })
+  })
+
+  return describe('with allowedKeys set', function () {
+    it('should skip the unknown key Random', function () {
+      const text = '@book{ id, Random = "ABC", title="VALUE" }'
+      const result = BibtexParser(text, ['title'])
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields).to.deep.equal({ title: 'VALUE' })
+    })
+
+    it('should still parse unknown entry types', function () {
+      const text = '@myCustomType{ id, Random = "ABC", title="VALUE" }'
+      const result = BibtexParser(text, ['title'])
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields).to.deep.equal({ title: 'VALUE' })
+    })
+
+    return it('should preserve the marco usage of RaNDoM', function () {
+      const text =
+        '@string{ RaNDoM="MACRO"}@book{ id, Random = "ABC", Title = RaNDoM }'
+      const result = BibtexParser(text, ['title'])
+      expect(result.entries.length).to.equal(1)
+      expect(result.errors.length).to.equal(0)
+      expect(result.entries[0].Fields).to.deep.equal({ title: 'MACRO' })
+    })
+  })
+})

services/web/test/unit/src/infrastructure/Features.test.mjs

@@ -59,7 +59,6 @@ describe('Features', function () {
         expect(ctx.Features.hasFeature('link-url')).to.be.false
         expect(ctx.Features.hasFeature('oauth')).to.be.false
         expect(ctx.Features.hasFeature('saas')).to.be.false
-        expect(ctx.Features.hasFeature('references')).to.be.false
         expect(ctx.Features.hasFeature('saml')).to.be.false
         expect(ctx.Features.hasFeature('templates-server-pro')).to.be.false
       })
@@ -82,7 +81,6 @@ describe('Features', function () {
           expect(ctx.Features.hasFeature('homepage')).to.be.false
           expect(ctx.Features.hasFeature('link-url')).to.be.false
           expect(ctx.Features.hasFeature('oauth')).to.be.false
-          expect(ctx.Features.hasFeature('references')).to.be.false
           expect(ctx.Features.hasFeature('saml')).to.be.false
           expect(ctx.Features.hasFeature('templates-server-pro')).to.be.false
         })
@@ -104,7 +102,6 @@ describe('Features', function () {
             expect(ctx.Features.hasFeature('affiliations')).to.be.true
             expect(ctx.Features.hasFeature('analytics')).to.be.true
             expect(ctx.Features.hasFeature('saas')).to.be.true
-            expect(ctx.Features.hasFeature('references')).to.be.true
             expect(ctx.Features.hasFeature('registration')).to.be.true
           })
           it('should return false', function (ctx) {
@@ -135,7 +132,6 @@ describe('Features', function () {
               expect(ctx.Features.hasFeature('link-url')).to.be.true
               expect(ctx.Features.hasFeature('oauth')).to.be.true
               expect(ctx.Features.hasFeature('saas')).to.be.true
-              expect(ctx.Features.hasFeature('references')).to.be.true
               expect(ctx.Features.hasFeature('registration')).to.be.true
               expect(ctx.Features.hasFeature('saml')).to.be.true
             })