Merge pull request #13805 from overleaf/mj-bibtex-content-outside

[lezer] Allow content outside declarations in bibtex

GitOrigin-RevId: 8051db5567eeb52c3b12f63afafe07a7a97df650

services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar

@@ -1,9 +1,10 @@
 @top Bibliography {
-  (Declaration | StringDeclaration)*
+  (Other | Declaration | CommentDeclaration | PreambleDeclaration | StringDeclaration)*
 }
 
 @tokens {
-  whiteSpace { @whitespace+ }
+  Other { ![@ \t\n] ![@]* }
+  whiteSpace { $[\t\n ]+ }
   Identifier { $[a-zA-Z:_0-9-/]+ }
   StringName { $[a-zA-Z:_] $[a-zA-Z:_0-9-]* }
   FieldName {$[a-zA-Z-_0-9]+}
@@ -13,15 +14,15 @@
   EntryTypeName { $[a-zA-Z]+ }
   Number { @digit+ }
   StringKeyword {"@"$[Ss]$[Tt]$[Rr]$[Ii]$[Nn]$[Gg]}
+  PreambleKeyword {"@"$[Pp]$[Rr]$[Ee]$[Aa]$[Mm]$[Bb]$[Ll]$[Ee]}
+  CommentKeyword {"@"$[Cc]$[Oo]$[Mm]$[Mm]$[Ee]$[Nn]$[Tt]}
+  CommentContents { ![@} \t\n] ![}@]* }
   "{"[closedBy="}"]
   "}"[openedBy="{"]
   "@" "\"" "," "#" "@string"
-  Comment { "%" ![\n]* }
 }
 
-// FIXME: Technically skipping comments is wrong here. They can only appear
-//        alone on a line, but I'm not sure how to express that easily in Lezer
-@skip {whiteSpace | Comment}
+@skip { whiteSpace }
 
 StringDeclaration {
   StringKeyword "{"
@@ -29,6 +30,18 @@ StringDeclaration {
   "}"
 }
 
+PreambleDeclaration {
+  PreambleKeyword "{"
+      Expression
+  "}"
+}
+
+CommentDeclaration {
+  CommentKeyword "{"
+      CommentContents*
+  "}"
+}
+
 Declaration {
   EntryName { "@" EntryTypeName }  "{"
       Identifier

services/web/frontend/js/features/source-editor/lezer-bibtex/highlight.mjs

@@ -10,6 +10,9 @@ export const highlighting = styleTags({
   Expression: t.attributeValue,
   '#': t.operator,
   StringKeyword: t.keyword,
+  PreambleKeyword: t.keyword,
+  CommentKeyword: t.keyword,
+  CommentContents: t.comment,
   StringName: t.variableName,
-  Comment: t.comment,
+  Other: t.comment,
 })