Merge pull request #13572 from overleaf/mj-bibtex-grammar

[cm6] Add support for bibtex

GitOrigin-RevId: 28bc8e47c53df1612c1e30cf690e893b0bbf500c

package-lock.json

@@ -5796,9 +5796,9 @@
       "devOptional": true
     },
     "node_modules/@lezer/common": {
-      "version": "1.0.2",
+      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.3.tgz",
-      "integrity": "sha512-SVgiGtMnMnW3ActR8SXgsDhw7a0w0ChHSYAyAUxxrOiJ1OqYWEKk/xJd84tTSPo1mo6DXLObAJALNnd0Hrv7Ng=="
+      "integrity": "sha512-JH4wAXCgUOcCGNekQPLhVeUtIqjH0yPBs7vvUdSjyQama9618IOKFJwkv2kcqdhF0my8hQEgCTEJU0GIgnahvA=="
     },
     "node_modules/@lezer/css": {
       "version": "1.0.0",
@@ -5810,22 +5810,22 @@
       }
     },
     "node_modules/@lezer/generator": {
-      "version": "1.1.3",
+      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.3.0.tgz",
-      "integrity": "sha512-qGF0I2TTJ+VBjjsVX8FGqKJy3laALBnVbD5EbXEu13Sgszl/vjnxjcZ69O8w9IK8/WtVFQLspU4UjCCUNRlWzA==",
+      "integrity": "sha512-7HfulDoOMOkskb97fnwgpC6StwPVSob4ptc0iuOH72rapNQBbp6lVj05y7vc5IM0E9pjFjiLmNQeiBiSbLpCtA==",
       "dev": true,
       "dependencies": {
-        "@lezer/common": "^1.0.0",
+        "@lezer/common": "^1.0.2",
-        "@lezer/lr": "^1.0.0"
+        "@lezer/lr": "^1.3.0"
       },
       "bin": {
         "lezer-generator": "dist/lezer-generator.cjs"
       }
     },
     "node_modules/@lezer/highlight": {
-      "version": "1.1.3",
+      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.6.tgz",
-      "integrity": "sha512-3vLKLPThO4td43lYRBygmMY18JN3CPh9w+XS2j8WC30vR4yZeFG4z1iFe4jXE43NtGqe//zHW5q8ENLlHvz9gw==",
+      "integrity": "sha512-cmSJYa2us+r3SePpRCjN5ymCqCPv+zyXmDl0ciWtVaNiORT/MxM7ZgOMQZADD0o51qOaOg24qc/zBViOIwAjJg==",
       "dependencies": {
         "@lezer/common": "^1.0.0"
       }
@@ -5849,17 +5849,17 @@
       }
     },
     "node_modules/@lezer/lr": {
-      "version": "1.3.3",
+      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.3.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.7.tgz",
-      "integrity": "sha512-JPQe3mwJlzEVqy67iQiiGozhcngbO8QBgpqZM6oL1Wj/dXckrEexpBLeFkq0edtW5IqnPRFxA24BHJni8Js69w==",
+      "integrity": "sha512-ssHKb3p0MxhJXT2i7UBmgAY1BIM3Uq/D772Qutu3EVmxWIyNMU12nQ0rL3Fhu+MiFtiTzyTmd3xGwEf3ON5PSA==",
       "dependencies": {
         "@lezer/common": "^1.0.0"
       }
     },
     "node_modules/@lezer/markdown": {
-      "version": "1.0.2",
+      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.3.tgz",
-      "integrity": "sha512-8CY0OoZ6V5EzPjSPeJ4KLVbtXdLBd8V6sRCooN5kHnO28ytreEGTyrtU/zUwo/XLRzGr/e1g44KlzKi3yWGB5A==",
+      "integrity": "sha512-QEcXFCKf1TBdVhmxL2V9afJTIs4w795DTl2NKnsYZyMOtMsA+5AlEy0biPo/Ojv05ELkk6HIPSDBj0g+ShlkBw==",
       "dependencies": {
         "@lezer/common": "^1.0.0",
         "@lezer/highlight": "^1.0.0"
@@ -41266,10 +41266,10 @@
         "@contentful/rich-text-html-renderer": "^16.0.2",
         "@contentful/rich-text-types": "^16.0.2",
         "@google-cloud/bigquery": "^6.0.1",
-        "@lezer/common": "^1.0.2",
+        "@lezer/common": "^1.0.3",
-        "@lezer/highlight": "^1.1.3",
+        "@lezer/highlight": "^1.1.6",
-        "@lezer/lr": "^1.3.3",
+        "@lezer/lr": "^1.3.7",
-        "@lezer/markdown": "^1.0.2",
+        "@lezer/markdown": "^1.0.3",
         "@node-oauth/oauth2-server": "^4.3.0",
         "@opentelemetry/api": "^1.0.4",
         "@opentelemetry/auto-instrumentations-web": "^0.27.2",
@@ -41441,7 +41441,7 @@
       "devDependencies": {
         "@babel/register": "^7.21.0",
         "@juggle/resize-observer": "^3.3.1",
-        "@lezer/generator": "^1.1.3",
+        "@lezer/generator": "^1.3.0",
         "@testing-library/cypress": "^9.0.0",
         "@testing-library/dom": "^9.3.0",
         "@testing-library/react": "^12.1.5",
@@ -46835,9 +46835,9 @@
       "devOptional": true
     },
     "@lezer/common": {
-      "version": "1.0.2",
+      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/common/-/common-1.0.3.tgz",
-      "integrity": "sha512-SVgiGtMnMnW3ActR8SXgsDhw7a0w0ChHSYAyAUxxrOiJ1OqYWEKk/xJd84tTSPo1mo6DXLObAJALNnd0Hrv7Ng=="
+      "integrity": "sha512-JH4wAXCgUOcCGNekQPLhVeUtIqjH0yPBs7vvUdSjyQama9618IOKFJwkv2kcqdhF0my8hQEgCTEJU0GIgnahvA=="
     },
     "@lezer/css": {
       "version": "1.0.0",
@@ -46849,19 +46849,19 @@
       }
     },
     "@lezer/generator": {
-      "version": "1.1.3",
+      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/generator/-/generator-1.3.0.tgz",
-      "integrity": "sha512-qGF0I2TTJ+VBjjsVX8FGqKJy3laALBnVbD5EbXEu13Sgszl/vjnxjcZ69O8w9IK8/WtVFQLspU4UjCCUNRlWzA==",
+      "integrity": "sha512-7HfulDoOMOkskb97fnwgpC6StwPVSob4ptc0iuOH72rapNQBbp6lVj05y7vc5IM0E9pjFjiLmNQeiBiSbLpCtA==",
       "dev": true,
       "requires": {
-        "@lezer/common": "^1.0.0",
+        "@lezer/common": "^1.0.2",
-        "@lezer/lr": "^1.0.0"
+        "@lezer/lr": "^1.3.0"
       }
     },
     "@lezer/highlight": {
-      "version": "1.1.3",
+      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/highlight/-/highlight-1.1.6.tgz",
-      "integrity": "sha512-3vLKLPThO4td43lYRBygmMY18JN3CPh9w+XS2j8WC30vR4yZeFG4z1iFe4jXE43NtGqe//zHW5q8ENLlHvz9gw==",
+      "integrity": "sha512-cmSJYa2us+r3SePpRCjN5ymCqCPv+zyXmDl0ciWtVaNiORT/MxM7ZgOMQZADD0o51qOaOg24qc/zBViOIwAjJg==",
       "requires": {
         "@lezer/common": "^1.0.0"
       }
@@ -46885,17 +46885,17 @@
       }
     },
     "@lezer/lr": {
-      "version": "1.3.3",
+      "version": "1.3.7",
-      "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.3.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/lr/-/lr-1.3.7.tgz",
-      "integrity": "sha512-JPQe3mwJlzEVqy67iQiiGozhcngbO8QBgpqZM6oL1Wj/dXckrEexpBLeFkq0edtW5IqnPRFxA24BHJni8Js69w==",
+      "integrity": "sha512-ssHKb3p0MxhJXT2i7UBmgAY1BIM3Uq/D772Qutu3EVmxWIyNMU12nQ0rL3Fhu+MiFtiTzyTmd3xGwEf3ON5PSA==",
       "requires": {
         "@lezer/common": "^1.0.0"
       }
     },
     "@lezer/markdown": {
-      "version": "1.0.2",
+      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/@lezer/markdown/-/markdown-1.0.3.tgz",
-      "integrity": "sha512-8CY0OoZ6V5EzPjSPeJ4KLVbtXdLBd8V6sRCooN5kHnO28ytreEGTyrtU/zUwo/XLRzGr/e1g44KlzKi3yWGB5A==",
+      "integrity": "sha512-QEcXFCKf1TBdVhmxL2V9afJTIs4w795DTl2NKnsYZyMOtMsA+5AlEy0biPo/Ojv05ELkk6HIPSDBj0g+ShlkBw==",
       "requires": {
         "@lezer/common": "^1.0.0",
         "@lezer/highlight": "^1.0.0"
@@ -50249,11 +50249,11 @@
         "@contentful/rich-text-types": "^16.0.2",
         "@google-cloud/bigquery": "^6.0.1",
         "@juggle/resize-observer": "^3.3.1",
-        "@lezer/common": "^1.0.2",
+        "@lezer/common": "^1.0.3",
-        "@lezer/generator": "^1.1.3",
+        "@lezer/generator": "^1.3.0",
-        "@lezer/highlight": "^1.1.3",
+        "@lezer/highlight": "^1.1.6",
-        "@lezer/lr": "^1.3.3",
+        "@lezer/lr": "^1.3.7",
-        "@lezer/markdown": "^1.0.2",
+        "@lezer/markdown": "^1.0.3",
         "@node-oauth/oauth2-server": "^4.3.0",
         "@opentelemetry/api": "^1.0.4",
         "@opentelemetry/auto-instrumentations-web": "^0.27.2",

services/web/.eslintignore

@@ -6,3 +6,5 @@ modules/**/frontend/js/vendor
 /public/
 frontend/js/features/source-editor/lezer-latex/latex.mjs
 frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
+frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
+frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs

services/web/.gitignore

@@ -94,5 +94,7 @@ frontend/js/features/source-editor/themes/ace/
 # Compiled parser files
 frontend/js/features/source-editor/lezer-latex/latex.mjs
 frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
+frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
+frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs
 
 !**/fixtures/**/*.log

services/web/.prettierignore

@@ -8,3 +8,5 @@ public/minjs
 frontend/stylesheets/components/nvd3.less
 frontend/js/features/source-editor/lezer-latex/latex.mjs
 frontend/js/features/source-editor/lezer-latex/latex.terms.mjs
+frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs
+frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs

services/web/frontend/js/features/source-editor/languages/bibtex/bibtex-language.ts

@@ -0,0 +1,10 @@
+import { LRLanguage } from '@codemirror/language'
+import { parser } from '../../lezer-bibtex/bibtex.mjs'
+import { bibtexEntryCompletions } from './completions/snippets'
+
+export const BibTeXLanguage = LRLanguage.define({
+  parser,
+  languageData: {
+    autocomplete: bibtexEntryCompletions,
+  },
+})

services/web/frontend/js/features/source-editor/languages/bibtex/completions/snippets.ts

@@ -0,0 +1,95 @@
+import { CompletionContext, snippet } from '@codemirror/autocomplete'
+
+type Environment = {
+  name: string
+  requiredAttributes: string[]
+}
+
+const environments: Environment[] = [
+  {
+    name: 'article',
+    requiredAttributes: ['author', 'title', 'journal', 'year'],
+  },
+  {
+    name: 'book',
+    requiredAttributes: ['author', 'title', 'publisher', 'year'],
+  },
+  {
+    name: 'booklet',
+    requiredAttributes: ['key', 'title'],
+  },
+  {
+    name: 'conference',
+    requiredAttributes: ['key', 'title', 'year'],
+  },
+  {
+    name: 'inbook',
+    requiredAttributes: ['author', 'title', 'publisher', 'year', 'chapter'],
+  },
+  {
+    name: 'incollection',
+    requiredAttributes: ['author', 'title', 'booktitle', 'publisher', 'year'],
+  },
+  {
+    name: 'inproceedings',
+    requiredAttributes: ['author', 'title', 'booktitle', 'year'],
+  },
+  {
+    name: 'manual',
+    requiredAttributes: ['key', 'title'],
+  },
+  {
+    name: 'masterthesis',
+    requiredAttributes: ['author', 'title', 'school', 'year'],
+  },
+  {
+    name: 'misc',
+    requiredAttributes: ['key', 'note'],
+  },
+  {
+    name: 'phdthesis',
+    requiredAttributes: ['author', 'title', 'school', 'year'],
+  },
+  {
+    name: 'proceedings',
+    requiredAttributes: ['key', 'title', 'year'],
+  },
+  {
+    name: 'techreport',
+    requiredAttributes: ['author', 'title', 'institution', 'year'],
+  },
+  {
+    name: 'unpublished',
+    requiredAttributes: ['author', 'title', 'note'],
+  },
+]
+
+const prepareSnippet = (environment: Environment) => {
+  return `@${
+    environment.name
+  }{#{citation-key},${environment.requiredAttributes.map(
+    attribute => `
+    ${attribute} = #{}`
+  )}
+}`
+}
+
+export function bibtexEntryCompletions(context: CompletionContext) {
+  const word = context.matchBefore(/@\w*/)
+  if (word?.from === word?.to && !context.explicit) return null
+  return {
+    from: word?.from ?? context.pos,
+    options: [
+      ...environments.map(env => ({
+        label: `@${env.name}`,
+        type: 'snippet',
+        apply: snippet(prepareSnippet(env)),
+      })),
+      {
+        label: '@string',
+        type: 'snippet',
+        apply: snippet('@string{#{string-key} = #{}}'),
+      },
+    ],
+  }
+}

services/web/frontend/js/features/source-editor/languages/bibtex/index.ts

@@ -0,0 +1,8 @@
+import { LanguageSupport, indentUnit } from '@codemirror/language'
+import { BibTeXLanguage } from './bibtex-language'
+
+export const bibtex = () => {
+  return new LanguageSupport(BibTeXLanguage, [
+    indentUnit.of('    '), // 4 spaces
+  ])
+}

services/web/frontend/js/features/source-editor/languages/index.ts

@@ -5,7 +5,6 @@ export const languages = [
     name: 'latex',
     extensions: [
       'tex',
-      'bib',
       'sty',
       'cls',
       'clo',
@@ -50,6 +49,13 @@ export const languages = [
       return import('./latex').then(m => m.latex())
     },
   }),
+  LanguageDescription.of({
+    name: 'bibtex',
+    extensions: ['bib'],
+    load: () => {
+      return import('./bibtex').then(m => m.bibtex())
+    },
+  }),
   LanguageDescription.of({
     name: 'markdown',
     extensions: ['md', 'markdown'],

services/web/frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar

@@ -0,0 +1,85 @@
+
+
+@top Bibliography {
+  (Declaration | StringDeclaration)*
+}
+
+@tokens {
+  whiteSpace { @whitespace+ }
+  Identifier { $[a-zA-Z:_0-9-]+ }
+  StringName { $[a-zA-Z:_] $[a-zA-Z:_0-9-]* }
+  FieldName {$[a-zA-Z-_0-9]+}
+  LiteralString {
+    '"' (!["] | "\\" _)* '"'?
+  }
+  EntryTypeName { $[a-zA-Z]+ }
+  Number { @digit+ }
+  StringKeyword {"@"$[Ss]$[Tt]$[Rr]$[Ii]$[Nn]$[Gg]}
+  "@" "{" "}" "\"" "," "#" "@string"
+  Comment { "%" ![\n]* }
+}
+
+// FIXME: Technically skipping comments is wrong here. They can only appear
+//        alone on a line, but I'm not sure how to express that easily in Lezer
+@skip {whiteSpace | Comment}
+
+StringDeclaration {
+  StringKeyword "{"
+      Field<StringName>*
+  "}"
+}
+
+Declaration {
+  EntryName { "@" EntryTypeName }  "{"
+      Identifier
+      
+      fieldEntry { 
+        (","  Field<FieldName> )
+      }*
+      ("," )?
+  "}"
+}
+
+Field<Name> {
+  Name  "="  Expression
+}
+
+Expression {
+  BracedString |
+  Number | 
+  StringConcatenation
+}
+
+@local tokens {
+  openBracedContents {"{"}
+  closeBracedContents {"}"}
+  @else nonClosingBracedContents
+}
+
+@skip {}{
+  bracedStringContents {
+  (
+      nonClosingBracedContents |
+      nestedBracedString {
+        openBracedContents 
+        bracedStringContents
+        closeBracedContents
+      }
+    )*
+}
+  BracedString {
+    "{"
+    bracedStringContents closeBracedContents
+  }  
+}
+
+// TODO: Implement this
+@precedence { concatenation @left }
+
+StringConcatenation {
+  StringConcatenation !concatenation "#" StringConcatenation |
+  LiteralString |
+  StringName
+}
+
+@external propSource highlighting from "./highlight.mjs"

services/web/frontend/js/features/source-editor/lezer-bibtex/highlight.mjs

@@ -0,0 +1,15 @@
+import { styleTags, tags as t } from '@lezer/highlight'
+
+export const highlighting = styleTags({
+  LiteralString: t.string,
+  'BracedString/...': t.string,
+  Number: t.number,
+  Identifier: t.name,
+  'EntryName/...': t.keyword,
+  FieldName: t.attributeName,
+  Expression: t.attributeValue,
+  '#': t.operator,
+  StringKeyword: t.keyword,
+  StringName: t.variableName,
+  Comment: t.comment,
+})

services/web/frontend/stories/source-editor/source-editor.stories.tsx

@@ -103,6 +103,21 @@ export const Visual = (args: any, { globals: { theme } }: any) => {
   return <SourceEditor />
 }
 
+export const Bibtex = (args: any, { globals: { theme } }: any) => {
+  useScope({
+    editor: {
+      sharejs_doc: mockDoc(content.bib, changes.bib),
+      open_doc_name: 'example.bib',
+    },
+    settings: {
+      ...settings,
+      overallTheme: theme === 'default-' ? '' : theme,
+    },
+  })
+
+  return <SourceEditor />
+}
+
 const MAX_DOC_LENGTH = 2 * 1024 * 1024 // window.maxDocLength
 
 const mockDoc = (content: string, changes: Array<Record<string, any>> = []) => {
@@ -163,6 +178,7 @@ const changes: Record<string, Array<Record<string, any>>> = {
     },
   ],
   md: [],
+  bib: [],
 }
 
 const content = {
@@ -290,4 +306,52 @@ We hope you find Overleaf useful, and do take a look at our \\href{https://www.o
 This is **bold**
 
 This is _italic_`,
+  bib: `@book{texbook,
+  author = {Donald E. Knuth},
+  year = {1986},
+  title = {The {\\TeX} Book},
+  publisher = {Addison-Wesley Professional}
+}
+
+@book{latex:companion,
+  author = {Frank Mittelbach and Michel Gossens
+            and Johannes Braams and David Carlisle
+            and Chris Rowley},
+  year = {2004},
+  title = {The {\\LaTeX} Companion},
+  publisher = {Addison-Wesley Professional},
+  edition = {2}
+}
+
+@book{latex2e,
+  author = {Leslie Lamport},
+  year = {1994},
+  title = {{\\LaTeX}: a Document Preparation System},
+  publisher = {Addison Wesley},
+  address = {Massachusetts},
+  edition = {2}
+}
+
+@article{knuth:1984,
+  title={Literate Programming},
+  author={Donald E. Knuth},
+  journal={The Computer Journal},
+  volume={27},
+  number={2},
+  pages={97--111},
+  year={1984},
+  publisher={Oxford University Press}
+}
+
+@inproceedings{lesk:1977,
+  title={Computer Typesetting of Technical Journals on {UNIX}},
+  author={Michael Lesk and Brian Kernighan},
+  booktitle={Proceedings of American Federation of
+              Information Processing Societies: 1977
+              National Computer Conference},
+  pages={879--888},
+  year={1977},
+  address={Dallas, Texas}
+}
+`,
 }

services/web/package.json

@@ -82,10 +82,10 @@
     "@contentful/rich-text-html-renderer": "^16.0.2",
     "@contentful/rich-text-types": "^16.0.2",
     "@google-cloud/bigquery": "^6.0.1",
-    "@lezer/common": "^1.0.2",
+    "@lezer/common": "^1.0.3",
-    "@lezer/highlight": "^1.1.3",
+    "@lezer/highlight": "^1.1.6",
-    "@lezer/lr": "^1.3.3",
+    "@lezer/lr": "^1.3.7",
-    "@lezer/markdown": "^1.0.2",
+    "@lezer/markdown": "^1.0.3",
     "@node-oauth/oauth2-server": "^4.3.0",
     "@opentelemetry/api": "^1.0.4",
     "@opentelemetry/auto-instrumentations-web": "^0.27.2",
@@ -257,7 +257,7 @@
   "devDependencies": {
     "@babel/register": "^7.21.0",
     "@juggle/resize-observer": "^3.3.1",
-    "@lezer/generator": "^1.1.3",
+    "@lezer/generator": "^1.3.0",
     "@testing-library/cypress": "^9.0.0",
     "@testing-library/dom": "^9.3.0",
     "@testing-library/react": "^12.1.5",

services/web/scripts/lezer-latex/generate.js

@@ -2,23 +2,39 @@ const { buildParserFile } = require('@lezer/generator')
 const { writeFileSync, readFileSync } = require('fs')
 const path = require('path')
 
-const options = {
+const grammars = [
-  grammarPath: path.resolve(
+  {
-    __dirname,
+    grammarPath: path.resolve(
-    '../../frontend/js/features/source-editor/lezer-latex/latex.grammar'
+      __dirname,
-  ),
+      '../../frontend/js/features/source-editor/lezer-latex/latex.grammar'
-  parserOutputPath: path.resolve(
+    ),
-    __dirname,
+    parserOutputPath: path.resolve(
-    '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
+      __dirname,
-  ),
+      '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
-  termsOutputPath: path.resolve(
+    ),
-    __dirname,
+    termsOutputPath: path.resolve(
-    '../../frontend/js/features/source-editor/lezer-latex/latex.terms.mjs'
+      __dirname,
-  ),
+      '../../frontend/js/features/source-editor/lezer-latex/latex.terms.mjs'
-}
+    ),
+  },
+  {
+    grammarPath: path.resolve(
+      __dirname,
+      '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.grammar'
+    ),
+    parserOutputPath: path.resolve(
+      __dirname,
+      '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs'
+    ),
+    termsOutputPath: path.resolve(
+      __dirname,
+      '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.terms.mjs'
+    ),
+  },
+]
 
-function compile() {
+function compile(grammar) {
-  const { grammarPath, termsOutputPath, parserOutputPath } = options
+  const { grammarPath, termsOutputPath, parserOutputPath } = grammar
   const moduleStyle = 'es'
   console.info(`Compiling ${grammarPath}`)
 
@@ -40,11 +56,11 @@ function compile() {
   console.info('Done!')
 }
 
-module.exports = { compile, options }
+module.exports = { compile, grammars }
 
 if (require.main === module) {
   try {
-    compile()
+    grammars.forEach(compile)
     process.exit(0)
   } catch (err) {
     console.error(err)

services/web/scripts/lezer-latex/run.mjs

@@ -1,11 +1,13 @@
 import { readFileSync } from 'fs'
 import { logTree } from '../../frontend/js/features/source-editor/lezer-latex/print-tree.mjs'
-import { parser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
+import { parser as LaTeXParser } from '../../frontend/js/features/source-editor/lezer-latex/latex.mjs'
+import { parser as BibTeXParser } from '../../frontend/js/features/source-editor/lezer-bibtex/bibtex.mjs'
 
-// Runs the lezer-latex parser on a supplied file, and prints the resulting
+// Runs the lezer-latex or lezer-bibtex parser on a supplied file, and prints the resulting
 // parse tree to stdout
 //
 // show parse tree:     lezer-latex-run.js test/frontend/shared/lezer-latex/examples/amsmath.tex
+//                      lezer-latex-run.js test/frontend/shared/lezer-latex/examples/overleaf.bib
 // show error summary:  lezer-latex-run.js coverage test/frontend/shared/lezer-latex/examples/amsmath.tex
 
 let files = process.argv.slice(2)
@@ -27,6 +29,7 @@ function reportErrorCounts(output) {
 function parseFile(filename) {
   const text = readFileSync(filename).toString()
   const t0 = process.hrtime()
+  const parser = filename.endsWith('.bib') ? BibTeXParser : LaTeXParser
   const tree = parser.parse(text)
   const dt = process.hrtime(t0)
   const timeTaken = dt[0] + dt[1] * 1e-9

services/web/webpack-plugins/lezer-grammar-compiler.js

@@ -4,45 +4,49 @@ const modulePath = path.resolve(__dirname, '../scripts/lezer-latex/generate.js')
 
 try {
   fs.accessSync(modulePath, fs.constants.W_OK)
-  const { compile, options } = require(modulePath)
+  const { compile, grammars } = require(modulePath)
   const PLUGIN_NAME = 'lezer-grammar-compiler'
   class LezerGrammarCompilerPlugin {
     apply(compiler) {
-      compiler.hooks.make.tap(PLUGIN_NAME, compilation => {
+      for (const grammar of grammars) {
-        // Add the grammar file to the file paths watched by webpack
+        compiler.hooks.make.tap(PLUGIN_NAME, compilation => {
-        compilation.fileDependencies.add(options.grammarPath)
+          // Add the grammar file to the file paths watched by webpack
-      })
+          compilation.fileDependencies.add(grammar.grammarPath)
-      compiler.hooks.beforeCompile.tapAsync(
+        })
-        PLUGIN_NAME,
+        compiler.hooks.beforeCompile.tapAsync(
-        (_compilation, callback) => {
+          PLUGIN_NAME,
-          // Check timestamps on grammar and parser files, and re-compile if needed.
+          (_compilation, callback) => {
-          // (Note: the compiled parser file is watched by webpack, and so will trigger
+            // Check timestamps on grammar and parser files, and re-compile if needed.
-          //  a second compilation immediately after. This seems harmless.)
+            // (Note: the compiled parser file is watched by webpack, and so will trigger
-          if (
+            //  a second compilation immediately after. This seems harmless.)
-            !fs.existsSync(options.parserOutputPath) ||
+            if (
-            !fs.existsSync(options.termsOutputPath)
+              !fs.existsSync(grammar.parserOutputPath) ||
-          ) {
+              !fs.existsSync(grammar.termsOutputPath)
-            console.log('Parser does not exist, compiling')
+            ) {
-            compile()
+              console.log('Parser does not exist, compiling')
-            return callback()
+              compile(grammar)
-          }
+              return callback()
-          fs.stat(options.grammarPath, (err, grammarStat) => {
-            if (err) {
-              return callback(err)
             }
-            fs.stat(options.parserOutputPath, (err, parserStat) => {
+            fs.stat(grammar.grammarPath, (err, grammarStat) => {
               if (err) {
                 return callback(err)
               }
-              callback()
+              fs.stat(grammar.parserOutputPath, (err, parserStat) => {
-              if (grammarStat.mtime > parserStat.mtime) {
+                if (err) {
-                console.log('Grammar file newer than parser file, re-compiling')
+                  return callback(err)
-                compile()
+                }
-              }
+                callback()
+                if (grammarStat.mtime > parserStat.mtime) {
+                  console.log(
+                    'Grammar file newer than parser file, re-compiling'
+                  )
+                  compile(grammar)
+                }
+              })
             })
-          })
+          }
-        }
+        )
-      )
+      }
     }
   }
   module.exports = { LezerGrammarCompilerPlugin }