Merge pull request #18901 from overleaf/mj-math-mode-args

[lezer] Parse command arguments in math mode

GitOrigin-RevId: 85a35f412de4c7238d735253c2642066ebffb393

services/web/frontend/js/features/source-editor/languages/latex/latex-language.ts

@@ -168,7 +168,7 @@ export const LaTeXLanguage = LRLanguage.define({
         'MathGroup/OpenBrace MathGroup/CloseBrace': t.string,
         'MathTextCommand/TextArgument/OpenBrace MathTextCommand/TextArgument/CloseBrace':
           t.string,
-        'MathOpening/LeftCtrlSeq MathClosing/RightCtrlSeq MathCommand/CtrlSeq MathTextCommand/CtrlSeq':
+        'MathOpening/LeftCtrlSeq MathClosing/RightCtrlSeq MathUnknownCommand/CtrlSeq MathTextCommand/CtrlSeq':
           t.literal,
         MathDelimiter: t.literal,
         DoubleDollar: t.keyword,

services/web/frontend/js/features/source-editor/lezer-latex/latex.grammar

@@ -142,7 +142,7 @@
 
     Number { $[0-9]+ ("." $[0-9]*)? }
     MathSpecialChar { $[^_=<>()\-+/*]+ }  // FIXME not all of these are special
-    MathChar { ![0-9^_=<>()\-+/*\\{}\[\]\n$%&~]+ }
+    MathChar { ![0-9^_=<>()\-+/*\\{}\[\]$%&~ \t\n]+ }
 
     @precedence { Number, MathSpecialChar, MathChar }
 
@@ -684,14 +684,13 @@ BracketMath {
 // because display math can contain blank lines while inline math cannot.
 
 Math {
-    ( MathTextCommand
+    ( MathCommand
-    | MathCommand
+    | Group<Math>
-    | MathCtrlSym
-    | MathGroup
     | MathDelimitedGroup
     | MathSpecialChar
     | Number
     | NewLine
+    | Whitespace
     | KnownEnvironment
     | Environment
     | MathChar
@@ -699,24 +698,38 @@ Math {
     | CloseBracket
     | Ampersand
     | Tilde
-    | Label {
-        LabelCtrlSeq optionalWhitespace? OptionalArgument? LabelArgument
-    }
-    | Ref {
-        (RefCtrlSeq | RefStarrableCtrlSeq "*"?) optionalWhitespace? OptionalArgument? optionalWhitespace? OptionalArgument? optionalWhitespace? RefArgument
-    }
     )+
 }
 
-MathTextCommand {
+
-    (MathTextCtrlSeq | HboxCtrlSeq) optionalWhitespace? "*"? TextArgument
+MathCommand {
+    MathKnownCommand
+  | MathUnknownCommand
 }
 
-MathCommand { CtrlSeq }
+MathKnownCommand {
+    KnownCtrlSym
+  | Label {
+        LabelCtrlSeq optionalWhitespace? OptionalArgument? LabelArgument
+    }
+  | Ref {
+     (RefCtrlSeq | RefStarrableCtrlSeq "*"?) optionalWhitespace? OptionalArgument? optionalWhitespace? OptionalArgument? optionalWhitespace? RefArgument
+    }
+  | MathTextCommand {
+    (MathTextCtrlSeq | HboxCtrlSeq) optionalWhitespace? "*"? TextArgument
+  }
+}
 
-MathCtrlSym { CtrlSym | KnownCtrlSym }
+@external tokens endOfArgumentListTokenizer from "./tokens.mjs" {
+    endOfArguments
+}
 
-MathGroup {
+MathUnknownCommand {
+    CtrlSeq (optionalWhitespace? MathArgument)* optionalWhitespace? endOfArguments
+  | CtrlSym
+}
+
+MathArgument {
     OpenBrace Math? CloseBrace
 }
 

services/web/frontend/js/features/source-editor/lezer-latex/tokens.mjs

@@ -80,6 +80,8 @@ import {
   BottomRuleCtrlSeq,
   TableEnvName,
   MultiColumnCtrlSeq,
+  // Marker for end of argument lists
+  endOfArguments,
 } from './latex.terms.mjs'
 
 function nameChar(ch) {
@@ -262,6 +264,22 @@ const CHAR_FULL_STOP = _char('.')
 const CHAR_BACKSLASH = _char('\\')
 const CHAR_OPEN_BRACE = _char('{')
 const CHAR_CLOSE_BRACE = _char('}')
+const CHAR_TAB = _char('\t')
+const CHAR_SPACE = _char(' ')
+const CHAR_NEWLINE = _char('\n')
+
+export const endOfArgumentListTokenizer = new ExternalTokenizer(
+  input => {
+    const { next } = input
+    if (next === CHAR_SPACE || next === CHAR_TAB) {
+      return
+    }
+    if (next !== CHAR_OPEN_BRACE) {
+      input.acceptToken(endOfArguments)
+    }
+  },
+  { contextual: false, fallback: true }
+)
 
 const ALLOWED_DELIMITER_NAMES = [
   'lfloor',
@@ -391,8 +409,6 @@ export const csnameTokenizer = new ExternalTokenizer((input, stack) => {
   return input.acceptToken(Csname, end + 1)
 })
 
-const CHAR_SPACE = _char(' ')
-const CHAR_NEWLINE = _char('\n')
 const END_DOCUMENT_MARK = '\\end{document}'.split('').reverse()
 
 export const trailingContentTokenizer = new ExternalTokenizer(

services/web/frontend/js/features/source-editor/utils/tree-operations/commands.ts

@@ -74,14 +74,15 @@ export const enterNode = (
     items.push(thisCommand)
   } else if (
     node.type.is('UnknownCommand') ||
-    node.type.is('MathCommand') ||
+    node.type.is('KnownCommand') ||
-    node.type.is('KnownCommand')
+    node.type.is('MathUnknownCommand') ||
+    node.type.is('MathKnownCommand')
   ) {
     let commandNode: SyntaxNode | null = node.node
-    if (node.type.is('KnownCommand')) {
+    if (node.type.is('KnownCommand') || node.type.is('MathKnownCommand')) {
-      // KnownCommands are defined as
+      // (Math)KnownCommands are defined as
       //
-      // KnownCommand {
+      // (Math)KnownCommand {
       //    CommandName {
       //       CommandCtrlSeq [args]
       //    }