Merge pull request #18901 from overleaf/mj-math-mode-args

[lezer] Parse command arguments in math mode

GitOrigin-RevId: 85a35f412de4c7238d735253c2642066ebffb393
This commit is contained in:
Mathias Jakobsen 2024-06-18 11:01:08 +01:00 committed by Copybot
parent d2ae97427d
commit 9b1c703f6b
4 changed files with 54 additions and 24 deletions

View file

@ -168,7 +168,7 @@ export const LaTeXLanguage = LRLanguage.define({
'MathGroup/OpenBrace MathGroup/CloseBrace': t.string,
'MathTextCommand/TextArgument/OpenBrace MathTextCommand/TextArgument/CloseBrace':
t.string,
'MathOpening/LeftCtrlSeq MathClosing/RightCtrlSeq MathCommand/CtrlSeq MathTextCommand/CtrlSeq':
'MathOpening/LeftCtrlSeq MathClosing/RightCtrlSeq MathUnknownCommand/CtrlSeq MathTextCommand/CtrlSeq':
t.literal,
MathDelimiter: t.literal,
DoubleDollar: t.keyword,

View file

@ -142,7 +142,7 @@
Number { $[0-9]+ ("." $[0-9]*)? }
MathSpecialChar { $[^_=<>()\-+/*]+ } // FIXME not all of these are special
MathChar { ![0-9^_=<>()\-+/*\\{}\[\]\n$%&~]+ }
MathChar { ![0-9^_=<>()\-+/*\\{}\[\]$%&~ \t\n]+ }
@precedence { Number, MathSpecialChar, MathChar }
@ -684,14 +684,13 @@ BracketMath {
// because display math can contain blank lines while inline math cannot.
Math {
( MathTextCommand
| MathCommand
| MathCtrlSym
| MathGroup
( MathCommand
| Group<Math>
| MathDelimitedGroup
| MathSpecialChar
| Number
| NewLine
| Whitespace
| KnownEnvironment
| Environment
| MathChar
@ -699,24 +698,38 @@ Math {
| CloseBracket
| Ampersand
| Tilde
| Label {
LabelCtrlSeq optionalWhitespace? OptionalArgument? LabelArgument
}
| Ref {
(RefCtrlSeq | RefStarrableCtrlSeq "*"?) optionalWhitespace? OptionalArgument? optionalWhitespace? OptionalArgument? optionalWhitespace? RefArgument
}
)+
}
MathTextCommand {
(MathTextCtrlSeq | HboxCtrlSeq) optionalWhitespace? "*"? TextArgument
MathCommand {
MathKnownCommand
| MathUnknownCommand
}
MathCommand { CtrlSeq }
MathKnownCommand {
KnownCtrlSym
| Label {
LabelCtrlSeq optionalWhitespace? OptionalArgument? LabelArgument
}
| Ref {
(RefCtrlSeq | RefStarrableCtrlSeq "*"?) optionalWhitespace? OptionalArgument? optionalWhitespace? OptionalArgument? optionalWhitespace? RefArgument
}
| MathTextCommand {
(MathTextCtrlSeq | HboxCtrlSeq) optionalWhitespace? "*"? TextArgument
}
}
MathCtrlSym { CtrlSym | KnownCtrlSym }
@external tokens endOfArgumentListTokenizer from "./tokens.mjs" {
endOfArguments
}
MathGroup {
MathUnknownCommand {
CtrlSeq (optionalWhitespace? MathArgument)* optionalWhitespace? endOfArguments
| CtrlSym
}
MathArgument {
OpenBrace Math? CloseBrace
}

View file

@ -80,6 +80,8 @@ import {
BottomRuleCtrlSeq,
TableEnvName,
MultiColumnCtrlSeq,
// Marker for end of argument lists
endOfArguments,
} from './latex.terms.mjs'
function nameChar(ch) {
@ -262,6 +264,22 @@ const CHAR_FULL_STOP = _char('.')
const CHAR_BACKSLASH = _char('\\')
const CHAR_OPEN_BRACE = _char('{')
const CHAR_CLOSE_BRACE = _char('}')
const CHAR_TAB = _char('\t')
const CHAR_SPACE = _char(' ')
const CHAR_NEWLINE = _char('\n')
export const endOfArgumentListTokenizer = new ExternalTokenizer(
input => {
const { next } = input
if (next === CHAR_SPACE || next === CHAR_TAB) {
return
}
if (next !== CHAR_OPEN_BRACE) {
input.acceptToken(endOfArguments)
}
},
{ contextual: false, fallback: true }
)
const ALLOWED_DELIMITER_NAMES = [
'lfloor',
@ -391,8 +409,6 @@ export const csnameTokenizer = new ExternalTokenizer((input, stack) => {
return input.acceptToken(Csname, end + 1)
})
const CHAR_SPACE = _char(' ')
const CHAR_NEWLINE = _char('\n')
const END_DOCUMENT_MARK = '\\end{document}'.split('').reverse()
export const trailingContentTokenizer = new ExternalTokenizer(

View file

@ -74,14 +74,15 @@ export const enterNode = (
items.push(thisCommand)
} else if (
node.type.is('UnknownCommand') ||
node.type.is('MathCommand') ||
node.type.is('KnownCommand')
node.type.is('KnownCommand') ||
node.type.is('MathUnknownCommand') ||
node.type.is('MathKnownCommand')
) {
let commandNode: SyntaxNode | null = node.node
if (node.type.is('KnownCommand')) {
// KnownCommands are defined as
if (node.type.is('KnownCommand') || node.type.is('MathKnownCommand')) {
// (Math)KnownCommands are defined as
//
// KnownCommand {
// (Math)KnownCommand {
// CommandName {
// CommandCtrlSeq [args]
// }