github/overleaf
1
0
Fork 0
mirror of https://github.com/overleaf/overleaf.git synced 2025-02-16 22:43:53 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #19050 from overleaf/mj-lezer-parse-comments-in-unknown-commands

Browse source 
[lezer] Introduce hasMoreArguments token for argument parsing

GitOrigin-RevId: a5898a2be01c19a39de15c784f184fe61140799a
This commit is contained in:
Mathias Jakobsen 2024-06-25 09:09:05 +01:00 committed by Copybot
parent 46a160f127
commit ddfdafd54c
2 changed files with 19 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
services/web/frontend/js/features/source-editor/lezer-latex/latex.grammar
View file

@ -694,12 +694,13 @@ MathCommand {
| KnownCtrlSym
}
@external tokens endOfArgumentListTokenizer from "./tokens.mjs" {
@external tokens argumentListTokenizer from "./tokens.mjs" {
hasMoreArguments,
endOfArguments
}
MathUnknownCommand {
CtrlSeq (optionalWhitespace? MathArgument)* optionalWhitespace? endOfArguments
CtrlSeq (hasMoreArguments optionalWhitespace? MathArgument)* endOfArguments
| CtrlSym
}

23
services/web/frontend/js/features/source-editor/lezer-latex/tokens.mjs
View file

@ -81,8 +81,11 @@ import {
MultiColumnCtrlSeq,
// Marker for end of argument lists
endOfArguments,
hasMoreArguments,
} from './latex.terms.mjs'
const MAX_ARGUMENT_LOOKAHEAD = 100
function nameChar(ch) {
// we accept A-Z a-z 0-9 * + @ in environment names
return (
@ -256,14 +259,20 @@ const CHAR_TAB = _char('\t')
const CHAR_SPACE = _char(' ')
const CHAR_NEWLINE = _char('\n')
export const endOfArgumentListTokenizer = new ExternalTokenizer(
export const argumentListTokenizer = new ExternalTokenizer(
input => {
const { next } = input
if (next === CHAR_SPACE || next === CHAR_TAB) {
return
}
if (next !== CHAR_OPEN_BRACE) {
input.acceptToken(endOfArguments)
for (let i = 0; i < MAX_ARGUMENT_LOOKAHEAD; ++i) {
const next = input.peek(i)
if (next === CHAR_SPACE || next === CHAR_TAB) {
continue
}
if (next === CHAR_OPEN_BRACE) {
input.acceptToken(hasMoreArguments)
return
} else {
input.acceptToken(endOfArguments)
return
}
}
},
{ contextual: false, fallback: true }