From da814556567eab9ba0ac5fef5314c3ad5ee50ccd Mon Sep 17 00:00:00 2001 From: Joshua Wong Date: Tue, 14 Jan 2020 13:36:33 +0800 Subject: [PATCH] Allow raw string literals in shortcode params --- parser/pageparser/pagelexer_shortcode.go | 47 +++++++++++++++++-- .../pageparser/pageparser_shortcode_test.go | 22 ++++++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/parser/pageparser/pagelexer_shortcode.go b/parser/pageparser/pagelexer_shortcode.go index dea1b317e..8238425bf 100644 --- a/parser/pageparser/pagelexer_shortcode.go +++ b/parser/pageparser/pagelexer_shortcode.go @@ -85,6 +85,8 @@ func lexShortcodeRightDelim(l *pageLexer) stateFunc { // 2. "param" or "param\" // 3. param="123" or param="123\" // 4. param="Some \"escaped\" text" +// 5. `param` +// 6. param=`123` func lexShortcodeParam(l *pageLexer, escapedQuoteStart bool) stateFunc { first := true @@ -95,14 +97,20 @@ func lexShortcodeParam(l *pageLexer, escapedQuoteStart bool) stateFunc { for { r = l.next() if first { - if r == '"' { + if r == '"' || (r == '`' && !escapedQuoteStart) { // a positional param with quotes if l.paramElements == 2 { return l.errorf("got quoted positional parameter. Cannot mix named and positional parameters") } l.paramElements = 1 l.backup() - return lexShortcodeQuotedParamVal(l, !escapedQuoteStart, tScParam) + if r == '"' { + return lexShortcodeQuotedParamVal(l, !escapedQuoteStart, tScParam) + } + return lexShortCodeParamRawStringVal(l, tScParam) + + } else if r == '`' && escapedQuoteStart { + return l.errorf("unrecognized escape character") } first = false } else if r == '=' { @@ -143,6 +151,32 @@ func lexShortcodeParamVal(l *pageLexer) stateFunc { return lexInsideShortcode } +func lexShortCodeParamRawStringVal(l *pageLexer, typ ItemType) stateFunc { + openBacktickFound := false + +Loop: + for { + switch r := l.next(); { + case r == '`': + if openBacktickFound { + l.backup() + break Loop + } else { + openBacktickFound = true + l.ignore() + } + case r == eof, r == '\n': + return l.errorf("unterminated raw string in shortcode parameter-argument: '%s'", l.current()) + } + } + + l.emitString(typ) + l.next() + l.ignore() + + return lexInsideShortcode +} + func lexShortcodeQuotedParamVal(l *pageLexer, escapedQuotedValuesAllowed bool, typ ItemType) stateFunc { openQuoteFound := false escapedInnerQuoteFound := false @@ -161,6 +195,8 @@ Loop: escapedInnerQuoteFound = true escapedQuoteState = 1 } + } else if l.peek() == '`' { + return l.errorf("unrecognized escape character") } case r == eof, r == '\n': return l.errorf("unterminated quoted string in shortcode parameter-argument: '%s'", l.current()) @@ -177,7 +213,6 @@ Loop: } else { escapedQuoteState = 0 } - } } @@ -284,6 +319,8 @@ func lexInsideShortcode(l *pageLexer) stateFunc { peek := l.peek() if peek == '"' || peek == '\\' { return lexShortcodeQuotedParamVal(l, peek != '\\', tScParamVal) + } else if peek == '`' { + return lexShortCodeParamRawStringVal(l, tScParamVal) } return lexShortcodeParamVal case r == '/': @@ -295,10 +332,10 @@ func lexInsideShortcode(l *pageLexer) stateFunc { l.emit(tScClose) case r == '\\': l.ignore() - if l.peek() == '"' { + if l.peek() == '"' || l.peek() == '`' { return lexShortcodeParam(l, true) } - case l.elementStepNum > 0 && (isAlphaNumericOrHyphen(r) || r == '"'): // positional params can have quotes + case l.elementStepNum > 0 && (isAlphaNumericOrHyphen(r) || r == '"' || r == '`'): // positional params can have quotes l.backup() return lexShortcodeParam(l, false) case isAlphaNumeric(r): diff --git a/parser/pageparser/pageparser_shortcode_test.go b/parser/pageparser/pageparser_shortcode_test.go index 4ce4bae31..8d1ebda2f 100644 --- a/parser/pageparser/pageparser_shortcode_test.go +++ b/parser/pageparser/pageparser_shortcode_test.go @@ -13,7 +13,9 @@ package pageparser -import "testing" +import ( + "testing" +) var ( tstEOF = nti(tEOF, "") @@ -77,13 +79,16 @@ var shortCodeLexerTests = []lexerTest{ tstLeftNoMD, tstSC1, nti(tScParam, "3.14"), tstRightNoMD, tstEOF}}, {"float param, named", `{{< sc1 param1=3.14 >}}`, []Item{ tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "3.14"), tstRightNoMD, tstEOF}}, + {"named param, raw string", `{{< sc1 param1=` + "`" + "Hello World" + "`" + " >}}", []Item{ + tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "Hello World"), tstRightNoMD, tstEOF}}, {"float param, named, space before", `{{< sc1 param1= 3.14 >}}`, []Item{ tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "3.14"), tstRightNoMD, tstEOF}}, {"Youtube id", `{{< sc1 -ziL-Q_456igdO-4 >}}`, []Item{ tstLeftNoMD, tstSC1, nti(tScParam, "-ziL-Q_456igdO-4"), tstRightNoMD, tstEOF}}, {"non-alphanumerics param quoted", `{{< sc1 "-ziL-.%QigdO-4" >}}`, []Item{ tstLeftNoMD, tstSC1, nti(tScParam, "-ziL-.%QigdO-4"), tstRightNoMD, tstEOF}}, - + {"raw string", `{{< sc1` + "`" + "Hello World" + "`" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, nti(tScParam, "Hello World"), tstRightNoMD, tstEOF}}, {"two params", `{{< sc1 param1 param2 >}}`, []Item{ tstLeftNoMD, tstSC1, tstParam1, tstParam2, tstRightNoMD, tstEOF}}, // issue #934 @@ -137,11 +142,24 @@ var shortCodeLexerTests = []lexerTest{ {"escaped quotes inside nonescaped quotes in positional param", `{{< sc1 "Hello \"escaped\" World" >}}`, []Item{ tstLeftNoMD, tstSC1, nti(tScParam, `Hello "escaped" World`), tstRightNoMD, tstEOF}}, + {"escaped raw string, named param", `{{< sc1 param1=` + `\` + "`" + "Hello World" + `\` + "`" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, tstParam1, nti(tError, "unrecognized escape character")}}, + {"escaped raw string, positional param", `{{< sc1 param1 ` + `\` + "`" + "Hello World" + `\` + "`" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, tstParam1, nti(tError, "unrecognized escape character")}}, + {"two raw string params", `{{< sc1` + "`" + "Hello World" + "`" + "`" + "Second Param" + "`" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, nti(tScParam, "Hello World"), nti(tScParam, "Second Param"), tstRightNoMD, tstEOF}}, {"unterminated quote", `{{< sc1 param2="Hello World>}}`, []Item{ tstLeftNoMD, tstSC1, tstParam2, nti(tError, "unterminated quoted string in shortcode parameter-argument: 'Hello World>}}'")}}, + {"unterminated raw string", `{{< sc1` + "`" + "Hello World" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, nti(tError, "unterminated raw string in shortcode parameter-argument: 'Hello World >}}'")}}, + {"unterminated raw string in second argument", `{{< sc1` + "`" + "Hello World" + "`" + "`" + "Second Param" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, nti(tScParam, "Hello World"), nti(tError, "unterminated raw string in shortcode parameter-argument: 'Second Param >}}'")}}, {"one named param, one not", `{{< sc1 param1="Hello World" p2 >}}`, []Item{ tstLeftNoMD, tstSC1, tstParam1, tstVal, nti(tError, "got positional parameter 'p2'. Cannot mix named and positional parameters")}}, + {"one named param, one quoted positional param, both raw strings", `{{< sc1 param1=` + "`" + "Hello World" + "`" + "`" + "Second Param" + "`" + ` >}}`, []Item{ + tstLeftNoMD, tstSC1, tstParam1, tstVal, + nti(tError, "got quoted positional parameter. Cannot mix named and positional parameters")}}, {"one named param, one quoted positional param", `{{< sc1 param1="Hello World" "And Universe" >}}`, []Item{ tstLeftNoMD, tstSC1, tstParam1, tstVal, nti(tError, "got quoted positional parameter. Cannot mix named and positional parameters")}},