mirror of
https://github.com/gohugoio/hugo.git
synced 2024-12-24 20:40:47 +00:00
8e5044d7f5
This issue was introduced in `v0.102.0`.
In 223bf28004 we removed the byte source from the parsed page result, which
meant we had to preserve exact positioning for all elements. This introduced some new `TypeIgnore` tokens
which we, wrongly, assumed didn't matter where we put in the result slice (they should be ignored anyway).
But it seems that this broke the logic where we determine if it's positional or named params in the case
where the paramater value contains escaped quoutes.
This commit makes sure that these ignore tokens (the back slashes) are never sent back to the client, which is how it was before `v0.102.0`.
This commit also fixes some lost error information in that same commit.
Fixes #10236
284 lines
14 KiB
Go
284 lines
14 KiB
Go
// Copyright 2018 The Hugo Authors. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package pageparser
|
|
|
|
import (
|
|
"testing"
|
|
|
|
qt "github.com/frankban/quicktest"
|
|
)
|
|
|
|
var (
|
|
tstEOF = nti(tEOF, "")
|
|
tstLeftNoMD = nti(tLeftDelimScNoMarkup, "{{<")
|
|
tstRightNoMD = nti(tRightDelimScNoMarkup, ">}}")
|
|
tstLeftMD = nti(tLeftDelimScWithMarkup, "{{%")
|
|
tstRightMD = nti(tRightDelimScWithMarkup, "%}}")
|
|
tstSCClose = nti(tScClose, "/")
|
|
tstSC1 = nti(tScName, "sc1")
|
|
tstSC1Inline = nti(tScNameInline, "sc1.inline")
|
|
tstSC2Inline = nti(tScNameInline, "sc2.inline")
|
|
tstSC2 = nti(tScName, "sc2")
|
|
tstSC3 = nti(tScName, "sc3")
|
|
tstSCSlash = nti(tScName, "sc/sub")
|
|
tstParam1 = nti(tScParam, "param1")
|
|
tstParam2 = nti(tScParam, "param2")
|
|
tstParamBoolTrue = nti(tScParam, "true")
|
|
tstParamBoolFalse = nti(tScParam, "false")
|
|
tstParamInt = nti(tScParam, "32")
|
|
tstParamFloat = nti(tScParam, "3.14")
|
|
tstVal = nti(tScParamVal, "Hello World")
|
|
tstText = nti(tText, "Hello World")
|
|
)
|
|
|
|
var shortCodeLexerTests = []lexerTest{
|
|
{"empty", "", []typeText{tstEOF}},
|
|
{"spaces", " \t\n", []typeText{nti(tText, " \t\n"), tstEOF}},
|
|
{"text", `to be or not`, []typeText{nti(tText, "to be or not"), tstEOF}},
|
|
{"no markup", `{{< sc1 >}}`, []typeText{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}},
|
|
{"with EOL", "{{< sc1 \n >}}", []typeText{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}},
|
|
|
|
{"forward slash inside name", `{{< sc/sub >}}`, []typeText{tstLeftNoMD, tstSCSlash, tstRightNoMD, tstEOF}},
|
|
|
|
{"simple with markup", `{{% sc1 %}}`, []typeText{tstLeftMD, tstSC1, tstRightMD, tstEOF}},
|
|
{"with spaces", `{{< sc1 >}}`, []typeText{tstLeftNoMD, tstSC1, tstRightNoMD, tstEOF}},
|
|
{"indented on new line", "Hello\n {{% sc1 %}}", []typeText{nti(tText, "Hello\n"), nti(tIndentation, " "), tstLeftMD, tstSC1, tstRightMD, tstEOF}},
|
|
{"indented on new line tab", "Hello\n\t{{% sc1 %}}", []typeText{nti(tText, "Hello\n"), nti(tIndentation, "\t"), tstLeftMD, tstSC1, tstRightMD, tstEOF}},
|
|
{"indented on first line", " {{% sc1 %}}", []typeText{nti(tIndentation, " "), tstLeftMD, tstSC1, tstRightMD, tstEOF}},
|
|
{"mismatched rightDelim", `{{< sc1 %}}`, []typeText{
|
|
tstLeftNoMD, tstSC1,
|
|
nti(tError, "unrecognized character in shortcode action: U+0025 '%'. Note: Parameters with non-alphanumeric args must be quoted"),
|
|
}},
|
|
{"inner, markup", `{{% sc1 %}} inner {{% /sc1 %}}`, []typeText{
|
|
tstLeftMD,
|
|
tstSC1,
|
|
tstRightMD,
|
|
nti(tText, " inner "),
|
|
tstLeftMD,
|
|
tstSCClose,
|
|
tstSC1,
|
|
tstRightMD,
|
|
tstEOF,
|
|
}},
|
|
{"close, but no open", `{{< /sc1 >}}`, []typeText{
|
|
tstLeftNoMD, nti(tError, "got closing shortcode, but none is open"),
|
|
}},
|
|
{"close wrong", `{{< sc1 >}}{{< /another >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstRightNoMD, tstLeftNoMD, tstSCClose,
|
|
nti(tError, "closing tag for shortcode 'another' does not match start tag"),
|
|
}},
|
|
{"close, but no open, more", `{{< sc1 >}}{{< /sc1 >}}{{< /another >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstRightNoMD, tstLeftNoMD, tstSCClose, tstSC1, tstRightNoMD, tstLeftNoMD, tstSCClose,
|
|
nti(tError, "closing tag for shortcode 'another' does not match start tag"),
|
|
}},
|
|
{"close with extra keyword", `{{< sc1 >}}{{< /sc1 keyword>}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstRightNoMD, tstLeftNoMD, tstSCClose, tstSC1,
|
|
nti(tError, "unclosed shortcode"),
|
|
}},
|
|
{"float param, positional", `{{< sc1 3.14 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "3.14"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"float param, named", `{{< sc1 param1=3.14 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "3.14"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"named param, raw string", `{{< sc1 param1=` + "`" + "Hello World" + "`" + " >}}", []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "Hello World"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"float param, named, space before", `{{< sc1 param1= 3.14 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, "3.14"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"Youtube id", `{{< sc1 -ziL-Q_456igdO-4 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "-ziL-Q_456igdO-4"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"non-alphanumerics param quoted", `{{< sc1 "-ziL-.%QigdO-4" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "-ziL-.%QigdO-4"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"raw string", `{{< sc1` + "`" + "Hello World" + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "Hello World"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"raw string with newline", `{{< sc1` + "`" + `Hello
|
|
World` + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, `Hello
|
|
World`), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"raw string with escape character", `{{< sc1` + "`" + `Hello \b World` + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, `Hello \b World`), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"two params", `{{< sc1 param1 param2 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstParam2, tstRightNoMD, tstEOF,
|
|
}},
|
|
// issue #934
|
|
{"self-closing", `{{< sc1 />}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstSCClose, tstRightNoMD, tstEOF,
|
|
}},
|
|
// Issue 2498
|
|
{"multiple self-closing", `{{< sc1 />}}{{< sc1 />}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstSCClose, tstRightNoMD,
|
|
tstLeftNoMD, tstSC1, tstSCClose, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"self-closing with param", `{{< sc1 param1 />}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstSCClose, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"multiple self-closing with param", `{{< sc1 param1 />}}{{< sc1 param1 />}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstSCClose, tstRightNoMD,
|
|
tstLeftNoMD, tstSC1, tstParam1, tstSCClose, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"multiple different self-closing with param", `{{< sc1 param1 />}}{{< sc2 param1 />}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstSCClose, tstRightNoMD,
|
|
tstLeftNoMD, tstSC2, tstParam1, tstSCClose, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"nested simple", `{{< sc1 >}}{{< sc2 >}}{{< /sc1 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstRightNoMD,
|
|
tstLeftNoMD, tstSC2, tstRightNoMD,
|
|
tstLeftNoMD, tstSCClose, tstSC1, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"nested complex", `{{< sc1 >}}ab{{% sc2 param1 %}}cd{{< sc3 >}}ef{{< /sc3 >}}gh{{% /sc2 %}}ij{{< /sc1 >}}kl`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstRightNoMD,
|
|
nti(tText, "ab"),
|
|
tstLeftMD, tstSC2, tstParam1, tstRightMD,
|
|
nti(tText, "cd"),
|
|
tstLeftNoMD, tstSC3, tstRightNoMD,
|
|
nti(tText, "ef"),
|
|
tstLeftNoMD, tstSCClose, tstSC3, tstRightNoMD,
|
|
nti(tText, "gh"),
|
|
tstLeftMD, tstSCClose, tstSC2, tstRightMD,
|
|
nti(tText, "ij"),
|
|
tstLeftNoMD, tstSCClose, tstSC1, tstRightNoMD,
|
|
nti(tText, "kl"), tstEOF,
|
|
}},
|
|
|
|
{"two quoted params", `{{< sc1 "param nr. 1" "param nr. 2" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "param nr. 1"), nti(tScParam, "param nr. 2"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"two named params", `{{< sc1 param1="Hello World" param2="p2Val">}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstVal, tstParam2, nti(tScParamVal, "p2Val"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"escaped quotes", `{{< sc1 param1=\"Hello World\" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstVal, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"escaped quotes, positional param", `{{< sc1 \"param1\" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"escaped quotes inside escaped quotes", `{{< sc1 param1=\"Hello \"escaped\" World\" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1,
|
|
nti(tScParamVal, `Hello `), nti(tError, `got positional parameter 'escaped'. Cannot mix named and positional parameters`),
|
|
}},
|
|
{
|
|
"escaped quotes inside nonescaped quotes",
|
|
`{{< sc1 param1="Hello \"escaped\" World" >}}`,
|
|
[]typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, nti(tScParamVal, `Hello "escaped" World`), tstRightNoMD, tstEOF,
|
|
},
|
|
},
|
|
{
|
|
"escaped quotes inside nonescaped quotes in positional param",
|
|
`{{< sc1 "Hello \"escaped\" World" >}}`,
|
|
[]typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, `Hello "escaped" World`), tstRightNoMD, tstEOF,
|
|
},
|
|
},
|
|
{"escaped raw string, named param", `{{< sc1 param1=` + `\` + "`" + "Hello World" + `\` + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, nti(tError, "unrecognized escape character"),
|
|
}},
|
|
{"escaped raw string, positional param", `{{< sc1 param1 ` + `\` + "`" + "Hello World" + `\` + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, nti(tError, "unrecognized escape character"),
|
|
}},
|
|
{"two raw string params", `{{< sc1` + "`" + "Hello World" + "`" + "`" + "Second Param" + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "Hello World"), nti(tScParam, "Second Param"), tstRightNoMD, tstEOF,
|
|
}},
|
|
{"unterminated quote", `{{< sc1 param2="Hello World>}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam2, nti(tError, "unterminated quoted string in shortcode parameter-argument: 'Hello World>}}'"),
|
|
}},
|
|
{"unterminated raw string", `{{< sc1` + "`" + "Hello World" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tError, "unterminated raw string in shortcode parameter-argument: 'Hello World >}}'"),
|
|
}},
|
|
{"unterminated raw string in second argument", `{{< sc1` + "`" + "Hello World" + "`" + "`" + "Second Param" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, nti(tScParam, "Hello World"), nti(tError, "unterminated raw string in shortcode parameter-argument: 'Second Param >}}'"),
|
|
}},
|
|
{"one named param, one not", `{{< sc1 param1="Hello World" p2 >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstVal,
|
|
nti(tError, "got positional parameter 'p2'. Cannot mix named and positional parameters"),
|
|
}},
|
|
{"one named param, one quoted positional param, both raw strings", `{{< sc1 param1=` + "`" + "Hello World" + "`" + "`" + "Second Param" + "`" + ` >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstVal,
|
|
nti(tError, "got quoted positional parameter. Cannot mix named and positional parameters"),
|
|
}},
|
|
{"one named param, one quoted positional param", `{{< sc1 param1="Hello World" "And Universe" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1, tstVal,
|
|
nti(tError, "got quoted positional parameter. Cannot mix named and positional parameters"),
|
|
}},
|
|
{"one quoted positional param, one named param", `{{< sc1 "param1" param2="And Universe" >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1,
|
|
nti(tError, "got named parameter 'param2'. Cannot mix named and positional parameters"),
|
|
}},
|
|
{"ono positional param, one not", `{{< sc1 param1 param2="Hello World">}}`, []typeText{
|
|
tstLeftNoMD, tstSC1, tstParam1,
|
|
nti(tError, "got named parameter 'param2'. Cannot mix named and positional parameters"),
|
|
}},
|
|
{"commented out", `{{</* sc1 */>}}`, []typeText{
|
|
nti(tText, "{{<"), nti(tText, " sc1 "), nti(tText, ">}}"), tstEOF,
|
|
}},
|
|
{"commented out, with asterisk inside", `{{</* sc1 "**/*.pdf" */>}}`, []typeText{
|
|
nti(tText, "{{<"), nti(tText, " sc1 \"**/*.pdf\" "), nti(tText, ">}}"), tstEOF,
|
|
}},
|
|
{"commented out, missing close", `{{</* sc1 >}}`, []typeText{
|
|
nti(tError, "comment must be closed"),
|
|
}},
|
|
{"commented out, misplaced close", `{{</* sc1 >}}*/`, []typeText{
|
|
nti(tError, "comment must be closed"),
|
|
}},
|
|
// Inline shortcodes
|
|
{"basic inline", `{{< sc1.inline >}}Hello World{{< /sc1.inline >}}`, []typeText{tstLeftNoMD, tstSC1Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSCClose, tstSC1Inline, tstRightNoMD, tstEOF}},
|
|
{"basic inline with space", `{{< sc1.inline >}}Hello World{{< / sc1.inline >}}`, []typeText{tstLeftNoMD, tstSC1Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSCClose, tstSC1Inline, tstRightNoMD, tstEOF}},
|
|
{"inline self closing", `{{< sc1.inline >}}Hello World{{< /sc1.inline >}}Hello World{{< sc1.inline />}}`, []typeText{tstLeftNoMD, tstSC1Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSCClose, tstSC1Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSC1Inline, tstSCClose, tstRightNoMD, tstEOF}},
|
|
{"inline self closing, then a new inline", `{{< sc1.inline >}}Hello World{{< /sc1.inline >}}Hello World{{< sc1.inline />}}{{< sc2.inline >}}Hello World{{< /sc2.inline >}}`, []typeText{
|
|
tstLeftNoMD, tstSC1Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSCClose, tstSC1Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSC1Inline, tstSCClose, tstRightNoMD,
|
|
tstLeftNoMD, tstSC2Inline, tstRightNoMD, tstText, tstLeftNoMD, tstSCClose, tstSC2Inline, tstRightNoMD, tstEOF,
|
|
}},
|
|
{"inline with template syntax", `{{< sc1.inline >}}{{ .Get 0 }}{{ .Get 1 }}{{< /sc1.inline >}}`, []typeText{tstLeftNoMD, tstSC1Inline, tstRightNoMD, nti(tText, "{{ .Get 0 }}"), nti(tText, "{{ .Get 1 }}"), tstLeftNoMD, tstSCClose, tstSC1Inline, tstRightNoMD, tstEOF}},
|
|
{"inline with nested shortcode (not supported)", `{{< sc1.inline >}}Hello World{{< sc1 >}}{{< /sc1.inline >}}`, []typeText{tstLeftNoMD, tstSC1Inline, tstRightNoMD, tstText, nti(tError, "inline shortcodes do not support nesting")}},
|
|
{"inline case mismatch", `{{< sc1.Inline >}}Hello World{{< /sc1.Inline >}}`, []typeText{tstLeftNoMD, nti(tError, "period in shortcode name only allowed for inline identifiers")}},
|
|
}
|
|
|
|
func TestShortcodeLexer(t *testing.T) {
|
|
t.Parallel()
|
|
c := qt.New(t)
|
|
for i, test := range shortCodeLexerTests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
items := collect([]byte(test.input), true, lexMainSection)
|
|
if !equal(test.input, items, test.items) {
|
|
got := itemsToString(items, []byte(test.input))
|
|
expected := testItemsToString(test.items)
|
|
c.Assert(got, qt.Equals, expected, qt.Commentf("Test %d: %s", i, test.name))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkShortcodeLexer(b *testing.B) {
|
|
testInputs := make([][]byte, len(shortCodeLexerTests))
|
|
for i, input := range shortCodeLexerTests {
|
|
testInputs[i] = []byte(input.input)
|
|
}
|
|
var cfg Config
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
for _, input := range testInputs {
|
|
items := collectWithConfig(input, true, lexMainSection, cfg)
|
|
if len(items) == 0 {
|
|
}
|
|
|
|
}
|
|
}
|
|
}
|