github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-12-29 01:21:52 +00:00
Code Issues Projects Releases Packages Wiki Activity
hugo/parser/parse_frontmatter_test.go

325 lines
13 KiB
Go
Raw Normal View History

Add copyright header to that source files that don’t have one. See #1646
2015-12-10 22:19:38 +00:00
// Copyright 2015 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
package parser
// TODO Support Mac Encoding (\r)
import (
"bufio"
"bytes"
"io"
"os"
"path/filepath"
"strings"
"testing"
)
parser: Make the constant vars ... constants
2016-03-23 14:05:32 +00:00
const (
parser: Accept JSON frontmatter without leading "{\n" Accept JSON frontmatter without leading "{\n" so that one line frontmatters such as `{"param":"paramvalue"}` no longer silently render empty html.
2016-12-20 13:54:52 +00:00
contentNoFrontmatter = "a page with no front matter"
contentWithFrontmatter = "---\ntitle: front matter\n---\nContent with front matter"
contentHTMLNoDoctype = "<html>\n\t<body>\n\t</body>\n</html>"
contentHTMLWithDoctype = "<!doctype html><html><body></body></html>"
contentHTMLWithFrontmatter = "---\ntitle: front matter\n---\n<!doctype><html><body></body></html>"
contentHTML = " <html><body></body></html>"
contentLinefeedAndHTML = "\n<html><body></body></html>"
contentIncompleteEndFrontmatterDelim = "---\ntitle: incomplete end fm delim\n--\nincomplete frontmatter delim"
contentMissingEndFrontmatterDelim = "---\ntitle: incomplete end fm delim\nincomplete frontmatter delim"
contentSlugWorking = "---\ntitle: slug doc 2\nslug: slug-doc-2\n\n---\nslug doc 2 content"
contentSlugWorkingVariation = "---\ntitle: slug doc 3\nslug: slug-doc 3\n---\nslug doc 3 content"
contentSlugBug = "---\ntitle: slug doc 2\nslug: slug-doc-2\n---\nslug doc 2 content"
contentSlugWithJSONFrontMatter = "{\n \"categories\": \"d\",\n \"tags\": [\n \"a\", \n \"b\", \n \"c\"\n ]\n}\nJSON Front Matter with tags and categories"
contentWithJSONLooseFrontmatter = "{\n \"categories\": \"d\"\n \"tags\": [\n \"a\" \n \"b\" \n \"c\"\n ]\n}\nJSON Front Matter with tags and categories"
contentSlugWithJSONFrontMatterFirstLineOnly = "{\"categories\":\"d\",\"tags\":[\"a\",\"b\",\"c\"]}\nJSON Front Matter with tags and categories"
contentSlugWithJSONFrontMatterFirstLine = "{\"categories\":\"d\",\n \"tags\":[\"a\",\"b\",\"c\"]}\nJSON Front Matter with tags and categories"
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
)
var lineEndings = []string{"\n", "\r\n"}
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
var delimiters = []string{"---", "+++"}
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
func pageMust(p Page, err error) *page {
if err != nil {
panic(err)
}
return p.(*page)
}
func TestDegenerateCreatePageFrom(t *testing.T) {
tests := []struct {
content string
}{
parser: Fix ALL_CAPS var names in test
2016-03-23 14:02:00 +00:00
{contentMissingEndFrontmatterDelim},
{contentIncompleteEndFrontmatterDelim},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
for _, test := range tests {
for _, ending := range lineEndings {
test.content = strings.Replace(test.content, "\n", ending, -1)
_, err := ReadFrom(strings.NewReader(test.content))
if err == nil {
t.Errorf("Content should return an err:\n%q\n", test.content)
}
}
}
}
func checkPageRender(t *testing.T, p *page, expected bool) {
if p.render != expected {
t.Errorf("page.render should be %t, got: %t", expected, p.render)
}
}
func checkPageFrontMatterIsNil(t *testing.T, p *page, content string, expected bool) {
if bool(p.frontmatter == nil) != expected {
t.Logf("\n%q\n", content)
t.Errorf("page.frontmatter == nil? %t, got %t", expected, p.frontmatter == nil)
}
}
func checkPageFrontMatterContent(t *testing.T, p *page, frontMatter string) {
if p.frontmatter == nil {
return
}
if !bytes.Equal(p.frontmatter, []byte(frontMatter)) {
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
t.Errorf("frontmatter mismatch\nexp: %q\ngot: %q", frontMatter, p.frontmatter)
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
}
func checkPageContent(t *testing.T, p *page, expected string) {
if !bytes.Equal(p.content, []byte(expected)) {
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
t.Errorf("content mismatch\nexp: %q\ngot: %q", expected, p.content)
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
}
func TestStandaloneCreatePageFrom(t *testing.T) {
tests := []struct {
content string
expectedMustRender bool
frontMatterIsNil bool
frontMatter string
bodycontent string
}{
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
parser: Fix ALL_CAPS var names in test
2016-03-23 14:02:00 +00:00
{contentNoFrontmatter, true, true, "", "a page with no front matter"},
{contentWithFrontmatter, true, false, "---\ntitle: front matter\n---\n", "Content with front matter"},
{contentHTMLNoDoctype, false, true, "", "<html>\n\t<body>\n\t</body>\n</html>"},
{contentHTMLWithDoctype, false, true, "", "<!doctype html><html><body></body></html>"},
{contentHTMLWithFrontmatter, true, false, "---\ntitle: front matter\n---\n", "<!doctype><html><body></body></html>"},
{contentHTML, false, true, "", "<html><body></body></html>"},
{contentLinefeedAndHTML, false, true, "", "<html><body></body></html>"},
{contentSlugWithJSONFrontMatter, true, false, "{\n \"categories\": \"d\",\n \"tags\": [\n \"a\", \n \"b\", \n \"c\"\n ]\n}", "JSON Front Matter with tags and categories"},
{contentWithJSONLooseFrontmatter, true, false, "{\n \"categories\": \"d\"\n \"tags\": [\n \"a\" \n \"b\" \n \"c\"\n ]\n}", "JSON Front Matter with tags and categories"},
parser: Accept JSON frontmatter without leading "{\n" Accept JSON frontmatter without leading "{\n" so that one line frontmatters such as `{"param":"paramvalue"}` no longer silently render empty html.
2016-12-20 13:54:52 +00:00
{contentSlugWithJSONFrontMatterFirstLineOnly, true, false, "{\"categories\":\"d\",\"tags\":[\"a\",\"b\",\"c\"]}", "JSON Front Matter with tags and categories"},
{contentSlugWithJSONFrontMatterFirstLine, true, false, "{\"categories\":\"d\",\n \"tags\":[\"a\",\"b\",\"c\"]}", "JSON Front Matter with tags and categories"},
parser: Fix ALL_CAPS var names in test
2016-03-23 14:02:00 +00:00
{contentSlugWorking, true, false, "---\ntitle: slug doc 2\nslug: slug-doc-2\n\n---\n", "slug doc 2 content"},
{contentSlugWorkingVariation, true, false, "---\ntitle: slug doc 3\nslug: slug-doc 3\n---\n", "slug doc 3 content"},
{contentSlugBug, true, false, "---\ntitle: slug doc 2\nslug: slug-doc-2\n---\n", "slug doc 2 content"},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
for _, test := range tests {
for _, ending := range lineEndings {
test.content = strings.Replace(test.content, "\n", ending, -1)
test.frontMatter = strings.Replace(test.frontMatter, "\n", ending, -1)
test.bodycontent = strings.Replace(test.bodycontent, "\n", ending, -1)
p := pageMust(ReadFrom(strings.NewReader(test.content)))
checkPageRender(t, p, test.expectedMustRender)
checkPageFrontMatterIsNil(t, p, test.content, test.frontMatterIsNil)
checkPageFrontMatterContent(t, p, test.frontMatter)
checkPageContent(t, p, test.bodycontent)
}
}
}
func BenchmarkLongFormRender(b *testing.B) {
tests := []struct {
filename string
buf []byte
}{
{filename: "long_text_test.md"},
}
for i, test := range tests {
path := filepath.FromSlash(test.filename)
f, err := os.Open(path)
if err != nil {
b.Fatalf("Unable to open %s: %s", path, err)
}
defer f.Close()
membuf := new(bytes.Buffer)
if _, err := io.Copy(membuf, f); err != nil {
b.Fatalf("Unable to read %s: %s", path, err)
}
tests[i].buf = membuf.Bytes()
}
b.ResetTimer()
for i := 0; i <= b.N; i++ {
for _, test := range tests {
ReadFrom(bytes.NewReader(test.buf))
}
}
}
func TestPageShouldRender(t *testing.T) {
tests := []struct {
content []byte
expected bool
}{
{[]byte{}, false},
{[]byte{'<'}, false},
{[]byte{'-'}, true},
{[]byte("--"), true},
{[]byte("---"), true},
{[]byte("---\n"), true},
{[]byte{'a'}, true},
}
for _, test := range tests {
for _, ending := range lineEndings {
test.content = bytes.Replace(test.content, []byte("\n"), []byte(ending), -1)
if render := shouldRender(test.content); render != test.expected {
t.Errorf("Expected %s to shouldRender = %t, got: %t", test.content, test.expected, render)
}
}
}
}
func TestPageHasFrontMatter(t *testing.T) {
tests := []struct {
content []byte
expected bool
}{
{[]byte{'-'}, false},
{[]byte("--"), false},
{[]byte("---"), false},
{[]byte("---\n"), true},
{[]byte("---\n"), true},
Trim trailing spaces from YAML and TOML delimiters When someone hits space after typing "---" (or "+++") but before they hit return, hugo silently failed to parse the file. This corrects that.
2015-08-02 05:24:22 +00:00
{[]byte("--- \n"), true},
{[]byte("--- \n"), true},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
{[]byte{'a'}, false},
{[]byte{'{'}, true},
{[]byte("{\n "), true},
{[]byte{'}'}, false},
}
for _, test := range tests {
for _, ending := range lineEndings {
test.content = bytes.Replace(test.content, []byte("\n"), []byte(ending), -1)
if isFrontMatterDelim := isFrontMatterDelim(test.content); isFrontMatterDelim != test.expected {
t.Errorf("Expected %q isFrontMatterDelim = %t, got: %t", test.content, test.expected, isFrontMatterDelim)
}
}
}
}
func TestExtractFrontMatter(t *testing.T) {
tests := []struct {
frontmatter string
extracted []byte
errIsNil bool
}{
{"", nil, false},
{"-", nil, false},
{"---\n", nil, false},
{"---\nfoobar", nil, false},
{"---\nfoobar\nbarfoo\nfizbaz\n", nil, false},
{"---\nblar\n-\n", nil, false},
{"---\nralb\n---\n", []byte("---\nralb\n---\n"), true},
Fix "hugo new" EOF error with an archetype file without the final EOL This rewrites `extractFrontMatterDelims` function to make it work with an archetype file without the final EOL and adds more detailed error messages and comments. It also removes `matches` and `matches_quick` functions which aren't called anywhere.
2015-01-10 07:15:51 +00:00
{"---\neof\n---", []byte("---\neof\n---"), true},
Trim trailing spaces from YAML and TOML delimiters When someone hits space after typing "---" (or "+++") but before they hit return, hugo silently failed to parse the file. This corrects that.
2015-08-02 05:24:22 +00:00
{"--- \neof\n---", []byte("---\neof\n---"), true},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
{"---\nminc\n---\ncontent", []byte("---\nminc\n---\n"), true},
Trim trailing spaces from YAML and TOML delimiters When someone hits space after typing "---" (or "+++") but before they hit return, hugo silently failed to parse the file. This corrects that.
2015-08-02 05:24:22 +00:00
{"---\nminc\n--- \ncontent", []byte("---\nminc\n---\n"), true},
{"--- \nminc\n--- \ncontent", []byte("---\nminc\n---\n"), true},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
{"---\ncnim\n---\ncontent\n", []byte("---\ncnim\n---\n"), true},
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
{"---\ntitle: slug doc 2\nslug: slug-doc-2\n---\ncontent\n", []byte("---\ntitle: slug doc 2\nslug: slug-doc-2\n---\n"), true},
Fix searching YAML/TOML delimiters in frontmatter When a YAML/TOML's delimiter character sequence is included in a frontmatter string, parser mistakes it as a delimiter. This fixes it by checking a character right before the delimiter sequence is '\n' or it is the beginning of the frontmatter. Fix #1320
2015-08-03 14:32:51 +00:00
{"---\npermalink: '/blog/title---subtitle.html'\n---\ncontent\n", []byte("---\npermalink: '/blog/title---subtitle.html'\n---\n"), true},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
for _, test := range tests {
for _, ending := range lineEndings {
test.frontmatter = strings.Replace(test.frontmatter, "\n", ending, -1)
test.extracted = bytes.Replace(test.extracted, []byte("\n"), []byte(ending), -1)
for _, delim := range delimiters {
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
test.frontmatter = strings.Replace(test.frontmatter, "---", delim, -1)
test.extracted = bytes.Replace(test.extracted, []byte("---"), []byte(delim), -1)
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
line, err := peekLine(bufio.NewReader(strings.NewReader(test.frontmatter)))
if err != nil {
continue
}
l, r := determineDelims(line)
fm, err := extractFrontMatterDelims(bufio.NewReader(strings.NewReader(test.frontmatter)), l, r)
if (err == nil) != test.errIsNil {
t.Logf("\n%q\n", string(test.frontmatter))
t.Errorf("Expected err == nil => %t, got: %t. err: %s", test.errIsNil, err == nil, err)
continue
}
if !bytes.Equal(fm, test.extracted) {
Fix parsing edge case of frontmatter When the frontmatter contains a - (or other delimiter) close to the closing frontmatter delimiter, frontmatter detection would fail.
2013-09-18 16:15:46 +00:00
t.Errorf("Frontmatter did not match:\nexp: %q\ngot: %q", string(test.extracted), fm)
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
}
}
}
}
func TestExtractFrontMatterDelim(t *testing.T) {
var (
noErrExpected = true
errExpected = false
)
tests := []struct {
frontmatter string
extracted string
errIsNil bool
}{
{"", "", errExpected},
{"{", "", errExpected},
{"{}", "{}", noErrExpected},
{"{} ", "{}", noErrExpected},
{"{ } ", "{ }", noErrExpected},
{"{ { }", "", errExpected},
{"{ { } }", "{ { } }", noErrExpected},
{"{ { } { } }", "{ { } { } }", noErrExpected},
{"{\n{\n}\n}\n", "{\n{\n}\n}", noErrExpected},
{"{\n \"categories\": \"d\",\n \"tags\": [\n \"a\", \n \"b\", \n \"c\"\n ]\n}\nJSON Front Matter with tags and categories", "{\n \"categories\": \"d\",\n \"tags\": [\n \"a\", \n \"b\", \n \"c\"\n ]\n}", noErrExpected},
Switching to the rjson library which is more friendly to human generated json.
2013-10-02 02:45:24 +00:00
{"{\n \"categories\": \"d\"\n \"tags\": [\n \"a\" \n \"b\" \n \"c\"\n ]\n}\nJSON Front Matter with tags and categories", "{\n \"categories\": \"d\"\n \"tags\": [\n \"a\" \n \"b\" \n \"c\"\n ]\n}", noErrExpected},
parser: Fix handling of quoted brackets in JSON front matter Also remove a broken JSON test. Fixes #3511
2017-06-19 14:45:52 +00:00
// Issue #3511
{`{ "title": "{" }`, `{ "title": "{" }`, noErrExpected},
{`{ "title": "{}" }`, `{ "title": "{}" }`, noErrExpected},
parser: Fix handling of JSON front matter with escaped quotes Fixes #3661
2017-07-03 07:00:17 +00:00
// Issue #3661
{`{ "title": "\"" }`, `{ "title": "\"" }`, noErrExpected},
{`{ "title": "\"{", "other": "\"{}" }`, `{ "title": "\"{", "other": "\"{}" }`, noErrExpected},
{`{ "title": "\"Foo\"" }`, `{ "title": "\"Foo\"" }`, noErrExpected},
{`{ "title": "\"Foo\"\"" }`, `{ "title": "\"Foo\"\"" }`, noErrExpected},
parser: Fix issue with escaped JSON front matter Fixes #3682
2017-07-08 08:56:56 +00:00
{`{ "url": "http:\/\/example.com\/play\/url?id=1" }`, `{ "url": "http:\/\/example.com\/play\/url?id=1" }`, noErrExpected},
parser: Final (!) fix for issue with escaped JSON front matter Fixes #3682
2017-07-08 16:43:36 +00:00
{`{ "test": "\"New\r\nString\"" }`, `{ "test": "\"New\r\nString\"" }`, noErrExpected},
{`{ "test": "RTS\/RPG" }`, `{ "test": "RTS\/RPG" }`, noErrExpected},
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
parser: Fix handling of quoted brackets in JSON front matter Also remove a broken JSON test. Fixes #3511
2017-06-19 14:45:52 +00:00
for i, test := range tests {
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
fm, err := extractFrontMatterDelims(bufio.NewReader(strings.NewReader(test.frontmatter)), []byte("{"), []byte("}"))
if (err == nil) != test.errIsNil {
t.Logf("\n%q\n", string(test.frontmatter))
parser: Fix handling of quoted brackets in JSON front matter Also remove a broken JSON test. Fixes #3511
2017-06-19 14:45:52 +00:00
t.Errorf("[%d] Expected err == nil => %t, got: %t. err: %s", i, test.errIsNil, err == nil, err)
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
continue
}
if !bytes.Equal(fm, []byte(test.extracted)) {
t.Logf("\n%q\n", string(test.frontmatter))
parser: Fix handling of JSON front matter with escaped quotes Fixes #3661
2017-07-03 07:00:17 +00:00
t.Errorf("[%d] Frontmatter did not match:\nexp: %q\ngot: %q", i, string(test.extracted), fm)
Merge remote-tracking branch 'origin/parser' into mrg_praser Also brought in parse for github.com/noahcampbell/akebia Conflicts: hugolib/page.go hugolib/page_test.go
2013-09-17 22:52:40 +00:00
}
}
}
Reference in a new issue Copy permalink