github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-11-21 20:46:30 -05:00
Code Issues Projects Releases Packages Wiki Activity

Render the shortcodes as late as possible

Browse source 
This is needed to make shortcode users happy with the new multilanguage support,
but it will also solve many other related posts about "stuff not available in the shortcode".

We will have to revisit this re the handler chain at some point, but that will be easier
now as the integration test story has improved so much.

As part of this commit, the site-building tests in page_test.go is refreshed, they now
tests for all the rendering engines (when available), and all of them now uses the
same code-path as used in production.

Fixes #1229
Fixes #2323
Fixes ##1076
This commit is contained in:
Bjørn Erik Pedersen 2016-08-01 23:04:44 +02:00
parent 708bc78770
commit ed0985404d
11 changed files with 633 additions and 266 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
helpers/content.go
View file

@ -41,6 +41,8 @@ var SummaryLength = 70
// SummaryDivider denotes where content summarization should end. The default is "<!--more-->".
var SummaryDivider = []byte("<!--more-->")
var summaryDividerAndNewLines = []byte("<!--more-->\n\n")
// Blackfriday holds configuration values for Blackfriday rendering.
type Blackfriday struct {
Smartypants bool
@ -390,8 +392,21 @@ func WordCount(s string) map[string]int {
}
// RemoveSummaryDivider removes summary-divider <!--more--> from content.
// TODO(bep) ml remove
func RemoveSummaryDivider(content []byte) []byte {
return bytes.Replace(content, SummaryDivider, []byte(""), -1)
b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1)
if len(b) != len(content) {
return b
}
return bytes.Replace(content, SummaryDivider, []byte(""), 1)
}
func removeInternalSummaryDivider(content []byte) []byte {
b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1)
if len(b) != len(content) {
return b
}
return bytes.Replace(content, SummaryDivider, []byte(""), 1)
}
// TruncateWordsByRune truncates words by runes.

2
hugolib/handler_meta.go
View file

@ -75,8 +75,6 @@ func (mh *MetaHandle) Convert(i interface{}, s *Site, results HandleResults) {
}
results <- h.PageConvert(p, s.Tmpl)
p.setSummary()
p.analyzePage()
}
}

43
hugolib/handler_page.go
View file

@ -14,10 +14,11 @@
package hugolib
import (
"bytes"
"github.com/spf13/hugo/helpers"
"github.com/spf13/hugo/source"
"github.com/spf13/hugo/tpl"
jww "github.com/spf13/jwalterweatherman"
"github.com/spf13/viper"
)
@ -67,18 +68,7 @@ type htmlHandler struct {
func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
p.ProcessShortcodes(t)
var err error
if len(p.contentShortCodes) > 0 {
p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
if err != nil {
jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
return HandledResult{err: err}
}
}
p.Content = helpers.BytesToHTML(p.rawContent)
return HandledResult{err: nil}
}
@ -112,27 +102,22 @@ func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
func commonConvert(p *Page, t tpl.Template) HandledResult {
p.ProcessShortcodes(t)
var err error
// TODO(bep) these page handlers need to be re-evaluated, as it is hard to
// process a page in isolation. See the new preRender func.
// TODO(bep) ml not so raw anymore, but do we need to keep it raw?
if viper.GetBool("EnableEmoji") {
p.rawContent = helpers.Emojify(p.rawContent)
}
renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
// TODO(bep) ml we let the summary divider survive the rendering. Must check if
// it actually survives, replace it with something more robus, or maybe
// rethink this fragile concept.
//p.rawContent = p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
// We have to replace the <!--more--> with something that survives all the
// rendering engines.
// TODO(bep) inline replace
p.rawContent = bytes.Replace(p.rawContent, []byte(helpers.SummaryDivider), internalSummaryDivider, 1)
p.rawContent = p.renderContent(p.rawContent)
if len(p.contentShortCodes) > 0 {
renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes)
if err != nil {
jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
return HandledResult{err: err}
}
}
tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent)
p.Content = helpers.BytesToHTML(tmpContent)
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
p.rendered = true
return HandledResult{err: nil}
}

109
hugolib/hugo_sites.go
View file

@ -16,8 +16,11 @@ package hugolib
import (
"errors"
"strings"
"sync"
"time"
"github.com/spf13/hugo/helpers"
"github.com/spf13/viper"
"github.com/fsnotify/fsnotify"
@ -106,24 +109,27 @@ func (h HugoSites) Build(config BuildCfg) error {
}
for _, s := range h.Sites {
if err := s.PostProcess(); err != nil {
return err
}
}
if err := h.preRender(); err != nil {
return err
}
for _, s := range h.Sites {
if !config.skipRender {
if err := s.Render(); err != nil {
return err
}
}
if config.PrintStats {
s.Stats()
}
}
// TODO(bep) ml lang in site.Info?
// TODO(bep) ml Page sorting?
}
if config.PrintStats {
@ -153,24 +159,28 @@ func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
// Assign pages to sites per translation.
h.setupTranslations(firstSite)
for _, s := range h.Sites {
if sourceChanged {
for _, s := range h.Sites {
if err := s.PostProcess(); err != nil {
return err
}
}
}
if err := h.preRender(); err != nil {
return err
}
if !config.skipRender {
for _, s := range h.Sites {
if err := s.Render(); err != nil {
return err
}
}
if config.PrintStats {
s.Stats()
}
}
}
if config.PrintStats {
jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds()))
@ -219,6 +229,87 @@ func (s *HugoSites) setupTranslations(master *Site) {
}
}
// preRender performs build tasks that needs to be done as late as possible.
// Shortcode handling is the main task in here.
// TODO(bep) We need to look at the whole handler-chain construct witht he below in mind.
func (h *HugoSites) preRender() error {
pageChan := make(chan *Page)
wg := &sync.WaitGroup{}
// We want all the pages, so just pick one.
s := h.Sites[0]
for i := 0; i < getGoMaxProcs()*4; i++ {
wg.Add(1)
go func(pages <-chan *Page, wg *sync.WaitGroup) {
defer wg.Done()
for p := range pages {
if err := handleShortcodes(p, s.Tmpl); err != nil {
jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
}
if p.Markup == "markdown" {
tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent)
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
p.rawContent = tmpContent
}
if p.Markup != "html" {
// Now we know enough to create a summary of the page and count some words
summaryContent, err := p.setUserDefinedSummaryIfProvided()
if err != nil {
jww.ERROR.Printf("Failed to set use defined summary: %s", err)
} else if summaryContent != nil {
p.rawContent = summaryContent.content
}
p.Content = helpers.BytesToHTML(p.rawContent)
p.rendered = true
if summaryContent == nil {
p.setAutoSummary()
}
}
//analyze for raw stats
p.analyzePage()
}
}(pageChan, wg)
}
for _, p := range s.AllPages {
pageChan <- p
}
close(pageChan)
wg.Wait()
return nil
}
func handleShortcodes(p *Page, t tpl.Template) error {
if len(p.contentShortCodes) > 0 {
jww.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName())
shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes)
if err != nil {
return err
}
p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes)
if err != nil {
jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
}
}
return nil
}
func (s *Site) updateBuildStats(page *Page) {
if page.IsDraft() {
s.draftCount++

5
hugolib/hugo_sites_test.go
View file

@ -37,6 +37,11 @@ func testCommonResetState() {
viper.Set("PublishDir", "public")
viper.Set("RSSUri", "rss")
viper.Set("Taxonomies", map[string]interface{}{
"tag": "tags",
"category": "categories",
})
if err := hugofs.Source().Mkdir("content", 0755); err != nil {
panic("Content folder creation failed.")
}

132
hugolib/page.go
View file

@ -70,7 +70,7 @@ type Page struct {
linkTitle string
frontmatter []byte
rawContent []byte
contentShortCodes map[string]string // TODO(bep) this shouldn't be needed.
contentShortCodes map[string]func() (string, error)
shortcodes map[string]shortcode
plain string // TODO should be []byte
plainWords []string
@ -212,35 +212,100 @@ func (p *Page) lineNumRawContentStart() int {
return bytes.Count(p.frontmatter, []byte("\n")) + 1
}
func (p *Page) setSummary() {
var (
internalSummaryDivider = []byte("HUGOMORE42")
internalSummaryDividerAndNewLines = []byte("HUGOMORE42\n\n")
)
// at this point, p.rawContent contains placeholders for the short codes,
// rendered and ready in p.contentShortcodes
// Returns the page as summary and main if a user defined split is provided.
func (p *Page) setUserDefinedSummaryIfProvided() (*summaryContent, error) {
sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContent)
if sc == nil {
// No divider found
return nil, nil
}
if bytes.Contains(p.rawContent, helpers.SummaryDivider) {
sections := bytes.Split(p.rawContent, helpers.SummaryDivider)
header := sections[0]
p.Truncated = true
if len(sections[1]) < 20 {
if len(sc.content) < 20 {
// only whitespace?
p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0
p.Truncated = len(bytes.Trim(sc.content, " \n\r")) > 0
}
// TODO(bep) consider doing this once only
renderedHeader := p.renderBytes(header)
if len(p.contentShortCodes) > 0 {
tmpContentWithTokensReplaced, err :=
replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, p.contentShortCodes)
if err != nil {
jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error())
} else {
renderedHeader = tmpContentWithTokensReplaced
p.Summary = helpers.BytesToHTML(sc.summary)
return sc, nil
}
// Make this explicit so there is no doubt about what is what.
type summaryContent struct {
summary []byte
content []byte
contentWithoutSummary []byte
}
func splitUserDefinedSummaryAndContent(markup string, c []byte) *summaryContent {
startDivider := bytes.Index(c, internalSummaryDivider)
if startDivider == -1 {
return nil
}
endDivider := startDivider + len(internalSummaryDivider)
endSummary := startDivider
var (
startMarkup []byte
endMarkup []byte
addDiv bool
divStart = []byte("<div class=\"document\">")
)
switch markup {
default:
startMarkup = []byte("<p>")
endMarkup = []byte("</p>")
case "asciidoc":
startMarkup = []byte("<div class=\"paragraph\">")
endMarkup = []byte("</div>")
case "rst":
startMarkup = []byte("<p>")
endMarkup = []byte("</p>")
addDiv = true
}
p.Summary = helpers.BytesToHTML(renderedHeader)
} else {
// If hugo defines split:
// render, strip html, then split
// Find the closest end/start markup string to the divider
//firstStart := bytes.Index(c[:startDivider], startMarkup)
fromIdx := bytes.LastIndex(c[:startDivider], startMarkup)
fromStart := startDivider - fromIdx - len(startMarkup)
fromEnd := bytes.Index(c[endDivider:], endMarkup)
if fromEnd != -1 && fromEnd <= fromStart {
endSummary = startDivider + fromEnd + len(endMarkup)
} else if fromStart != -1 {
endSummary = startDivider - fromStart - len(startMarkup)
}
withoutDivider := bytes.TrimSpace(append(c[:startDivider], c[endDivider:]...))
contentWithoutSummary := bytes.TrimSpace(withoutDivider[endSummary:])
summary := bytes.TrimSpace(withoutDivider[:endSummary])
if addDiv {
// For the rst
summary = append(append([]byte(nil), summary...), []byte("</div>")...)
// TODO(bep) include the document class, maybe
contentWithoutSummary = append(divStart, contentWithoutSummary...)
}
return &summaryContent{
summary: summary,
content: withoutDivider,
contentWithoutSummary: contentWithoutSummary,
}
}
func (p *Page) setAutoSummary() error {
var summary string
var truncated bool
if p.isCJKLanguage {
@ -251,7 +316,7 @@ func (p *Page) setSummary() {
p.Summary = template.HTML(summary)
p.Truncated = truncated
}
return nil
}
func (p *Page) renderBytes(content []byte) []byte {
@ -972,27 +1037,6 @@ func (p *Page) ProcessShortcodes(t tpl.Template) {
}
// TODO(spf13): Remove this entirely
// Here for backwards compatibility & testing. Only works in isolation
func (p *Page) Convert() error {
var h Handler
if p.Markup != "" {
h = FindHandler(p.Markup)
} else {
h = FindHandler(p.File.Extension())
}
if h != nil {
h.PageConvert(p, tpl.T())
}
//// now we know enough to create a summary of the page and count some words
p.setSummary()
//analyze for raw stats
p.analyzePage()
return nil
}
func (p *Page) FullFilePath() string {
return filepath.Join(p.Dir(), p.LogicalName())
}

392
hugolib/page_test.go
View file

@ -27,6 +27,7 @@ import (
"github.com/spf13/hugo/helpers"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var emptyPage = ""
@ -130,6 +131,15 @@ Summary Next Line
<!--more-->
Some more text
`
simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder = `---
title: Simple
---
The [best static site generator][hugo].[^1]
<!--more-->
[hugo]: http://gohugo.io/
[^1]: Many people say so.
`
simplePageWithShortcodeInSummary = `---
title: Simple
@ -485,21 +495,33 @@ func checkPageTitle(t *testing.T, page *Page, title string) {
}
}
func checkPageContent(t *testing.T, page *Page, content string) {
if page.Content != template.HTML(content) {
t.Fatalf("Page content is:\n%q\nExpected:\n%q", page.Content, content)
func checkPageContent(t *testing.T, page *Page, content string, msg ...interface{}) {
a := normalizeContent(content)
b := normalizeContent(string(page.Content))
if a != b {
t.Fatalf("Page content is:\n%q\nExpected:\n%q (%q)", b, a, msg)
}
}
func normalizeContent(c string) string {
norm := strings.Replace(c, "\n", "", -1)
norm = strings.Replace(norm, " ", " ", -1)
norm = strings.Replace(norm, " ", " ", -1)
norm = strings.Replace(norm, " ", " ", -1)
return strings.TrimSpace(norm)
}
func checkPageTOC(t *testing.T, page *Page, toc string) {
if page.TableOfContents != template.HTML(toc) {
t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents, toc)
}
}
func checkPageSummary(t *testing.T, page *Page, summary string) {
if page.Summary != template.HTML(summary) {
t.Fatalf("Page summary is: %q.\nExpected %q", page.Summary, summary)
func checkPageSummary(t *testing.T, page *Page, summary string, msg ...interface{}) {
a := normalizeContent(string(page.Summary))
b := normalizeContent(summary)
if a != b {
t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg)
}
}
@ -534,147 +556,285 @@ func checkTruncation(t *testing.T, page *Page, shouldBe bool, msg string) {
}
}
func TestCreateNewPage(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePage))
p.Convert()
func normalizeExpected(ext, str string) string {
str = normalizeContent(str)
switch ext {
default:
return str
case "html":
return strings.Trim(helpers.StripHTML(str), " ")
case "ad":
paragraphs := strings.Split(str, "</p>")
expected := ""
for _, para := range paragraphs {
if para == "" {
continue
}
expected += fmt.Sprintf("<div class=\"paragraph\">\n%s</p></div>\n", para)
}
return expected
case "rst":
return fmt.Sprintf("<div class=\"document\">\n\n\n%s</div>", str)
}
}
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
func testAllMarkdownEnginesForPage(t *testing.T,
assertFunc func(t *testing.T, ext string, p *Page), baseFilename, pageContent string) {
engines := []struct {
ext string
shouldExecute func() bool
}{
{"ad", func() bool { return helpers.HasAsciidoc() }},
{"md", func() bool { return true }},
{"mmark", func() bool { return true }},
// TODO(bep) figure a way to include this without too much work.{"html", func() bool { return true }},
{"rst", func() bool { return helpers.HasRst() }},
}
for _, e := range engines {
if !e.shouldExecute() {
continue
}
filename := baseFilename + "." + e.ext
s := newSiteFromSources(filename, pageContent)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
assertFunc(t, e.ext, p)
}
}
func TestCreateNewPage(t *testing.T) {
assertFunc := func(t *testing.T, ext string, p *Page) {
assert.False(t, p.IsHome)
checkPageTitle(t, p, "Simple")
checkPageContent(t, p, "<p>Simple Page</p>\n")
checkPageContent(t, p, normalizeExpected(ext, "<p>Simple Page</p>\n"))
checkPageSummary(t, p, "Simple Page")
checkPageType(t, p, "page")
checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
checkTruncation(t, p, false, "simple short page")
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePage)
}
func TestSplitSummaryAndContent(t *testing.T) {
for i, this := range []struct {
markup string
content string
expectedSummary string
expectedContent string
expectedContentWithoutSummary string
}{
{"markdown", `<p>Summary Same LineHUGOMORE42</p>
<p>Some more text</p>`, "<p>Summary Same Line</p>", "<p>Summary Same Line</p>\n\n<p>Some more text</p>", "<p>Some more text</p>"},
{"asciidoc", `<div class="paragraph"><p>sn</p></div><div class="paragraph"><p>HUGOMORE42Some more text</p></div>`,
"<div class=\"paragraph\"><p>sn</p></div>",
"<div class=\"paragraph\"><p>sn</p></div><div class=\"paragraph\"><p>Some more text</p></div>",
"<div class=\"paragraph\"><p>Some more text</p></div>"},
{"rst",
"<div class=\"document\"><p>Summary Next Line</p><p>HUGOMORE42Some more text</p></div>",
"<div class=\"document\"><p>Summary Next Line</p></div>",
"<div class=\"document\"><p>Summary Next Line</p><p>Some more text</p></div>",
"<div class=\"document\"><p>Some more text</p></div>"},
{"markdown", "<p>a</p><p>b</p><p>HUGOMORE42c</p>", "<p>a</p><p>b</p>", "<p>a</p><p>b</p><p>c</p>", "<p>c</p>"},
{"markdown", "<p>a</p><p>b</p><p>cHUGOMORE42</p>", "<p>a</p><p>b</p><p>c</p>", "<p>a</p><p>b</p><p>c</p>", ""},
{"markdown", "<p>a</p><p>bHUGOMORE42</p><p>c</p>", "<p>a</p><p>b</p>", "<p>a</p><p>b</p><p>c</p>", "<p>c</p>"},
{"markdown", "<p>aHUGOMORE42</p><p>b</p><p>c</p>", "<p>a</p>", "<p>a</p><p>b</p><p>c</p>", "<p>b</p><p>c</p>"},
} {
sc := splitUserDefinedSummaryAndContent(this.markup, []byte(this.content))
require.NotNil(t, sc, fmt.Sprintf("[%d] Nil %s", i, this.markup))
require.Equal(t, this.expectedSummary, string(sc.summary), fmt.Sprintf("[%d] Summary markup %s", i, this.markup))
require.Equal(t, this.expectedContent, string(sc.content), fmt.Sprintf("[%d] Content markup %s", i, this.markup))
require.Equal(t, this.expectedContentWithoutSummary, string(sc.contentWithoutSummary), fmt.Sprintf("[%d] Content without summary, markup %s", i, this.markup))
}
if true {
return
}
ad := `<div class="paragraph"><p>sn</p></div>
<div class="paragraph">
<p>HUGOMORE42
Some more text</p>
</div>
`
md := `<p>Summary Same LineHUGOMORE42</p>
<p>Some more text</p>`
sc := splitUserDefinedSummaryAndContent("markdown", []byte(md))
require.Equal(t, "adf", string(sc.summary))
require.Equal(t, "asdf", string(sc.content))
if true {
return
}
sc = splitUserDefinedSummaryAndContent("asciidoc", []byte(ad))
require.Equal(t, "<div class=\"paragraph\"><p>sn</p></div>", string(sc.summary))
require.Equal(t, "\n<div class=\"paragraph\">\n<p> \nSome more text</p>\n</div>\n", string(sc.summary))
}
func TestPageWithDelimiter(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithSummaryDelimiter))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
assertFunc := func(t *testing.T, ext string, p *Page) {
checkPageTitle(t, p, "Simple")
checkPageContent(t, p, "<p>Summary Next Line</p>\n\n<p>Some more text</p>\n")
checkPageSummary(t, p, "<p>Summary Next Line</p>\n")
checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Next Line</p>\n\n<p>Some more text</p>\n"), ext)
checkPageSummary(t, p, normalizeExpected(ext, "<p>Summary Next Line</p>"), ext)
checkPageType(t, p, "page")
checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
checkTruncation(t, p, true, "page with summary delimiter")
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiter)
}
// Issue #1076
func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
s := newSiteFromSources("simple.md", simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
if p.Summary != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a rel=\"footnote\" href=\"#fn:1\">1</a></sup>\n</p>") {
t.Fatalf("Got summary:\n%q", p.Summary)
}
if p.Content != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a rel=\"footnote\" href=\"#fn:1\">1</a></sup>\n</p>\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>") {
t.Fatalf("Got content:\n%q", p.Content)
}
}
func TestPageWithShortCodeInSummary(t *testing.T) {
s := new(Site)
s.prepTemplates(nil)
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithShortcodeInSummary))
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
p.Convert()
assertFunc := func(t *testing.T, ext string, p *Page) {
checkPageTitle(t, p, "Simple")
checkPageContent(t, p, "<p>Summary Next Line. \n<figure >\n \n <img src=\"/not/real\" />\n \n \n</figure>\n.\nMore text here.</p>\n\n<p>Some more text</p>\n")
checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text")
checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Next Line. <figure > <img src=\"/not/real\" /> </figure>.\nMore text here.</p><p>Some more text</p>"), ext)
checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text", ext)
checkPageType(t, p, "page")
checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithShortcodeInSummary)
}
func TestPageWithEmbeddedScriptTag(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithEmbeddedScript))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
assertFunc := func(t *testing.T, ext string, p *Page) {
if ext == "ad" || ext == "rst" {
// TOD(bep)
return
}
checkPageContent(t, p, "<script type='text/javascript'>alert('the script tags are still there, right?');</script>\n")
checkPageContent(t, p, "<script type='text/javascript'>alert('the script tags are still there, right?');</script>\n", ext)
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithEmbeddedScript)
}
func TestPageWithAdditionalExtension(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithAdditionalExtension))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
s := newSiteFromSources("simple.md", simplePageWithAdditionalExtension)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
checkPageContent(t, p, "<p>first line.<br />\nsecond line.</p>\n\n<p>fourth line.</p>\n")
}
func TestTableOfContents(t *testing.T) {
p, _ := NewPage("tocpage.md")
_, err := p.ReadFrom(strings.NewReader(pageWithToC))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
s := newSiteFromSources("tocpage.md", pageWithToC)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
checkPageContent(t, p, "\n\n<p>For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.</p>\n\n<h2 id=\"aa\">AA</h2>\n\n<p>I have no idea, of course, how long it took me to reach the limit of the plain,\nbut at last I entered the foothills, following a pretty little canyon upward\ntoward the mountains. Beside me frolicked a laughing brooklet, hurrying upon\nits noisy way down to the silent sea. In its quieter pools I discovered many\nsmall fish, of four-or five-pound weight I should imagine. In appearance,\nexcept as to size and color, they were not unlike the whale of our own seas. As\nI watched them playing about I discovered, not only that they suckled their\nyoung, but that at intervals they rose to the surface to breathe as well as to\nfeed upon certain grasses and a strange, scarlet lichen which grew upon the\nrocks just above the water line.</p>\n\n<h3 id=\"aaa\">AAA</h3>\n\n<p>I remember I felt an extraordinary persuasion that I was being played with,\nthat presently, when I was upon the very verge of safety, this mysterious\ndeath&ndash;as swift as the passage of light&ndash;would leap after me from the pit about\nthe cylinder and strike me down. ## BB</p>\n\n<h3 id=\"bbb\">BBB</h3>\n\n<p>&ldquo;You&rsquo;re a great Granser,&rdquo; he cried delightedly, &ldquo;always making believe them little marks mean something.&rdquo;</p>\n")
checkPageTOC(t, p, "<nav id=\"TableOfContents\">\n<ul>\n<li>\n<ul>\n<li><a href=\"#aa\">AA</a>\n<ul>\n<li><a href=\"#aaa\">AAA</a></li>\n<li><a href=\"#bbb\">BBB</a></li>\n</ul></li>\n</ul></li>\n</ul>\n</nav>")
}
func TestPageWithMoreTag(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithSummaryDelimiterSameLine))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
assertFunc := func(t *testing.T, ext string, p *Page) {
checkPageTitle(t, p, "Simple")
checkPageContent(t, p, "<p>Summary Same Line</p>\n\n<p>Some more text</p>\n")
checkPageSummary(t, p, "<p>Summary Same Line</p>\n")
checkPageContent(t, p, normalizeExpected(ext, "<p>Summary Same Line</p>\n\n<p>Some more text</p>\n"))
checkPageSummary(t, p, normalizeExpected(ext, "<p>Summary Same Line</p>"))
checkPageType(t, p, "page")
checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html")
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiterSameLine)
}
func TestPageWithDate(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageRFC3339Date))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
d, err := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z")
if err != nil {
t.Fatalf("Unable to prase page.")
s := newSiteFromSources("simple.md", simplePageRFC3339Date)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
d, _ := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z")
checkPageDate(t, p, d)
}
func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) {
testCommonResetState()
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithAllCJKRunes))
p.Convert()
p.analyzePage()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
assertFunc := func(t *testing.T, ext string, p *Page) {
if p.WordCount != 8 {
t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 8, p.WordCount)
}
}
if p.WordCount != 8 {
t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 8, p.WordCount)
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithAllCJKRunes)
}
func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) {
testCommonResetState()
viper.Set("HasCJKLanguage", true)
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithAllCJKRunes))
p.Convert()
p.analyzePage()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
assertFunc := func(t *testing.T, ext string, p *Page) {
if p.WordCount != 15 {
t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 15, p.WordCount)
}
}
if p.WordCount != 15 {
t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 15, p.WordCount)
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithAllCJKRunes)
}
func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) {
@ -682,69 +842,60 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) {
viper.Set("HasCJKLanguage", true)
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithMainEnglishWithCJKRunes))
p.Convert()
p.analyzePage()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
assertFunc := func(t *testing.T, ext string, p *Page) {
if p.WordCount != 74 {
t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 74, p.WordCount)
t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount)
}
if p.Summary != simplePageWithMainEnglishWithCJKRunesSummary {
t.Fatalf("incorrect Summary for content '%s'. expected %v, got %v", p.plain,
t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain,
simplePageWithMainEnglishWithCJKRunesSummary, p.Summary)
}
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithMainEnglishWithCJKRunes)
}
func TestWordCountWithIsCJKLanguageFalse(t *testing.T) {
testCommonResetState()
viper.Set("HasCJKLanguage", true)
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithIsCJKLanguageFalse))
p.Convert()
p.analyzePage()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
assertFunc := func(t *testing.T, ext string, p *Page) {
if p.WordCount != 75 {
t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 75, p.WordCount)
t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount)
}
if p.Summary != simplePageWithIsCJKLanguageFalseSummary {
t.Fatalf("incorrect Summary for content '%s'. expected %v, got %v", p.plain,
t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain,
simplePageWithIsCJKLanguageFalseSummary, p.Summary)
}
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithIsCJKLanguageFalse)
}
func TestWordCount(t *testing.T) {
p, _ := NewPage("simple.md")
_, err := p.ReadFrom(strings.NewReader(simplePageWithLongContent))
p.Convert()
p.analyzePage()
if err != nil {
t.Fatalf("Unable to create a page with frontmatter and body content: %s", err)
}
assertFunc := func(t *testing.T, ext string, p *Page) {
if p.WordCount != 483 {
t.Fatalf("incorrect word count. expected %v, got %v", 483, p.WordCount)
t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount)
}
if p.FuzzyWordCount != 500 {
t.Fatalf("incorrect word count. expected %v, got %v", 500, p.WordCount)
t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.WordCount)
}
if p.ReadingTime != 3 {
t.Fatalf("incorrect min read. expected %v, got %v", 3, p.ReadingTime)
t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime)
}
checkTruncation(t, p, true, "long page")
}
testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithLongContent)
}
func TestCreatePage(t *testing.T) {
@ -1049,15 +1200,18 @@ func TestPageSimpleMethods(t *testing.T) {
}
func TestChompBOM(t *testing.T) {
p, _ := NewPage("simple.md")
const utf8BOM = "\xef\xbb\xbf"
_, err := p.ReadFrom(strings.NewReader(utf8BOM + simplePage))
p.Convert()
if err != nil {
t.Fatalf("Unable to create a page with BOM prefixed frontmatter and body content: %s", err)
s := newSiteFromSources("simple.md", utf8BOM+simplePage)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
checkPageTitle(t, p, "Simple")
}

33
hugolib/shortcode.go
View file

@ -160,7 +160,11 @@ func HandleShortcodes(stringToParse string, page *Page, t tpl.Template) (string,
}
if len(tmpShortcodes) > 0 {
tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, tmpShortcodes)
shortcodes, err := executeShortcodeFuncMap(tmpShortcodes)
if err != nil {
return "", err
}
tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, shortcodes)
if err != nil {
return "", fmt.Errorf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error())
@ -274,7 +278,7 @@ func renderShortcode(sc shortcode, parent *ShortcodeWithPage, p *Page, t tpl.Tem
return renderShortcodeWithPage(tmpl, data)
}
func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]string, error) {
func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]func() (string, error), error) {
if p.rendered {
panic("Illegal state: Page already marked as rendered, please reuse the shortcodes")
@ -297,15 +301,32 @@ func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (
}
func renderShortcodes(shortcodes map[string]shortcode, p *Page, t tpl.Template) map[string]string {
renderedShortcodes := make(map[string]string)
var emptyShortcodeFn = func() (string, error) { return "", nil }
func executeShortcodeFuncMap(funcs map[string]func() (string, error)) (map[string]string, error) {
result := make(map[string]string)
for k, v := range funcs {
s, err := v()
if err != nil {
return nil, fmt.Errorf("Failed to execute shortcode with key %s: %s", k, err)
}
result[k] = s
}
return result, nil
}
func renderShortcodes(shortcodes map[string]shortcode, p *Page, t tpl.Template) map[string]func() (string, error) {
renderedShortcodes := make(map[string]func() (string, error))
for key, sc := range shortcodes {
if sc.err != nil {
// need to have something to replace with
renderedShortcodes[key] = ""
renderedShortcodes[key] = emptyShortcodeFn
} else {
renderedShortcodes[key] = renderShortcode(sc, nil, p, t)
shorctode := sc
renderedShortcodes[key] = func() (string, error) { return renderShortcode(shorctode, nil, p, t), nil }
}
}

31
hugolib/shortcode_test.go
View file

@ -484,9 +484,34 @@ e`,
{"sect/doc8.rst", `**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`,
filepath.FromSlash("sect/doc8/index.html"),
"<div class=\"document\">\n\n\n<p><strong>Shortcodes:</strong> <em>b: b c: c</em></p>\n</div>"},
{"sect/doc9.mmark", `**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`,
{"sect/doc9.mmark", `
---
menu:
main:
parent: 'parent'
---
**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`,
filepath.FromSlash("sect/doc9/index.html"),
"<p><strong>Shortcodes:</strong> <em>b: b c: c</em></p>\n"},
// Issue #1229: Menus not available in shortcode.
{"sect/doc10.md", `---
menu:
main:
identifier: 'parent'
tags:
- Menu
---
**Menus:** {{< menu >}}`,
filepath.FromSlash("sect/doc10/index.html"),
"<p><strong>Menus:</strong> 1</p>\n"},
// Issue #2323: Taxonomies not available in shortcode.
{"sect/doc11.md", `---
tags:
- Bugs
---
**Tags:** {{< tags >}}`,
filepath.FromSlash("sect/doc11/index.html"),
"<p><strong>Tags:</strong> 2</p>\n"},
}
sources := make([]source.ByteSource, len(tests))
@ -507,6 +532,8 @@ e`,
templ.AddInternalShortcode("b.html", `b`)
templ.AddInternalShortcode("c.html", `c`)
templ.AddInternalShortcode("d.html", `d`)
templ.AddInternalShortcode("menu.html", `{{ len (index .Page.Menus "main").Children }}`)
templ.AddInternalShortcode("tags.html", `{{ len .Page.Site.Taxonomies.tags }}`)
return nil
@ -540,7 +567,7 @@ e`,
content := helpers.ReaderToString(file)
if content != test.expected {
t.Errorf("%s content expected:\n%q\ngot:\n%q", test.outFile, test.expected, content)
t.Fatalf("%s content expected:\n%q\ngot:\n%q", test.outFile, test.expected, content)
}
}

20
hugolib/site.go
View file

@ -113,6 +113,26 @@ func newSiteDefaultLang() *Site {
return NewSite(newDefaultLanguage())
}
// Convenience func used in tests.
func newSiteFromSources(pathContentPairs ...string) *Site {
if len(pathContentPairs)%2 != 0 {
panic("pathContentPairs must come in pairs")
}
sources := make([]source.ByteSource, 0)
for i := 0; i < len(pathContentPairs); i += 2 {
path := pathContentPairs[i]
content := pathContentPairs[i+1]
sources = append(sources, source.ByteSource{Name: filepath.FromSlash(path), Content: []byte(content)})
}
return &Site{
Source: &source.InMemorySource{ByteSource: sources},
Language: newDefaultLanguage(),
}
}
type targetList struct {
page target.Output
pageUgly target.Output

15
hugolib/site_test.go
View file

@ -30,6 +30,7 @@ import (
"github.com/spf13/hugo/target"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
@ -84,10 +85,16 @@ func pageMust(p *Page, err error) *Page {
}
func TestDegenerateRenderThingMissingTemplate(t *testing.T) {
p, _ := NewPageFrom(strings.NewReader(pageSimpleTitle), "content/a/file.md")
p.Convert()
s := new(Site)
s.prepTemplates(nil)
s := newSiteFromSources("content/a/file.md", pageSimpleTitle)
if err := buildSiteSkipRender(s); err != nil {
t.Fatalf("Failed to build site: %s", err)
}
require.Len(t, s.Pages, 1)
p := s.Pages[0]
err := s.renderThing(p, "foobar", nil)
if err == nil {
t.Errorf("Expected err to be returned when missing the template.")