Make .Content (almost) always available in shortcodes

This resolves some surprising behaviour when reading other pages' content from shortcodes. Before this commit, that behaviour was undefined. Note that this has never been an issue from regular templates.

It will still not be possible to get **the current shortcode's  page's rendered content**. That would have impressed Einstein.

The new and well defined rules are:

* `.Page.Content` from a shortcode will be empty. The related `.Page.Truncated` `.Page.Summary`, `.Page.WordCount`, `.Page.ReadingTime`, `.Page.Plain` and `.Page.PlainWords` will also have empty values.
* For _other pages_ (retrieved via `.Page.Site.GetPage`, `.Site.Pages` etc.) the `.Content` is there to use as you please as long as you don't have infinite content recursion in your shortcode/content setup. See below.
* `.Page.TableOfContents` is good to go (but does not support shortcodes in headlines; this is unchanged)

If you get into a situation of infinite recursion, the `.Content` will be empty. Run `hugo -v` for more information.

Fixes #4632
Fixes #4653
Fixes #4655

deps/deps.go

@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"time"
 
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
@@ -54,6 +55,9 @@ type Deps struct {
 	translationProvider ResourceProvider
 
 	Metrics metrics.Provider
+
+	// Timeout is configurable in site config.
+	Timeout time.Duration
 }
 
 // ResourceProvider is used to create and refresh, and clone resources needed.
@@ -128,6 +132,11 @@ func New(cfg DepsCfg) (*Deps, error) {
 
 	sp := source.NewSourceSpec(ps, fs.Source)
 
+	timeoutms := cfg.Language.GetInt("timeout")
+	if timeoutms <= 0 {
+		timeoutms = 3000
+	}
+
 	d := &Deps{
 		Fs:                  fs,
 		Log:                 logger,
@@ -139,6 +148,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		SourceSpec:          sp,
 		Cfg:                 cfg.Language,
 		Language:            cfg.Language,
+		Timeout:             time.Duration(timeoutms) * time.Millisecond,
 	}
 
 	if cfg.Cfg.GetBool("templateMetrics") {

helpers/content.go

@@ -400,6 +400,9 @@ func (c ContentSpec) mmarkRender(ctx *RenderingContext) []byte {
 
 // ExtractTOC extracts Table of Contents from content.
 func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
+	if !bytes.Contains(content, []byte("<nav>")) {
+		return content, nil
+	}
 	origContent := make([]byte, len(content))
 	copy(origContent, content)
 	first := []byte(`<nav>

hugolib/config.go

@@ -435,6 +435,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("disableAliases", false)
 	v.SetDefault("debug", false)
 	v.SetDefault("disableFastRender", false)
+	v.SetDefault("timeout", 10000) // 10 seconds
 
 	// Remove in Hugo 0.39
 

hugolib/embedded_shortcodes_test.go

@@ -69,7 +69,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 
 	require.Len(t, s.RegularPages, 1)
 
-	output := string(s.RegularPages[0].content)
+	output := string(s.RegularPages[0].content())
 
 	if !strings.Contains(output, expected) {
 		t.Errorf("Got\n%q\nExpected\n%q", output, expected)

hugolib/hugo_sites.go

@@ -560,37 +560,22 @@ func (h *HugoSites) setupTranslations() {
 }
 
 func (s *Site) preparePagesForRender(cfg *BuildCfg) {
-
-	pageChan := make(chan *Page)
-	wg := &sync.WaitGroup{}
-
-	numWorkers := getGoMaxProcs() * 4
-
-	for i := 0; i < numWorkers; i++ {
-		wg.Add(1)
-		go func(pages <-chan *Page, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for p := range pages {
-				if err := p.prepareForRender(cfg); err != nil {
-					s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
-
-				}
-			}
-		}(pageChan, wg)
-	}
-
 	for _, p := range s.Pages {
-		pageChan <- p
+		p.setContentInit(cfg)
+		// The skip render flag is used in many tests. To make sure that they
+		// have access to the content, we need to manually initialize it here.
+		if cfg.SkipRender {
+			p.initContent()
+		}
 	}
 
 	for _, p := range s.headlessPages {
-		pageChan <- p
+		p.setContentInit(cfg)
+		if cfg.SkipRender {
+			p.initContent()
+		}
 	}
 
-	close(pageChan)
-
-	wg.Wait()
-
 }
 
 // Pages returns all pages for all sites.
@@ -598,7 +583,7 @@ func (h *HugoSites) Pages() Pages {
 	return h.Sites[0].AllPages
 }
 
-func handleShortcodes(p *Page, rawContentCopy []byte) ([]byte, error) {
+func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) {
 	if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {
 		p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())
 		err := p.shortcodeState.executeShortcodesForDelta(p)

hugolib/hugo_sites_build.go

@@ -224,6 +224,7 @@ func (h *HugoSites) render(config *BuildCfg) error {
 		s.initRenderFormats()
 		for i, rf := range s.renderFormats {
 			s.rc = &siteRenderingContext{Format: rf}
+
 			s.preparePagesForRender(config)
 
 			if !config.SkipRender {

hugolib/hugo_sites_build_test.go

@@ -378,9 +378,9 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	b.AssertFileContent("public/en/tags/tag1/index.html", "Tag1|Hello|http://example.com/blog/en/tags/tag1/")
 
 	// Check Blackfriday config
-	require.True(t, strings.Contains(string(doc1fr.content), "«"), string(doc1fr.content))
+	require.True(t, strings.Contains(string(doc1fr.content()), "«"), string(doc1fr.content()))
-	require.False(t, strings.Contains(string(doc1en.content), "«"), string(doc1en.content))
+	require.False(t, strings.Contains(string(doc1en.content()), "«"), string(doc1en.content()))
-	require.True(t, strings.Contains(string(doc1en.content), "“"), string(doc1en.content))
+	require.True(t, strings.Contains(string(doc1en.content()), "“"), string(doc1en.content()))
 
 	// Check that the drafts etc. are not built/processed/rendered.
 	assertShouldNotBuild(t, b.H)
@@ -630,9 +630,9 @@ func assertShouldNotBuild(t *testing.T, sites *HugoSites) {
 	for _, p := range s.rawAllPages {
 		// No HTML when not processed
 		require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("</")), p.BaseFileName()+": "+string(p.workContent))
-		require.Equal(t, p.shouldBuild(), p.content != "", p.BaseFileName())
+		require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
 
-		require.Equal(t, p.shouldBuild(), p.content != "", p.BaseFileName())
+		require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
 
 	}
 }
@@ -753,6 +753,29 @@ var tocShortcode = `
 {{ .Page.TableOfContents }}
 `
 
+func TestSelfReferencedContentInShortcode(t *testing.T) {
+	t.Parallel()
+
+	b := newMultiSiteTestDefaultBuilder(t)
+
+	var (
+		shortcode = `{{- .Page.Content -}}{{- .Page.Summary -}}{{- .Page.Plain -}}{{- .Page.PlainWords -}}{{- .Page.WordCount -}}{{- .Page.ReadingTime -}}`
+
+		page = `---
+title: sctest
+---
+Empty:{{< mycontent >}}:
+`
+	)
+
+	b.WithTemplatesAdded("layouts/shortcodes/mycontent.html", shortcode)
+	b.WithContent("post/simple.en.md", page)
+
+	b.CreateSites().Build(BuildCfg{})
+
+	b.AssertFileContent("public/en/post/simple/index.html", "Empty:[]00:")
+}
+
 var tocPageSimple = `---
 title: tocTest
 publishdate: "2000-01-01"

hugolib/page.go

@@ -15,6 +15,7 @@ package hugolib
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"reflect"
@@ -89,6 +90,7 @@ const (
 
 type Page struct {
 	*pageInit
+	*pageContentInit
 
 	// Kind is the discriminator that identifies the different page types
 	// in the different page collections. This can, as an example, be used
@@ -127,17 +129,22 @@ type Page struct {
 	// Params contains configuration defined in the params section of page frontmatter.
 	params map[string]interface{}
 
+	// Called when needed to init the content (render shortcodes etc.).
+	contentInitFn func(p *Page) func()
+
 	// Content sections
-	content         template.HTML
+	contentv        template.HTML
-	Summary         template.HTML
+	summary         template.HTML
 	TableOfContents template.HTML
+	// Passed to the shortcodes
+	pageWithoutContent *PageWithoutContent
 
 	Aliases []string
 
 	Images []Image
 	Videos []Video
 
-	Truncated bool
+	truncated bool
 	Draft     bool
 	Status    string
 
@@ -263,8 +270,69 @@ type Page struct {
 	targetPathDescriptorPrototype *targetPathDescriptor
 }
 
+func (p *Page) initContent() {
+	p.contentInit.Do(func() {
+		// This careful dance is here to protect against circular loops in shortcode/content
+		// constructs.
+		// TODO(bep) context vs the remote shortcodes
+		ctx, cancel := context.WithTimeout(context.Background(), p.s.Timeout)
+		defer cancel()
+		c := make(chan error, 1)
+
+		go func() {
+			var err error
+			p.contentInitMu.Lock()
+			defer p.contentInitMu.Unlock()
+
+			if p.contentInitFn != nil {
+				p.contentInitFn(p)()
+			}
+			if len(p.summary) == 0 {
+				if err = p.setAutoSummary(); err != nil {
+					err = fmt.Errorf("Failed to set user auto summary for page %q: %s", p.pathOrTitle(), err)
+				}
+			}
+			c <- err
+		}()
+
+		select {
+		case <-ctx.Done():
+			p.s.Log.WARN.Printf(`WARNING: Timed out creating content for page %q (.Content will be empty). This is most likely a circular shortcode content loop that should be fixed. If this is just a shortcode calling a slow remote service, try to set "timeout=20000" (or higher, value is in milliseconds) in config.toml.`, p.pathOrTitle())
+		case err := <-c:
+			if err != nil {
+				p.s.Log.ERROR.Println(err)
+			}
+		}
+	})
+
+}
+
+// This is sent to the shortcodes for this page. Not doing that will create an infinite regress. So,
+// shortcodes can access .Page.TableOfContents, but not .Page.Content etc.
+func (p *Page) withoutContent() *PageWithoutContent {
+	p.pageInit.withoutContentInit.Do(func() {
+		p.pageWithoutContent = &PageWithoutContent{Page: p}
+	})
+	return p.pageWithoutContent
+}
+
 func (p *Page) Content() (interface{}, error) {
-	return p.content, nil
+	return p.content(), nil
+}
+
+func (p *Page) Truncated() bool {
+	p.initContent()
+	return p.truncated
+}
+
+func (p *Page) content() template.HTML {
+	p.initContent()
+	return p.contentv
+}
+
+func (p *Page) Summary() template.HTML {
+	p.initContent()
+	return p.summary
 }
 
 // Sites is a convenience method to get all the Hugo sites/languages configured.
@@ -341,9 +409,25 @@ type pageInit struct {
 	pageMenusInit       sync.Once
 	pageMetaInit        sync.Once
 	pageOutputInit      sync.Once
-	plainInit           sync.Once
-	plainWordsInit      sync.Once
 	renderingConfigInit sync.Once
+	withoutContentInit  sync.Once
+}
+
+type pageContentInit struct {
+	contentInitMu  sync.Mutex
+	contentInit    sync.Once
+	plainInit      sync.Once
+	plainWordsInit sync.Once
+}
+
+func (p *Page) resetContent(init func(page *Page) func()) {
+	p.pageContentInit = &pageContentInit{}
+	if init == nil {
+		init = func(page *Page) func() {
+			return func() {}
+		}
+	}
+	p.contentInitFn = init
 }
 
 // IsNode returns whether this is an item of one of the list types in Hugo,
@@ -455,26 +539,34 @@ func (p *Page) createWorkContentCopy() {
 }
 
 func (p *Page) Plain() string {
-	p.initPlain()
+	p.initContent()
+	p.initPlain(true)
 	return p.plain
 }
 
-func (p *Page) PlainWords() []string {
+func (p *Page) initPlain(lock bool) {
-	p.initPlainWords()
-	return p.plainWords
-}
-
-func (p *Page) initPlain() {
 	p.plainInit.Do(func() {
-		p.plain = helpers.StripHTML(string(p.content))
+		if lock {
-		return
+			p.contentInitMu.Lock()
+			defer p.contentInitMu.Unlock()
+		}
+		p.plain = helpers.StripHTML(string(p.contentv))
 	})
 }
 
-func (p *Page) initPlainWords() {
+func (p *Page) PlainWords() []string {
+	p.initContent()
+	p.initPlainWords(true)
+	return p.plainWords
+}
+
+func (p *Page) initPlainWords(lock bool) {
 	p.plainWordsInit.Do(func() {
-		p.plainWords = strings.Fields(p.Plain())
+		if lock {
-		return
+			p.contentInitMu.Lock()
+			defer p.contentInitMu.Unlock()
+		}
+		p.plainWords = strings.Fields(p.plain)
 	})
 }
 
@@ -622,7 +714,7 @@ func (p *Page) replaceDivider(content []byte) []byte {
 
 	replaced, truncated := replaceDivider(content, summaryDivider, internalSummaryDivider)
 
-	p.Truncated = truncated
+	p.truncated = truncated
 
 	return replaced
 }
@@ -641,7 +733,7 @@ func (p *Page) setUserDefinedSummaryIfProvided(rawContentCopy []byte) (*summaryC
 		return nil, nil
 	}
 
-	p.Summary = helpers.BytesToHTML(sc.summary)
+	p.summary = helpers.BytesToHTML(sc.summary)
 
 	return sc, nil
 }
@@ -731,15 +823,21 @@ func splitUserDefinedSummaryAndContent(markup string, c []byte) (sc *summaryCont
 func (p *Page) setAutoSummary() error {
 	var summary string
 	var truncated bool
+	// This careful init dance could probably be refined, but it is purely for performance
+	// reasons. These "plain" methods are expensive if the plain content is never actually
+	// used.
+	p.initPlain(false)
 	if p.isCJKLanguage {
-		summary, truncated = p.s.ContentSpec.TruncateWordsByRune(p.PlainWords())
+		p.initPlainWords(false)
+		summary, truncated = p.s.ContentSpec.TruncateWordsByRune(p.plainWords)
 	} else {
-		summary, truncated = p.s.ContentSpec.TruncateWordsToWholeSentence(p.Plain())
+		summary, truncated = p.s.ContentSpec.TruncateWordsToWholeSentence(p.plain)
 	}
-	p.Summary = template.HTML(summary)
+	p.summary = template.HTML(summary)
-	p.Truncated = truncated
+	p.truncated = truncated
 
 	return nil
+
 }
 
 func (p *Page) renderContent(content []byte) []byte {
@@ -788,11 +886,12 @@ func (s *Site) newPage(filename string) *Page {
 
 func (s *Site) newPageFromFile(fi *fileInfo) *Page {
 	return &Page{
-		pageInit:    &pageInit{},
+		pageInit:        &pageInit{},
-		Kind:        kindFromFileInfo(fi),
+		pageContentInit: &pageContentInit{},
-		contentType: "",
+		Kind:            kindFromFileInfo(fi),
-		Source:      Source{File: fi},
+		contentType:     "",
-		Keywords:    []string{}, Sitemap: Sitemap{Priority: -1},
+		Source:          Source{File: fi},
+		Keywords:        []string{}, Sitemap: Sitemap{Priority: -1},
 		params:       make(map[string]interface{}),
 		translations: make(Pages, 0),
 		sections:     sectionsFromFile(fi),
@@ -876,10 +975,11 @@ func (p *Page) FuzzyWordCount() int {
 }
 
 func (p *Page) analyzePage() {
+	p.initContent()
 	p.pageMetaInit.Do(func() {
 		if p.isCJKLanguage {
 			p.wordCount = 0
-			for _, word := range p.PlainWords() {
+			for _, word := range p.plainWords {
 				runeCount := utf8.RuneCountInString(word)
 				if len(word) == runeCount {
 					p.wordCount++
@@ -888,7 +988,7 @@ func (p *Page) analyzePage() {
 				}
 			}
 		} else {
-			p.wordCount = helpers.TotalWords(p.Plain())
+			p.wordCount = helpers.TotalWords(p.plain)
 		}
 
 		// TODO(bep) is set in a test. Fix that.
@@ -1045,10 +1145,8 @@ func (p *Page) subResourceTargetPathFactory(base string) string {
 	return path.Join(p.relTargetPathBase, base)
 }
 
-func (p *Page) prepareForRender(cfg *BuildCfg) error {
+func (p *Page) setContentInit(cfg *BuildCfg) error {
-	s := p.s
+	if !p.shouldRenderTo(p.s.rc.Format) {
-
-	if !p.shouldRenderTo(s.rc.Format) {
 		// No need to prepare
 		return nil
 	}
@@ -1058,11 +1156,40 @@ func (p *Page) prepareForRender(cfg *BuildCfg) error {
 		shortcodeUpdate = p.shortcodeState.updateDelta()
 	}
 
-	if !shortcodeUpdate && !cfg.whatChanged.other {
+	resetFunc := func(page *Page) func() {
-		// No need to process it again.
+		return func() {
-		return nil
+			err := page.prepareForRender(cfg)
+			if err != nil {
+				p.s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", page.Path(), err)
+			}
+		}
 	}
 
+	if shortcodeUpdate || cfg.whatChanged.other {
+		p.resetContent(resetFunc)
+	}
+
+	// Handle bundled pages.
+	for _, r := range p.Resources.ByType(pageResourceType) {
+		shortcodeUpdate = false
+		bp := r.(*Page)
+
+		if bp.shortcodeState != nil {
+			shortcodeUpdate = bp.shortcodeState.updateDelta()
+		}
+
+		if shortcodeUpdate || cfg.whatChanged.other {
+			p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages)
+			bp.resetContent(resetFunc)
+		}
+	}
+
+	return nil
+}
+
+func (p *Page) prepareForRender(cfg *BuildCfg) error {
+	s := p.s
+
 	// If we got this far it means that this is either a new Page pointer
 	// or a template or similar has changed so wee need to do a rerendering
 	// of the shortcodes etc.
@@ -1080,14 +1207,10 @@ func (p *Page) prepareForRender(cfg *BuildCfg) error {
 		workContentCopy = p.workContent
 	}
 
-	if p.Markup == "markdown" {
-		tmpContent, tmpTableOfContents := helpers.ExtractTOC(workContentCopy)
-		p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-		workContentCopy = tmpContent
-	}
-
 	var err error
-	if workContentCopy, err = handleShortcodes(p, workContentCopy); err != nil {
+	// Note: The shortcodes in a page cannot access the page content it lives in,
+	// hence the withoutContent().
+	if workContentCopy, err = handleShortcodes(p.withoutContent(), workContentCopy); err != nil {
 		s.Log.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
 	}
 
@@ -1102,28 +1225,10 @@ func (p *Page) prepareForRender(cfg *BuildCfg) error {
 			workContentCopy = summaryContent.content
 		}
 
-		p.content = helpers.BytesToHTML(workContentCopy)
+		p.contentv = helpers.BytesToHTML(workContentCopy)
-
-		if summaryContent == nil {
-			if err := p.setAutoSummary(); err != nil {
-				s.Log.ERROR.Printf("Failed to set user auto summary for page %q: %s", p.pathOrTitle(), err)
-			}
-		}
 
 	} else {
-		p.content = helpers.BytesToHTML(workContentCopy)
+		p.contentv = helpers.BytesToHTML(workContentCopy)
-	}
-
-	//analyze for raw stats
-	p.analyzePage()
-
-	// Handle bundled pages.
-	for _, r := range p.Resources.ByType(pageResourceType) {
-		p.s.PathSpec.ProcessingStats.Incr(&p.s.PathSpec.ProcessingStats.Pages)
-		bp := r.(*Page)
-		if err := bp.prepareForRender(cfg); err != nil {
-			s.Log.ERROR.Printf("Failed to prepare bundled page %q for render: %s", bp.BaseFileName(), err)
-		}
 	}
 
 	return nil
@@ -1701,9 +1806,10 @@ func (p *Page) SaveSource() error {
 	return p.SaveSourceAs(p.FullFilePath())
 }
 
+// TODO(bep) lazy consolidate
 func (p *Page) processShortcodes() error {
 	p.shortcodeState = newShortcodeHandler(p)
-	tmpContent, err := p.shortcodeState.extractShortcodes(string(p.workContent), p)
+	tmpContent, err := p.shortcodeState.extractShortcodes(string(p.workContent), p.withoutContent())
 	if err != nil {
 		return err
 	}
@@ -1724,7 +1830,7 @@ func (p *Page) prepareLayouts() error {
 	if p.Kind == KindPage {
 		if !p.IsRenderable() {
 			self := "__" + p.UniqueID()
-			err := p.s.TemplateHandler().AddLateTemplate(self, string(p.content))
+			err := p.s.TemplateHandler().AddLateTemplate(self, string(p.content()))
 			if err != nil {
 				return err
 			}
@@ -1833,8 +1939,11 @@ func (p *Page) updatePageDates() {
 // copy creates a copy of this page with the lazy sync.Once vars reset
 // so they will be evaluated again, for word count calculations etc.
 func (p *Page) copy() *Page {
+	p.contentInitMu.Lock()
 	c := *p
+	p.contentInitMu.Unlock()
 	c.pageInit = &pageInit{}
+	c.pageContentInit = &pageContentInit{}
 	return &c
 }
 

hugolib/pageSort.go

@@ -237,7 +237,7 @@ func (p Pages) ByLength() Pages {
 	key := "pageSort.ByLength"
 
 	length := func(p1, p2 *Page) bool {
-		return len(p1.content) < len(p2.content)
+		return len(p1.content()) < len(p2.content())
 	}
 
 	pages, _ := spc.get(key, pageBy(length).Sort, p)

hugolib/pageSort_test.go

@@ -80,7 +80,7 @@ func TestSortByN(t *testing.T) {
 		{(Pages).ByPublishDate, func(p Pages) bool { return p[0].PublishDate == d4 }},
 		{(Pages).ByExpiryDate, func(p Pages) bool { return p[0].ExpiryDate == d4 }},
 		{(Pages).ByLastmod, func(p Pages) bool { return p[1].Lastmod == d3 }},
-		{(Pages).ByLength, func(p Pages) bool { return p[0].content == "b_content" }},
+		{(Pages).ByLength, func(p Pages) bool { return p[0].content() == "b_content" }},
 	} {
 		setSortVals([4]time.Time{d1, d2, d3, d4}, [4]string{"b", "ab", "cde", "fg"}, [4]int{0, 3, 2, 1}, p)
 
@@ -168,7 +168,7 @@ func setSortVals(dates [4]time.Time, titles [4]string, weights [4]int, pages Pag
 		pages[len(dates)-1-i].linkTitle = pages[i].title + "l"
 		pages[len(dates)-1-i].PublishDate = dates[i]
 		pages[len(dates)-1-i].ExpiryDate = dates[i]
-		pages[len(dates)-1-i].content = template.HTML(titles[i] + "_content")
+		pages[len(dates)-1-i].contentv = template.HTML(titles[i] + "_content")
 	}
 	lastLastMod := pages[2].Lastmod
 	pages[2].Lastmod = pages[1].Lastmod

hugolib/page_bundler_handlers.go

@@ -286,6 +286,10 @@ func (c *contentHandlers) handlePageContent() contentHandler {
 		p.workContent = p.replaceDivider(p.workContent)
 		p.workContent = p.renderContent(p.workContent)
 
+		tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.workContent)
+		p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
+		p.workContent = tmpContent
+
 		if !ctx.doNotAddToSiteCollections {
 			ctx.pages <- p
 		}

hugolib/page_bundler_test.go

@@ -87,7 +87,7 @@ func TestPageBundlerSiteRegular(t *testing.T) {
 				assert.Equal(singlePage, s.getPage("page", "a/1"))
 				assert.Equal(singlePage, s.getPage("page", "1"))
 
-				assert.Contains(singlePage.content, "TheContent")
+				assert.Contains(singlePage.content(), "TheContent")
 
 				if ugly {
 					assert.Equal("/a/1.html", singlePage.RelPermalink())
@@ -129,9 +129,12 @@ func TestPageBundlerSiteRegular(t *testing.T) {
 				firstPage := pageResources[0].(*Page)
 				secondPage := pageResources[1].(*Page)
 				assert.Equal(filepath.FromSlash("b/my-bundle/1.md"), firstPage.pathOrTitle(), secondPage.pathOrTitle())
-				assert.Contains(firstPage.content, "TheContent")
+				assert.Contains(firstPage.content(), "TheContent")
 				assert.Equal(6, len(leafBundle1.Resources))
 
+				// Verify shortcode in bundled page
+				assert.Contains(secondPage.content(), filepath.FromSlash("MyShort in b/my-bundle/2.md"))
+
 				// https://github.com/gohugoio/hugo/issues/4582
 				assert.Equal(leafBundle1, firstPage.Parent())
 				assert.Equal(leafBundle1, secondPage.Parent())
@@ -395,7 +398,7 @@ HEADLESS {{< myShort >}}
 	assert.Equal("Headless Bundle in Topless Bar", headless.Title())
 	assert.Equal("", headless.RelPermalink())
 	assert.Equal("", headless.Permalink())
-	assert.Contains(headless.content, "HEADLESS SHORTCODE")
+	assert.Contains(headless.content(), "HEADLESS SHORTCODE")
 
 	headlessResources := headless.Resources
 	assert.Equal(3, len(headlessResources))
@@ -404,7 +407,7 @@ HEADLESS {{< myShort >}}
 	assert.NotNil(pageResource)
 	assert.IsType(&Page{}, pageResource)
 	p := pageResource.(*Page)
-	assert.Contains(p.content, "SHORTCODE")
+	assert.Contains(p.content(), "SHORTCODE")
 	assert.Equal("p1.md", p.Name())
 
 	th := testHelper{s.Cfg, s.Fs, t}
@@ -439,6 +442,17 @@ date: 2017-10-09
 ---
 
 TheContent.
+`
+
+	pageContentShortcode := `---
+title: "Bundle Galore"
+slug: pageslug
+date: 2017-10-09
+---
+
+TheContent.
+
+{{< myShort >}}
 `
 
 	pageWithImageShortcodeAndResourceMetadataContent := `---
@@ -487,6 +501,7 @@ Thumb RelPermalink: {{ $thumb.RelPermalink }}
 `
 
 	myShort := `
+MyShort in {{ .Page.Path }}:
 {{ $sunset := .Page.Resources.GetByPrefix "my-sunset-2" }}
 {{ with $sunset }}
 Short Sunset RelPermalink: {{ .RelPermalink }}
@@ -520,7 +535,7 @@ Short Thumb Width: {{ $thumb.Width }}
 	// Bundle
 	writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "index.md"), pageWithImageShortcodeAndResourceMetadataContent)
 	writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "1.md"), pageContent)
-	writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "2.md"), pageContent)
+	writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "2.md"), pageContentShortcode)
 	writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "custom-mime.bep"), "bepsays")
 	writeSource(t, fs, filepath.Join(workDir, "base", "b", "my-bundle", "c", "logo.png"), "content")
 

hugolib/page_test.go

@@ -481,7 +481,7 @@ func checkPageTitle(t *testing.T, page *Page, title string) {
 
 func checkPageContent(t *testing.T, page *Page, content string, msg ...interface{}) {
 	a := normalizeContent(content)
-	b := normalizeContent(string(page.content))
+	b := normalizeContent(string(page.content()))
 	if a != b {
 		t.Fatalf("Page content is:\n%q\nExpected:\n%q (%q)", b, a, msg)
 	}
@@ -505,7 +505,7 @@ func checkPageTOC(t *testing.T, page *Page, toc string) {
 }
 
 func checkPageSummary(t *testing.T, page *Page, summary string, msg ...interface{}) {
-	a := normalizeContent(string(page.Summary))
+	a := normalizeContent(string(page.summary))
 	b := normalizeContent(summary)
 	if a != b {
 		t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg)
@@ -525,10 +525,10 @@ func checkPageDate(t *testing.T, page *Page, time time.Time) {
 }
 
 func checkTruncation(t *testing.T, page *Page, shouldBe bool, msg string) {
-	if page.Summary == "" {
+	if page.summary == "" {
 		t.Fatal("page has no summary, can not check truncation")
 	}
-	if page.Truncated != shouldBe {
+	if page.truncated != shouldBe {
 		if shouldBe {
 			t.Fatalf("page wasn't truncated: %s", msg)
 		} else {
@@ -616,7 +616,7 @@ func testAllMarkdownEnginesForPages(t *testing.T,
 		require.NoError(t, err)
 		require.NotNil(t, home)
 		require.Equal(t, homePath, home.Path())
-		require.Contains(t, home.content, "Home Page Content")
+		require.Contains(t, home.content(), "Home Page Content")
 
 	}
 
@@ -722,12 +722,12 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) {
 
 	p := s.RegularPages[0]
 
-	if p.Summary != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup>\n</p>") {
+	if p.summary != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup>\n</p>") {
-		t.Fatalf("Got summary:\n%q", p.Summary)
+		t.Fatalf("Got summary:\n%q", p.summary)
 	}
 
-	if p.content != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup>\n</p>\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>") {
+	if p.content() != template.HTML("<p>The <a href=\"http://gohugo.io/\">best static site generator</a>.<sup class=\"footnote-ref\" id=\"fnref:1\"><a href=\"#fn:1\">1</a></sup>\n</p>\n<div class=\"footnotes\">\n\n<hr />\n\n<ol>\n<li id=\"fn:1\">Many people say so.\n <a class=\"footnote-return\" href=\"#fnref:1\"><sup>[return]</sup></a></li>\n</ol>\n</div>") {
-		t.Fatalf("Got content:\n%q", p.content)
+		t.Fatalf("Got content:\n%q", p.content())
 	}
 }
 
@@ -876,8 +876,8 @@ func TestSummaryWithHTMLTagsOnNextLine(t *testing.T) {
 
 	assertFunc := func(t *testing.T, ext string, pages Pages) {
 		p := pages[0]
-		require.Contains(t, p.Summary, "Happy new year everyone!")
+		require.Contains(t, p.summary, "Happy new year everyone!")
-		require.NotContains(t, p.Summary, "User interface")
+		require.NotContains(t, p.summary, "User interface")
 	}
 
 	testAllMarkdownEnginesForPages(t, assertFunc, nil, `---
@@ -1037,9 +1037,9 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) {
 			t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount())
 		}
 
-		if p.Summary != simplePageWithMainEnglishWithCJKRunesSummary {
+		if p.summary != simplePageWithMainEnglishWithCJKRunesSummary {
 			t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain,
-				simplePageWithMainEnglishWithCJKRunesSummary, p.Summary)
+				simplePageWithMainEnglishWithCJKRunesSummary, p.summary)
 		}
 	}
 
@@ -1058,9 +1058,9 @@ func TestWordCountWithIsCJKLanguageFalse(t *testing.T) {
 			t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount())
 		}
 
-		if p.Summary != simplePageWithIsCJKLanguageFalseSummary {
+		if p.summary != simplePageWithIsCJKLanguageFalseSummary {
 			t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain,
-				simplePageWithIsCJKLanguageFalseSummary, p.Summary)
+				simplePageWithIsCJKLanguageFalseSummary, p.summary)
 		}
 	}
 
@@ -1511,7 +1511,8 @@ func TestPageSimpleMethods(t *testing.T) {
 	} {
 
 		p, _ := s.NewPage("Test")
-		p.content = "<h1>Do Be Do Be Do</h1>"
+		p.contentv = "<h1>Do Be Do Be Do</h1>"
+		p.initContent()
 		if !this.assertFunc(p) {
 			t.Errorf("[%d] Page method error", i)
 		}

hugolib/page_without_content.go

@@ -0,0 +1,67 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"html/template"
+)
+
+// PageWithoutContent is sent to the shortcodes. They cannot access the content
+// they're a part of. It would cause an infinite regress.
+//
+// Go doesn't support virtual methods, so this careful dance is currently (I think)
+// the best we can do.
+type PageWithoutContent struct {
+	*Page
+}
+
+// Content returns an empty string.
+func (p *PageWithoutContent) Content() (interface{}, error) {
+	return "", nil
+}
+
+// Truncated always returns false.
+func (p *PageWithoutContent) Truncated() bool {
+	return false
+}
+
+// Summary returns an empty string.
+func (p *PageWithoutContent) Summary() template.HTML {
+	return ""
+}
+
+// WordCount always returns 0.
+func (p *PageWithoutContent) WordCount() int {
+	return 0
+}
+
+// ReadingTime always returns 0.
+func (p *PageWithoutContent) ReadingTime() int {
+	return 0
+}
+
+// FuzzyWordCount always returns 0.
+func (p *PageWithoutContent) FuzzyWordCount() int {
+	return 0
+}
+
+// Plain returns an empty string.
+func (p *PageWithoutContent) Plain() string {
+	return ""
+}
+
+// PlainWords returns an empty string slice.
+func (p *PageWithoutContent) PlainWords() []string {
+	return []string{}
+}

hugolib/shortcode.go

@@ -37,7 +37,7 @@ import (
 type ShortcodeWithPage struct {
 	Params        interface{}
 	Inner         template.HTML
-	Page          *Page
+	Page          *PageWithoutContent
 	Parent        *ShortcodeWithPage
 	IsNamedParams bool
 	scratch       *Scratch
@@ -177,7 +177,7 @@ func newDefaultScKey(shortcodeplaceholder string) scKey {
 type shortcodeHandler struct {
 	init sync.Once
 
-	p *Page
+	p *PageWithoutContent
 
 	// This is all shortcode rendering funcs for all potential output formats.
 	contentShortcodes map[scKey]func() (string, error)
@@ -196,11 +196,26 @@ type shortcodeHandler struct {
 
 	// All the shortcode names in this set.
 	nameSet map[string]bool
+
+	placeholderID   int
+	placeholderFunc func() string
+}
+
+func (s *shortcodeHandler) nextPlaceholderID() int {
+	s.placeholderID++
+	return s.placeholderID
+}
+
+func (s *shortcodeHandler) createShortcodePlaceholder() string {
+	if s.placeholderFunc != nil {
+		return s.placeholderFunc()
+	}
+	return fmt.Sprintf("HAHA%s-%p-%d-HBHB", shortcodePlaceholderPrefix, s.p.Page, s.nextPlaceholderID())
 }
 
 func newShortcodeHandler(p *Page) *shortcodeHandler {
 	return &shortcodeHandler{
-		p:                  p,
+		p:                  p.withoutContent(),
 		contentShortcodes:  make(map[scKey]func() (string, error)),
 		shortcodes:         make(map[string]shortcode),
 		nameSet:            make(map[string]bool),
@@ -240,15 +255,11 @@ func clearIsInnerShortcodeCache() {
 	isInnerShortcodeCache.m = make(map[string]bool)
 }
 
-func createShortcodePlaceholder(id int) string {
-	return fmt.Sprintf("HAHA%s-%dHBHB", shortcodePlaceholderPrefix, id)
-}
-
 const innerNewlineRegexp = "\n"
 const innerCleanupRegexp = `\A<p>(.*)</p>\n\z`
 const innerCleanupExpand = "$1"
 
-func prepareShortcodeForPage(placeholder string, sc shortcode, parent *ShortcodeWithPage, p *Page) map[scKey]func() (string, error) {
+func prepareShortcodeForPage(placeholder string, sc shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) {
 
 	m := make(map[scKey]func() (string, error))
 	lang := p.Lang()
@@ -268,7 +279,7 @@ func renderShortcode(
 	tmplKey scKey,
 	sc shortcode,
 	parent *ShortcodeWithPage,
-	p *Page) string {
+	p *PageWithoutContent) string {
 
 	tmpl := getShortcodeTemplateForTemplateKey(tmplKey, sc.name, p.s.Tmpl)
 	if tmpl == nil {
@@ -347,7 +358,7 @@ func renderShortcode(
 // the content from the previous output format, if any.
 func (s *shortcodeHandler) updateDelta() bool {
 	s.init.Do(func() {
-		s.contentShortcodes = createShortcodeRenderers(s.shortcodes, s.p)
+		s.contentShortcodes = createShortcodeRenderers(s.shortcodes, s.p.withoutContent())
 	})
 
 	contentShortcodes := s.contentShortcodesForOutputFormat(s.p.s.rc.Format)
@@ -399,7 +410,7 @@ func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) map
 	return contentShortcodesForOuputFormat
 }
 
-func (s *shortcodeHandler) executeShortcodesForDelta(p *Page) error {
+func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) error {
 
 	for k, render := range s.contentShortcodesDelta {
 		renderedShortcode, err := render()
@@ -414,7 +425,7 @@ func (s *shortcodeHandler) executeShortcodesForDelta(p *Page) error {
 
 }
 
-func createShortcodeRenderers(shortcodes map[string]shortcode, p *Page) map[scKey]func() (string, error) {
+func createShortcodeRenderers(shortcodes map[string]shortcode, p *PageWithoutContent) map[scKey]func() (string, error) {
 
 	shortcodeRenderers := make(map[scKey]func() (string, error))
 
@@ -433,7 +444,7 @@ var errShortCodeIllegalState = errors.New("Illegal shortcode state")
 // pageTokens state:
 // - before: positioned just before the shortcode start
 // - after: shortcode(s) consumed (plural when they are nested)
-func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *Page) (shortcode, error) {
+func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (shortcode, error) {
 	sc := shortcode{}
 	var isInner = false
 
@@ -555,7 +566,7 @@ Loop:
 	return sc, nil
 }
 
-func (s *shortcodeHandler) extractShortcodes(stringToParse string, p *Page) (string, error) {
+func (s *shortcodeHandler) extractShortcodes(stringToParse string, p *PageWithoutContent) (string, error) {
 
 	startIdx := strings.Index(stringToParse, "{{")
 
@@ -569,8 +580,6 @@ func (s *shortcodeHandler) extractShortcodes(stringToParse string, p *Page) (str
 	// it seems that the time isn't really spent in the byte copy operations, and the impl. gets a lot cleaner
 	pt := &pageTokens{lexer: newShortcodeLexer("parse-page", stringToParse, pos(startIdx))}
 
-	id := 1 // incremented id, will be appended onto temp. shortcode placeholders
-
 	result := bp.GetBuffer()
 	defer bp.PutBuffer(result)
 	//var result bytes.Buffer
@@ -605,10 +614,9 @@ Loop:
 				currShortcode.params = make([]string, 0)
 			}
 
-			placeHolder := createShortcodePlaceholder(id)
+			placeHolder := s.createShortcodePlaceholder()
 			result.WriteString(placeHolder)
 			s.shortcodes[placeHolder] = currShortcode
-			id++
 		case tEOF:
 			break Loop
 		case tError:

hugolib/shortcode_test.go

@@ -87,7 +87,7 @@ title: "Title"
 
 	require.Len(t, h.Sites[0].RegularPages, 1)
 
-	output := strings.TrimSpace(string(h.Sites[0].RegularPages[0].content))
+	output := strings.TrimSpace(string(h.Sites[0].RegularPages[0].content()))
 	output = strings.TrimPrefix(output, "<p>")
 	output = strings.TrimSuffix(output, "</p>")
 
@@ -390,8 +390,16 @@ func TestExtractShortcodes(t *testing.T) {
 			return nil
 		})
 
+		counter := 0
+
 		s := newShortcodeHandler(p)
-		content, err := s.extractShortcodes(this.input, p)
+
+		s.placeholderFunc = func() string {
+			counter++
+			return fmt.Sprintf("HAHA%s-%dHBHB", shortcodePlaceholderPrefix, counter)
+		}
+
+		content, err := s.extractShortcodes(this.input, p.withoutContent())
 
 		if b, ok := this.expect.(bool); ok && !b {
 			if err == nil {
@@ -446,7 +454,7 @@ func TestExtractShortcodes(t *testing.T) {
 		if this.expectShortCodes != "" {
 			shortCodesAsStr := fmt.Sprintf("map%q", collectAndSortShortcodes(shortCodes))
 			if !strings.Contains(shortCodesAsStr, this.expectShortCodes) {
-				t.Fatalf("[%d] %s: Shortcodes not as expected, got %s but expected %s", i, this.name, shortCodesAsStr, this.expectShortCodes)
+				t.Fatalf("[%d] %s: Shortcodes not as expected, got\n%s but expected\n%s", i, this.name, shortCodesAsStr, this.expectShortCodes)
 			}
 		}
 	}

hugolib/site.go

@@ -1865,14 +1865,15 @@ func getGoMaxProcs() int {
 
 func (s *Site) newNodePage(typ string, sections ...string) *Page {
 	p := &Page{
-		language: s.Language,
+		language:        s.Language,
-		pageInit: &pageInit{},
+		pageInit:        &pageInit{},
-		Kind:     typ,
+		pageContentInit: &pageContentInit{},
-		Source:   Source{File: &source.FileInfo{}},
+		Kind:            typ,
-		Data:     make(map[string]interface{}),
+		Source:          Source{File: &source.FileInfo{}},
-		Site:     &s.Info,
+		Data:            make(map[string]interface{}),
-		sections: sections,
+		Site:            &s.Info,
-		s:        s}
+		sections:        sections,
+		s:               s}
 
 	p.outputFormats = p.s.outputFormats[p.Kind]
 

hugolib/site_render.go

@@ -44,7 +44,6 @@ func (s *Site) renderPages(cfg *BuildCfg) error {
 	if len(s.headlessPages) > 0 {
 		wg.Add(1)
 		go headlessPagesPublisher(s, wg)
-
 	}
 
 	for _, page := range s.Pages {
@@ -70,6 +69,10 @@ func headlessPagesPublisher(s *Site, wg *sync.WaitGroup) {
 	defer wg.Done()
 	for _, page := range s.headlessPages {
 		outFormat := page.outputFormats[0] // There is only one
+		if outFormat != s.rc.Format {
+			// Avoid double work.
+			continue
+		}
 		pageOutput, err := newPageOutput(page, false, outFormat)
 		if err == nil {
 			page.mainPageOutput = pageOutput