From e590cc26eb1363a4b84603f051b20bd43fd1f7bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?=
 <bjorn.erik.pedersen@gmail.com>
Date: Thu, 19 Apr 2018 12:04:34 +0200
Subject: [PATCH] Improve .Content vs shortcodes

For the content from other pages in shortcodes there are some chicken and
egg dependencies that is hard to get around. But we can improve on this  by preparing the pages in a certain order:

 1. The headless bundles goes first. These are page typically page and image collections..
 2. Leaf bundles
 3. Regular single pages
 4. Branch bundles

Fixes #4632
---
 hugolib/hugo_sites.go            | 81 +++++++++++++++++++++++++-------
 hugolib/hugo_sites_build.go      |  1 +
 hugolib/page.go                  |  2 +
 hugolib/page_bundler_handlers.go |  2 +
 hugolib/site.go                  | 17 +++----
 5 files changed, 77 insertions(+), 26 deletions(-)

diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go
index 7eec98329..6a71fd97b 100644
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -559,37 +559,82 @@ func (h *HugoSites) setupTranslations() {
 	}
 }
 
-func (s *Site) preparePagesForRender(cfg *BuildCfg) {
-
-	pageChan := make(chan *Page)
-	wg := &sync.WaitGroup{}
+type pagesRenderPreparer struct {
+	numWorkers int
+	s          *Site
+	cfg        *BuildCfg
+	wg         *sync.WaitGroup
+	pages      chan *Page
+}
 
+func newStartedRenderPreparator(s *Site, cfg *BuildCfg) *pagesRenderPreparer {
 	numWorkers := getGoMaxProcs() * 4
+	pp := &pagesRenderPreparer{
+		s:          s,
+		cfg:        cfg,
+		numWorkers: numWorkers,
+		wg:         &sync.WaitGroup{},
+		pages:      make(chan *Page),
+	}
 
-	for i := 0; i < numWorkers; i++ {
-		wg.Add(1)
-		go func(pages <-chan *Page, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for p := range pages {
-				if err := p.prepareForRender(cfg); err != nil {
-					s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
+	pp.start()
+	return pp
+}
+
+func (pp *pagesRenderPreparer) start() {
+	for i := 0; i < pp.numWorkers; i++ {
+		pp.wg.Add(1)
+		go func() {
+			defer pp.wg.Done()
+			for p := range pp.pages {
+				if err := p.prepareForRender(pp.cfg); err != nil {
+					pp.s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
 
 				}
 			}
-		}(pageChan, wg)
+		}()
 	}
+}
 
-	for _, p := range s.Pages {
-		pageChan <- p
-	}
+func (pp *pagesRenderPreparer) add(p *Page) {
+	pp.pages <- p
+}
 
+func (pp *pagesRenderPreparer) done() {
+	close(pp.pages)
+	pp.wg.Wait()
+}
+
+func (s *Site) preparePagesForRender(cfg *BuildCfg) {
+
+	// For the content from other pages in shortcodes there are some chicken and
+	// egg dependencies that is hard to get around. But we can improve on this
+	// by preparing the pages in a certain order.
+	// So the headless pages goes first. These are typically collection of
+	// pages and images etc. used by others.
+	batch := newStartedRenderPreparator(s, cfg)
 	for _, p := range s.headlessPages {
-		pageChan <- p
+		batch.add(p)
 	}
 
-	close(pageChan)
+	batch.done()
 
-	wg.Wait()
+	// Then the rest in the following order:
+	order := []bundleDirType{bundleLeaf, bundleNot, bundleBranch}
+
+	for _, tp := range order {
+		batch = newStartedRenderPreparator(s, cfg)
+		for _, p := range s.Pages {
+			// sanity check
+			if p.bundleType < 0 || p.bundleType > bundleBranch {
+				panic("unknown bundle type")
+			}
+			if p.bundleType == tp {
+				batch.add(p)
+			}
+		}
+		batch.done()
+	}
 
 }
 
diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go
index 1c4ee7b63..dcff4b3b2 100644
--- a/hugolib/hugo_sites_build.go
+++ b/hugolib/hugo_sites_build.go
@@ -224,6 +224,7 @@ func (h *HugoSites) render(config *BuildCfg) error {
 		s.initRenderFormats()
 		for i, rf := range s.renderFormats {
 			s.rc = &siteRenderingContext{Format: rf}
+
 			s.preparePagesForRender(config)
 
 			if !config.SkipRender {
diff --git a/hugolib/page.go b/hugolib/page.go
index ebd7a3a2a..d0951bff7 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -243,6 +243,8 @@ type Page struct {
 	// 3. But you can get it via .Site.GetPage
 	headless bool
 
+	bundleType bundleDirType
+
 	layoutDescriptor output.LayoutDescriptor
 
 	scratch *Scratch
diff --git a/hugolib/page_bundler_handlers.go b/hugolib/page_bundler_handlers.go
index c22b719d1..b38a3c35a 100644
--- a/hugolib/page_bundler_handlers.go
+++ b/hugolib/page_bundler_handlers.go
@@ -218,6 +218,8 @@ func (c *contentHandlers) parsePage(h contentHandler) contentHandler {
 		ctx.currentPage = p
 
 		if ctx.bundle != nil {
+			p.bundleType = ctx.bundle.tp
+
 			// Add the bundled files
 			for _, fi := range ctx.bundle.resources {
 				childCtx := ctx.childCtx(fi)
diff --git a/hugolib/site.go b/hugolib/site.go
index 262bf2d4d..820a2c829 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -1865,14 +1865,15 @@ func getGoMaxProcs() int {
 
 func (s *Site) newNodePage(typ string, sections ...string) *Page {
 	p := &Page{
-		language: s.Language,
-		pageInit: &pageInit{},
-		Kind:     typ,
-		Source:   Source{File: &source.FileInfo{}},
-		Data:     make(map[string]interface{}),
-		Site:     &s.Info,
-		sections: sections,
-		s:        s}
+		bundleType: bundleBranch,
+		language:   s.Language,
+		pageInit:   &pageInit{},
+		Kind:       typ,
+		Source:     Source{File: &source.FileInfo{}},
+		Data:       make(map[string]interface{}),
+		Site:       &s.Info,
+		sections:   sections,
+		s:          s}
 
 	p.outputFormats = p.s.outputFormats[p.Kind]