Improve .Content vs shortcodes

For the content from other pages in shortcodes there are some chicken and
egg dependencies that is hard to get around. But we can improve on this  by preparing the pages in a certain order:

 1. The headless bundles goes first. These are page typically page and image collections..
 2. Leaf bundles
 3. Regular single pages
 4. Branch bundles

Fixes #4632

hugolib/hugo_sites.go

@@ -559,37 +559,82 @@ func (h *HugoSites) setupTranslations() {
 	}
 }
 
-func (s *Site) preparePagesForRender(cfg *BuildCfg) {
-
-	pageChan := make(chan *Page)
-	wg := &sync.WaitGroup{}
+type pagesRenderPreparer struct {
+	numWorkers int
+	s          *Site
+	cfg        *BuildCfg
+	wg         *sync.WaitGroup
+	pages      chan *Page
+}
 
+func newStartedRenderPreparator(s *Site, cfg *BuildCfg) *pagesRenderPreparer {
 	numWorkers := getGoMaxProcs() * 4
+	pp := &pagesRenderPreparer{
+		s:          s,
+		cfg:        cfg,
+		numWorkers: numWorkers,
+		wg:         &sync.WaitGroup{},
+		pages:      make(chan *Page),
+	}
 
-	for i := 0; i < numWorkers; i++ {
-		wg.Add(1)
-		go func(pages <-chan *Page, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for p := range pages {
-				if err := p.prepareForRender(cfg); err != nil {
-					s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
+	pp.start()
+	return pp
+}
+
+func (pp *pagesRenderPreparer) start() {
+	for i := 0; i < pp.numWorkers; i++ {
+		pp.wg.Add(1)
+		go func() {
+			defer pp.wg.Done()
+			for p := range pp.pages {
+				if err := p.prepareForRender(pp.cfg); err != nil {
+					pp.s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
 
 				}
 			}
-		}(pageChan, wg)
+		}()
 	}
+}
 
-	for _, p := range s.Pages {
-		pageChan <- p
-	}
+func (pp *pagesRenderPreparer) add(p *Page) {
+	pp.pages <- p
+}
 
+func (pp *pagesRenderPreparer) done() {
+	close(pp.pages)
+	pp.wg.Wait()
+}
+
+func (s *Site) preparePagesForRender(cfg *BuildCfg) {
+
+	// For the content from other pages in shortcodes there are some chicken and
+	// egg dependencies that is hard to get around. But we can improve on this
+	// by preparing the pages in a certain order.
+	// So the headless pages goes first. These are typically collection of
+	// pages and images etc. used by others.
+	batch := newStartedRenderPreparator(s, cfg)
 	for _, p := range s.headlessPages {
-		pageChan <- p
+		batch.add(p)
 	}
 
-	close(pageChan)
+	batch.done()
 
-	wg.Wait()
+	// Then the rest in the following order:
+	order := []bundleDirType{bundleLeaf, bundleNot, bundleBranch}
+
+	for _, tp := range order {
+		batch = newStartedRenderPreparator(s, cfg)
+		for _, p := range s.Pages {
+			// sanity check
+			if p.bundleType < 0 || p.bundleType > bundleBranch {
+				panic("unknown bundle type")
+			}
+			if p.bundleType == tp {
+				batch.add(p)
+			}
+		}
+		batch.done()
+	}
 
 }
 

hugolib/hugo_sites_build.go

@@ -224,6 +224,7 @@ func (h *HugoSites) render(config *BuildCfg) error {
 		s.initRenderFormats()
 		for i, rf := range s.renderFormats {
 			s.rc = &siteRenderingContext{Format: rf}
+
 			s.preparePagesForRender(config)
 
 			if !config.SkipRender {

hugolib/page.go

@@ -243,6 +243,8 @@ type Page struct {
 	// 3. But you can get it via .Site.GetPage
 	headless bool
 
+	bundleType bundleDirType
+
 	layoutDescriptor output.LayoutDescriptor
 
 	scratch *Scratch

hugolib/page_bundler_handlers.go

@@ -218,6 +218,8 @@ func (c *contentHandlers) parsePage(h contentHandler) contentHandler {
 		ctx.currentPage = p
 
 		if ctx.bundle != nil {
+			p.bundleType = ctx.bundle.tp
+
 			// Add the bundled files
 			for _, fi := range ctx.bundle.resources {
 				childCtx := ctx.childCtx(fi)

hugolib/site.go

@@ -1865,14 +1865,15 @@ func getGoMaxProcs() int {
 
 func (s *Site) newNodePage(typ string, sections ...string) *Page {
 	p := &Page{
-		language: s.Language,
-		pageInit: &pageInit{},
-		Kind:     typ,
-		Source:   Source{File: &source.FileInfo{}},
-		Data:     make(map[string]interface{}),
-		Site:     &s.Info,
-		sections: sections,
-		s:        s}
+		bundleType: bundleBranch,
+		language:   s.Language,
+		pageInit:   &pageInit{},
+		Kind:       typ,
+		Source:     Source{File: &source.FileInfo{}},
+		Data:       make(map[string]interface{}),
+		Site:       &s.Info,
+		sections:   sections,
+		s:          s}
 
 	p.outputFormats = p.s.outputFormats[p.Kind]