node to page: Refactor the build process

To make it easier to follow and understand.

Both building and rebuilding now follow a four step flow:

1. Init
2. Process
3. Assemble
4. Render

And now there are only one Build method, used for both builds and rebuilds.

Updates #2297

commands/hugo.go

@@ -670,7 +670,7 @@ func rebuildSites(events []fsnotify.Event) error {
 	if err := initSites(); err != nil {
 		return err
 	}
-	return Hugo.Rebuild(hugolib.BuildCfg{PrintStats: !quiet, Watching: true}, events...)
+	return Hugo.Build(hugolib.BuildCfg{PrintStats: !quiet, Watching: true}, events...)
 }
 
 // NewWatcher creates a new watcher to watch filesystem events.

hugolib/hugo_sites.go

@@ -14,21 +14,18 @@
 package hugolib
 
 import (
-	"errors"
 	"fmt"
 	"html/template"
 	"os"
 	"path"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/spf13/hugo/helpers"
 
 	"github.com/spf13/viper"
 
 	"github.com/bep/inflect"
-	"github.com/fsnotify/fsnotify"
 	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/tpl"
 	jww "github.com/spf13/jwalterweatherman"
@@ -109,7 +106,7 @@ func (h *HugoSites) reset() {
 	tpl.ResetCaches()
 }
 
-func (h *HugoSites) reCreateFromConfig() error {
+func (h *HugoSites) createSitesFromConfig() error {
 
 	sites, err := createSitesFromConfig()
 
@@ -158,189 +155,8 @@ type BuildCfg struct {
 	// Use this to add templates to use for rendering.
 	// Useful for testing.
 	withTemplate func(templ tpl.Template) error
-}
+	// Use this to indicate what changed (for rebuilds).
-
+	whatChanged *whatChanged
-// Build builds all sites.
-func (h *HugoSites) Build(config BuildCfg) error {
-
-	t0 := time.Now()
-
-	// TODO(bep) np init page collections
-	for _, s := range h.Sites {
-		if s.PageCollections == nil {
-			s.PageCollections = newPageCollections()
-		}
-	}
-
-	if config.ResetState {
-		h.reset()
-	}
-
-	if config.CreateSitesFromConfig {
-		if err := h.reCreateFromConfig(); err != nil {
-			return err
-		}
-	}
-
-	h.runMode.Watching = config.Watching
-
-	// We should probably refactor the Site and pull up most of the logic from there to here,
-	// but that seems like a daunting task.
-	// So for now, if there are more than one site (language),
-	// we pre-process the first one, then configure all the sites based on that.
-	firstSite := h.Sites[0]
-
-	if err := firstSite.preProcess(config); err != nil {
-		return err
-	}
-
-	h.setupTranslations()
-
-	if len(h.Sites) > 1 {
-		// Initialize the rest
-		for _, site := range h.Sites[1:] {
-			site.initializeSiteInfo()
-		}
-	}
-
-	// TODO(bep) make a more logical grouping of these.
-	h.assembleGitInfo()
-
-	for _, s := range h.Sites {
-		if err := s.postProcess(); err != nil {
-			return err
-		}
-	}
-
-	// TODO(bep) np createMissingNodes needs taxonomies and sections
-	if err := h.createMissingNodes(); err != nil {
-		return err
-	}
-
-	for _, s := range h.Sites {
-		// TODO(bep) np Needed by all who use .Pages, .AllPages, .indexPages
-		s.refreshPageCaches()
-		s.setupPrevNext()
-	}
-
-	if err := h.assignMissingTranslations(); err != nil {
-		return err
-	}
-
-	if err := h.preRender(config, whatChanged{source: true, other: true}); err != nil {
-		return err
-	}
-
-	if !config.SkipRender {
-		for _, s := range h.Sites {
-
-			if err := s.render(); err != nil {
-				return err
-			}
-
-			if config.PrintStats {
-				s.Stats()
-			}
-		}
-
-		if err := h.render(); err != nil {
-			return err
-		}
-	}
-
-	if config.PrintStats {
-		jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds()))
-	}
-
-	return nil
-
-}
-
-// Rebuild rebuilds all sites.
-func (h *HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
-	t0 := time.Now()
-
-	if config.CreateSitesFromConfig {
-		return errors.New("Rebuild does not support 'CreateSitesFromConfig'. Use Build.")
-	}
-
-	if config.ResetState {
-		return errors.New("Rebuild does not support 'ResetState'. Use Build.")
-	}
-
-	if !config.Watching {
-		return errors.New("Rebuild called when not in watch mode")
-	}
-
-	h.runMode.Watching = config.Watching
-
-	firstSite := h.Sites[0]
-
-	for _, s := range h.Sites {
-		s.resetBuildState()
-	}
-
-	helpers.InitLoggers()
-
-	changed, err := firstSite.reBuild(events)
-
-	if err != nil {
-		return err
-	}
-
-	// Assign pages to sites per translation.
-	h.setupTranslations()
-
-	if changed.source {
-		h.assembleGitInfo()
-		for _, s := range h.Sites {
-			if err := s.postProcess(); err != nil {
-				return err
-			}
-		}
-
-	}
-
-	// TODO(bep) np consolidate the build lifecycle methods
-	// See also the regular Build() method, and check vs. the changed.source
-	if err := h.createMissingNodes(); err != nil {
-		return err
-	}
-
-	for _, s := range h.Sites {
-		s.refreshPageCaches()
-		s.setupPrevNext()
-	}
-
-	if err := h.assignMissingTranslations(); err != nil {
-		return err
-	}
-
-	if err := h.preRender(config, changed); err != nil {
-		return err
-	}
-
-	if !config.SkipRender {
-		for _, s := range h.Sites {
-			if err := s.render(); err != nil {
-				return err
-			}
-			if config.PrintStats {
-				s.Stats()
-			}
-		}
-
-		if err := h.render(); err != nil {
-			return err
-		}
-	}
-
-	if config.PrintStats {
-		jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds()))
-	}
-
-	return nil
-
 }
 
 // Analyze prints a build report to Stdout.
@@ -353,8 +169,7 @@ func (h *HugoSites) Analyze() error {
 	return s.ShowPlan(os.Stdout)
 }
 
-// Render the cross-site artifacts.
+func (h *HugoSites) renderCrossSitesArtifacts() error {
-func (h *HugoSites) render() error {
 
 	if !h.multilingual.enabled() {
 		return nil
@@ -494,6 +309,7 @@ func (h *HugoSites) createMissingNodes() error {
 	return nil
 }
 
+// TODO(bep) np move
 // Move the new* methods after cleanup in site.go
 func (s *Site) newNodePage(typ NodeType) *Page {
 	return &Page{

hugolib/hugo_sites_build.go

@@ -0,0 +1,197 @@
+// Copyright 2016-present The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"time"
+
+	"errors"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/spf13/hugo/helpers"
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+// Build builds all sites. If filesystem events are provided,
+// this is considered to be a potential partial rebuild.
+func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
+	t0 := time.Now()
+
+	// Need a pointer as this may be modified.
+	conf := &config
+
+	if conf.whatChanged == nil {
+		// Assume everything has changed
+		conf.whatChanged = &whatChanged{source: true, other: true}
+	}
+
+	if len(events) > 0 {
+		// Rebuild
+		if err := h.initRebuild(conf); err != nil {
+			return err
+		}
+	} else {
+		if err := h.init(conf); err != nil {
+			return err
+		}
+	}
+
+	if err := h.process(conf, events...); err != nil {
+		return err
+	}
+
+	if err := h.assemble(conf); err != nil {
+		return err
+	}
+
+	if err := h.render(conf); err != nil {
+		return err
+	}
+
+	if config.PrintStats {
+		jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds()))
+	}
+
+	return nil
+
+}
+
+// Build lifecycle methods below.
+// The order listed matches the order of execution.
+
+func (h *HugoSites) init(config *BuildCfg) error {
+
+	for _, s := range h.Sites {
+		if s.PageCollections == nil {
+			s.PageCollections = newPageCollections()
+		}
+	}
+
+	if config.ResetState {
+		h.reset()
+	}
+
+	if config.CreateSitesFromConfig {
+		if err := h.createSitesFromConfig(); err != nil {
+			return err
+		}
+	}
+
+	h.runMode.Watching = config.Watching
+
+	return nil
+}
+
+func (h *HugoSites) initRebuild(config *BuildCfg) error {
+	if config.CreateSitesFromConfig {
+		return errors.New("Rebuild does not support 'CreateSitesFromConfig'.")
+	}
+
+	if config.ResetState {
+		return errors.New("Rebuild does not support 'ResetState'.")
+	}
+
+	if !config.Watching {
+		return errors.New("Rebuild called when not in watch mode")
+	}
+
+	h.runMode.Watching = config.Watching
+
+	for _, s := range h.Sites {
+		s.resetBuildState()
+	}
+
+	helpers.InitLoggers()
+
+	return nil
+}
+
+func (h *HugoSites) process(config *BuildCfg, events ...fsnotify.Event) error {
+	// We should probably refactor the Site and pull up most of the logic from there to here,
+	// but that seems like a daunting task.
+	// So for now, if there are more than one site (language),
+	// we pre-process the first one, then configure all the sites based on that.
+	firstSite := h.Sites[0]
+
+	if len(events) > 0 {
+		// This is a rebuild
+		changed, err := firstSite.reProcess(events)
+		config.whatChanged = &changed
+		return err
+	}
+
+	return firstSite.process(*config)
+
+}
+
+func (h *HugoSites) assemble(config *BuildCfg) error {
+	// TODO(bep) np we could probably wait and do this in one go later
+	h.setupTranslations()
+
+	if len(h.Sites) > 1 {
+		// The first is initialized during process; initialize the rest
+		for _, site := range h.Sites[1:] {
+			site.initializeSiteInfo()
+		}
+	}
+
+	if config.whatChanged.source {
+		h.assembleGitInfo()
+
+		for _, s := range h.Sites {
+			if err := s.buildSiteMeta(); err != nil {
+				return err
+			}
+		}
+	}
+
+	if err := h.createMissingNodes(); err != nil {
+		return err
+	}
+
+	for _, s := range h.Sites {
+		s.refreshPageCaches()
+		s.setupPrevNext()
+	}
+
+	if err := h.assignMissingTranslations(); err != nil {
+		return err
+	}
+
+	if err := h.preRender(*config, whatChanged{source: true, other: true}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *HugoSites) render(config *BuildCfg) error {
+	if !config.SkipRender {
+		for _, s := range h.Sites {
+			if err := s.render(); err != nil {
+				return err
+			}
+
+			if config.PrintStats {
+				s.Stats()
+			}
+		}
+
+		if err := h.renderCrossSitesArtifacts(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

hugolib/hugo_sites_test.go

@@ -543,7 +543,8 @@ func TestMultiSitesRebuild(t *testing.T) {
 		if this.preFunc != nil {
 			this.preFunc(t)
 		}
-		err = sites.Rebuild(cfg, this.events...)
+
+		err = sites.Build(cfg, this.events...)
 
 		if err != nil {
 			t.Fatalf("[%d] Failed to rebuild sites: %s", i, err)

hugolib/site.go

@@ -482,7 +482,7 @@ type whatChanged struct {
 
 // reBuild partially rebuilds a site given the filesystem events.
 // It returns whetever the content source was changed.
-func (s *Site) reBuild(events []fsnotify.Event) (whatChanged, error) {
+func (s *Site) reProcess(events []fsnotify.Event) (whatChanged, error) {
 
 	jww.DEBUG.Printf("Rebuild for events %q", events)
 
@@ -763,7 +763,7 @@ func (s *Site) readDataFromSourceFS() error {
 	return err
 }
 
-func (s *Site) preProcess(config BuildCfg) (err error) {
+func (s *Site) process(config BuildCfg) (err error) {
 	s.timerStep("Go initialization")
 	if err = s.initialize(); err != nil {
 		return
@@ -785,16 +785,6 @@ func (s *Site) preProcess(config BuildCfg) (err error) {
 
 }
 
-func (s *Site) postProcess() (err error) {
-
-	if err = s.buildSiteMeta(); err != nil {
-		return
-	}
-
-	s.timerStep("build taxonomies")
-	return
-}
-
 func (s *Site) setupPrevNext() {
 	for i, page := range s.Pages {
 		if i < len(s.Pages)-1 {
@@ -1333,6 +1323,7 @@ func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
 }
 
 func (s *Site) buildSiteMeta() (err error) {
+	defer s.timerStep("build Site meta")
 
 	s.assembleMenus()