node to page: Handle aliases, 404, robots.txt, sitemap

Updates #2297

hugolib/hugo_sites.go

@@ -37,11 +37,7 @@ import (
 // Temporary feature flag to ease the refactoring of node vs page, see
 // https://github.com/spf13/hugo/issues/2297
 // TODO(bep) eventually remove
-var nodePageFeatureFlag bool
+var nodePageFeatureFlag bool = true
-
-func toggleNodePageFeatureFlag() {
-	nodePageFeatureFlag = !nodePageFeatureFlag
-}
 
 // HugoSites represents the sites to build. Each site represents a language.
 type HugoSites struct {
@@ -516,8 +512,10 @@ func (h *HugoSites) createMissingNodes() error {
 
 // Move the new* methods after cleanup in site.go
 func (s *Site) newNodePage(typ NodeType) *Page {
-
+	return &Page{
-	return &Page{Node: Node{
+		Node: Node{
+			Date:     s.Info.LastChange,
+			Lastmod:  s.Info.LastChange,
 			NodeType: typ,
 			Data:     make(map[string]interface{}),
 			Site:     &s.Info,

hugolib/menu.go

@@ -21,6 +21,8 @@ import (
 	"github.com/spf13/cast"
 )
 
+// TODO(bep) np menu entries in section content etc.?
+
 // MenuEntry represents a menu item defined in either Page front matter
 // or in the site config.
 type MenuEntry struct {

hugolib/menu_test.go

@@ -477,10 +477,10 @@ func TestTaxonomyNodeMenu(t *testing.T) {
 			&MenuEntry{Name: "Somewhere else", URL: "/somewhereelse"}, false, false},
 	} {
 
-		n, _ := s.newTaxonomyNode(true, this.taxInfo, i)
+		p := s.newTaxonomyPage(this.taxInfo.plural, this.taxInfo.key)
 
-		isMenuCurrent := n.IsMenuCurrent(this.menu, this.menuItem)
+		isMenuCurrent := p.IsMenuCurrent(this.menu, this.menuItem)
-		hasMenuCurrent := n.HasMenuCurrent(this.menu, this.menuItem)
+		hasMenuCurrent := p.HasMenuCurrent(this.menu, this.menuItem)
 
 		if isMenuCurrent != this.isMenuCurrent {
 			t.Errorf("[%d] Wrong result from IsMenuCurrent: %v", i, isMenuCurrent)

hugolib/node.go

@@ -41,6 +41,12 @@ const (
 	NodeSection
 	NodeTaxonomy
 	NodeTaxonomyTerms
+
+	// The following are (currently) temporary nodes,
+	// i.e. nodes we create just to render in isolation.
+	NodeSitemap
+	NodeRobotsTXT
+	Node404
 )
 
 func (p NodeType) String() string {
@@ -55,6 +61,12 @@ func (p NodeType) String() string {
 		return "taxonomy list"
 	case NodeTaxonomyTerms:
 		return "taxonomy terms"
+	case NodeSitemap:
+		return "sitemap"
+	case NodeRobotsTXT:
+		return "robots.txt"
+	case Node404:
+		return "404 Not Found"
 	case NodeUnknown:
 		return "unknown"
 	default:

hugolib/node_as_page_test.go

@@ -34,9 +34,6 @@ func TestNodesAsPage(t *testing.T) {
 	//jww.SetStdoutThreshold(jww.LevelDebug)
 	jww.SetStdoutThreshold(jww.LevelFatal)
 
-	nodePageFeatureFlag = true
-	defer toggleNodePageFeatureFlag()
-
 	/* Will have to decide what to name the node content files, but:
 
 		Home page should have:
@@ -152,9 +149,6 @@ func TestNodesWithNoContentFile(t *testing.T) {
 	//jww.SetStdoutThreshold(jww.LevelDebug)
 	jww.SetStdoutThreshold(jww.LevelFatal)
 
-	nodePageFeatureFlag = true
-	defer toggleNodePageFeatureFlag()
-
 	testCommonResetState()
 
 	writeLayoutsForNodeAsPageTests(t)
@@ -220,9 +214,6 @@ Content Page %02d
 
 func TestNodesAsPageMultilingual(t *testing.T) {
 
-	nodePageFeatureFlag = true
-	defer toggleNodePageFeatureFlag()
-
 	testCommonResetState()
 
 	writeLayoutsForNodeAsPageTests(t)

hugolib/page.go

@@ -1202,13 +1202,13 @@ func (p *Page) TargetPath() (outfile string) {
 	// TODO(bep) np
 	switch p.NodeType {
 	case NodeHome:
-		return p.addLangFilepathPrefix("index.html")
+		return p.addLangFilepathPrefix("/")
 	case NodeSection:
-		return p.addLangFilepathPrefix(filepath.Join(p.sections[0], "index.html"))
+		return p.addLangFilepathPrefix(p.sections[0])
 	case NodeTaxonomy:
-		return p.addLangFilepathPrefix(filepath.Join(append(p.sections, "index.html")...))
+		return p.addLangFilepathPrefix(filepath.Join(p.sections...))
 	case NodeTaxonomyTerms:
-		return p.addLangFilepathPrefix(filepath.Join(append(p.sections, "index.html")...))
+		return p.addLangFilepathPrefix(filepath.Join(p.sections...))
 	}
 
 	// Always use URL if it's specified

hugolib/site.go

@@ -838,36 +838,21 @@ func (s *Site) render() (err error) {
 		return
 	}
 
-	if err = s.renderAliases(); err != nil {
-		return
-	}
-	s.timerStep("render and write aliases")
-	if err = s.renderTaxonomiesLists(false); err != nil {
-		return
-	}
-	s.timerStep("render and write taxonomies")
-	if err = s.renderListsOfTaxonomyTerms(false); err != nil {
-		return
-	}
-	s.timerStep("render & write taxonomy lists")
-	if err = s.renderSectionLists(false); err != nil {
-		return
-	}
-	s.timerStep("render and write lists")
-
 	if err = s.preparePages(); err != nil {
 		return
 	}
+	s.timerStep("prepare pages")
 
 	if err = s.renderPages(); err != nil {
 		return
 	}
-
 	s.timerStep("render and write pages")
-	if err = s.renderHomePage(false); err != nil {
+
+	if err = s.renderAliases(); err != nil {
 		return
 	}
-	s.timerStep("render and write homepage")
+	s.timerStep("render and write aliases")
+
 	if err = s.renderSitemap(); err != nil {
 		return
 	}
@@ -878,6 +863,11 @@ func (s *Site) render() (err error) {
 	}
 	s.timerStep("render and write robots.txt")
 
+	if err = s.render404(); err != nil {
+		return
+	}
+	s.timerStep("render and write 404")
+
 	return
 }
 
@@ -1601,44 +1591,6 @@ func (s *Site) nodeTypeFromSections(sections []string) NodeType {
 	return NodeSection
 }
 
-// renderAliases renders shell pages that simply have a redirect in the header.
-func (s *Site) renderAliases() error {
-	for _, p := range s.Pages {
-		if len(p.Aliases) == 0 {
-			continue
-		}
-
-		plink, err := p.Permalink()
-		if err != nil {
-			return err
-		}
-		for _, a := range p.Aliases {
-			if err := s.writeDestAlias(a, plink, p); err != nil {
-				return err
-			}
-		}
-	}
-
-	if s.owner.multilingual.enabled() {
-		mainLang := s.owner.multilingual.DefaultLang.Lang
-		if s.Info.defaultContentLanguageInSubdir {
-			mainLangURL := s.Info.pathSpec.AbsURL(mainLang, false)
-			jww.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL)
-			if err := s.publishDestAlias(s.languageAliasTarget(), "/", mainLangURL, nil); err != nil {
-				return err
-			}
-		} else {
-			mainLangURL := s.Info.pathSpec.AbsURL("", false)
-			jww.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL)
-			if err := s.publishDestAlias(s.languageAliasTarget(), mainLang, mainLangURL, nil); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
 func (s *Site) preparePages() error {
 	var errors []error
 
@@ -2085,21 +2037,8 @@ func (s *Site) renderHomePage(prepare bool) error {
 		}
 	}
 
-	if viper.GetBool("disable404") {
 	return nil
-	}
 
-	node404 := s.newNode("404")
-	node404.Title = "404 Page not found"
-	node404.Data["Pages"] = s.Pages
-	s.setURLs(node404, "404.html")
-
-	nfLayouts := []string{"404.html"}
-	if nfErr := s.renderAndWritePage("404 page", "404.html", node404, s.appendThemeTemplates(nfLayouts)...); nfErr != nil {
-		return nfErr
-	}
-
-	return nil
 }
 
 func (s *Site) newHomeNode(prepare bool, counter int) *Node {
@@ -2115,80 +2054,6 @@ func (s *Site) newHomeNode(prepare bool, counter int) *Node {
 	return n
 }
 
-func (s *Site) newPage() *Page {
-	page := &Page{}
-	page.language = s.Language
-	page.Date = s.Info.LastChange
-	page.Lastmod = s.Info.LastChange
-	page.Site = &s.Info
-	return page
-}
-
-func (s *Site) renderSitemap() error {
-	if viper.GetBool("disableSitemap") {
-		return nil
-	}
-
-	sitemapDefault := parseSitemap(viper.GetStringMap("sitemap"))
-
-	n := s.newNode("sitemap")
-
-	// Prepend homepage to the list of pages
-	pages := make(Pages, 0)
-
-	page := s.newPage()
-	page.URLPath.URL = ""
-	page.Sitemap.ChangeFreq = sitemapDefault.ChangeFreq
-	page.Sitemap.Priority = sitemapDefault.Priority
-
-	pages = append(pages, page)
-	pages = append(pages, s.Pages...)
-
-	n.Data["Pages"] = pages
-
-	for _, page := range pages {
-		if page.Sitemap.ChangeFreq == "" {
-			page.Sitemap.ChangeFreq = sitemapDefault.ChangeFreq
-		}
-
-		if page.Sitemap.Priority == -1 {
-			page.Sitemap.Priority = sitemapDefault.Priority
-		}
-
-		if page.Sitemap.Filename == "" {
-			page.Sitemap.Filename = sitemapDefault.Filename
-		}
-	}
-
-	smLayouts := []string{"sitemap.xml", "_default/sitemap.xml", "_internal/_default/sitemap.xml"}
-	addLanguagePrefix := n.Site.IsMultiLingual()
-	if err := s.renderAndWriteXML("sitemap", n.addLangPathPrefixIfFlagSet(page.Sitemap.Filename, addLanguagePrefix), n, s.appendThemeTemplates(smLayouts)...); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *Site) renderRobotsTXT() error {
-	if !viper.GetBool("enableRobotsTXT") {
-		return nil
-	}
-
-	n := s.newNode("robots")
-	n.Data["Pages"] = s.Pages
-
-	rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
-	outBuffer := bp.GetBuffer()
-	defer bp.PutBuffer(outBuffer)
-	err := s.renderForLayouts("robots", n, outBuffer, s.appendThemeTemplates(rLayouts)...)
-
-	if err == nil {
-		err = s.writeDestFile("robots.txt", outBuffer)
-	}
-
-	return err
-}
-
 // Stats prints Hugo builds stats to the console.
 // This is what you see after a successful hugo build.
 func (s *Site) Stats() {
@@ -2223,15 +2088,19 @@ func (s *SiteInfo) permalinkStr(plink string) string {
 		s.pathSpec.URLizeAndPrep(plink)).String()
 }
 
+// TODO(bep) np remove
 func (s *Site) newNode(nodeID string) *Node {
-	return s.nodeLookup(nodeID, 0, true)
+	return nil //s.nodeLookup(nodeID, 0, true)
 }
 
 func (s *Site) getNode(nodeID string) *Node {
-	return s.getOrAddNode(nodeID, false)
+	return nil //s.getOrAddNode(nodeID, false)
 }
 
 func (s *Site) getOrAddNode(nodeID string, add bool) *Node {
+	if true {
+		return nil
+	}
 	s.nodeCacheInit.Do(func() {
 		s.nodeCache = &nodeCache{m: make(map[string]*Node)}
 	})
@@ -2342,7 +2211,8 @@ func (s *Site) renderAndWritePage(name string, dest string, d interface{}, layou
 
 	var pageTarget target.Output
 
-	if p, ok := d.(*Page); ok && path.Ext(p.URLPath.URL) != "" {
+	// TODO(bep) np ugly urls vs frontmatter
+	if p, ok := d.(*Page); ok && p.IsPage() && path.Ext(p.URLPath.URL) != "" {
 		// user has explicitly set a URL with extension for this page
 		// make sure it sticks even if "ugly URLs" are turned off.
 		pageTarget = s.pageUglyTarget()
@@ -2361,7 +2231,7 @@ func (s *Site) renderAndWritePage(name string, dest string, d interface{}, layou
 	}
 
 	// For performance reasons we only inject the Hugo generator tag on the home page.
-	if n, ok := d.(*Node); ok && n.IsHome() {
+	if n, ok := d.(*Page); ok && n.IsHome() {
 		if !viper.GetBool("disableHugoGeneratorInject") {
 			transformLinks = append(transformLinks, transform.HugoGeneratorInject)
 		}

hugolib/site_render.go

@@ -19,7 +19,9 @@ import (
 	"path/filepath"
 	"sync"
 
+	bp "github.com/spf13/hugo/bufferpool"
 	"github.com/spf13/hugo/helpers"
+	"github.com/spf13/viper"
 
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -152,3 +154,125 @@ func (s *Site) renderRSS(p *Page) error {
 
 	return nil
 }
+
+func (s *Site) render404() error {
+	if viper.GetBool("disable404") {
+		return nil
+	}
+
+	p := s.newNodePage(Node404)
+	p.Title = "404 Page not found"
+	p.Data["Pages"] = s.Pages
+	s.setPageURLs(p, "404.html")
+
+	nfLayouts := []string{"404.html"}
+	if nfErr := s.renderAndWritePage("404 page", "404.html", p, s.appendThemeTemplates(nfLayouts)...); nfErr != nil {
+		return nfErr
+	}
+
+	return nil
+}
+
+func (s *Site) renderSitemap() error {
+	if viper.GetBool("disableSitemap") {
+		return nil
+	}
+
+	sitemapDefault := parseSitemap(viper.GetStringMap("sitemap"))
+
+	n := s.newNodePage(NodeSitemap)
+
+	// Prepend homepage to the list of pages
+	pages := make(Pages, 0)
+
+	page := s.newNodePage(NodeSitemap)
+	page.URLPath.URL = ""
+	page.Sitemap.ChangeFreq = sitemapDefault.ChangeFreq
+	page.Sitemap.Priority = sitemapDefault.Priority
+
+	pages = append(pages, page)
+	pages = append(pages, s.Pages...)
+
+	n.Data["Pages"] = pages
+
+	for _, page := range pages {
+		if page.Sitemap.ChangeFreq == "" {
+			page.Sitemap.ChangeFreq = sitemapDefault.ChangeFreq
+		}
+
+		if page.Sitemap.Priority == -1 {
+			page.Sitemap.Priority = sitemapDefault.Priority
+		}
+
+		if page.Sitemap.Filename == "" {
+			page.Sitemap.Filename = sitemapDefault.Filename
+		}
+	}
+
+	smLayouts := []string{"sitemap.xml", "_default/sitemap.xml", "_internal/_default/sitemap.xml"}
+	addLanguagePrefix := n.Site.IsMultiLingual()
+	if err := s.renderAndWriteXML("sitemap", n.addLangPathPrefixIfFlagSet(page.Sitemap.Filename, addLanguagePrefix), n, s.appendThemeTemplates(smLayouts)...); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Site) renderRobotsTXT() error {
+	if !viper.GetBool("enableRobotsTXT") {
+		return nil
+	}
+
+	n := s.newNodePage(NodeRobotsTXT)
+	n.Data["Pages"] = s.Pages
+
+	rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
+	outBuffer := bp.GetBuffer()
+	defer bp.PutBuffer(outBuffer)
+	err := s.renderForLayouts("robots", n, outBuffer, s.appendThemeTemplates(rLayouts)...)
+
+	if err == nil {
+		err = s.writeDestFile("robots.txt", outBuffer)
+	}
+
+	return err
+}
+
+// renderAliases renders shell pages that simply have a redirect in the header.
+// TODO(bep) np aliases of node types
+func (s *Site) renderAliases() error {
+	for _, p := range s.Pages {
+		if len(p.Aliases) == 0 {
+			continue
+		}
+
+		plink, err := p.Permalink()
+		if err != nil {
+			return err
+		}
+		for _, a := range p.Aliases {
+			if err := s.writeDestAlias(a, plink, p); err != nil {
+				return err
+			}
+		}
+	}
+
+	if s.owner.multilingual.enabled() {
+		mainLang := s.owner.multilingual.DefaultLang.Lang
+		if s.Info.defaultContentLanguageInSubdir {
+			mainLangURL := s.Info.pathSpec.AbsURL(mainLang, false)
+			jww.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL)
+			if err := s.publishDestAlias(s.languageAliasTarget(), "/", mainLangURL, nil); err != nil {
+				return err
+			}
+		} else {
+			mainLangURL := s.Info.pathSpec.AbsURL("", false)
+			jww.DEBUG.Printf("Write redirect to main language %s: %s", mainLang, mainLangURL)
+			if err := s.publishDestAlias(s.languageAliasTarget(), mainLang, mainLangURL, nil); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}

hugolib/site_test.go

@@ -393,6 +393,7 @@ func doTestShouldAlwaysHaveUglyURLs(t *testing.T, uglyURLs bool) {
 
 // Issue #1176
 func TestSectionNaming(t *testing.T) {
+	//jww.SetStdoutThreshold(jww.LevelDebug)
 
 	for _, canonify := range []bool{true, false} {
 		for _, uglify := range []bool{true, false} {
@@ -404,7 +405,6 @@ func TestSectionNaming(t *testing.T) {
 }
 
 func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) {
-	hugofs.InitMemFs()
 	testCommonResetState()
 
 	viper.Set("baseURL", "http://auth/sub/")
@@ -427,12 +427,12 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) {
 		{Name: filepath.FromSlash("ラーメン/doc3.html"), Content: []byte("doc3")},
 	}
 
-	s := &Site{
+	for _, source := range sources {
-		Source:   &source.InMemorySource{ByteSource: sources},
+		writeSource(t, filepath.Join("content", source.Name), string(source.Content))
-		targets:  targetList{page: &target.PagePub{UglyURLs: uglify}},
-		Language: helpers.NewDefaultLanguage(),
 	}
 
+	s := newSiteDefaultLang()
+
 	if err := buildAndRenderSite(s,
 		"_default/single.html", "{{.Content}}",
 		"_default/list.html", "{{ .Title }}"); err != nil {
@@ -453,20 +453,12 @@ func doTestSectionNaming(t *testing.T, canonify, uglify, pluralize bool) {
 	}
 
 	for _, test := range tests {
-		file, err := hugofs.Destination().Open(test.doc)
-		if err != nil {
-			t.Fatalf("Did not find %s in target: %s", test.doc, err)
-		}
-
-		content := helpers.ReaderToString(file)
 
 		if test.pluralAware && pluralize {
 			test.expected = inflect.Pluralize(test.expected)
 		}
 
-		if content != test.expected {
+		assertFileContent(t, filepath.Join("public", test.doc), true, test.expected)
-			t.Errorf("%s content expected:\n%q\ngot:\n%q", test.doc, test.expected, content)
-		}
 	}
 
 }