Fix 0.62.1 server rebuild slowdown regression

Fixes #6784

hugofs/fileinfo.go

@@ -35,8 +35,8 @@ import (
 
 const (
 	metaKeyFilename = "filename"
-	metaKeyPathFile                   = "pathFile"    // Path of filename relative to a root.
-	metaKeyIsFileMount                = "isFileMount" // Whether the source mount was a file.
+
+	metaKeyBaseDir                    = "baseDir" // Abs base directory of source file.
 	metaKeyMountRoot                  = "mountRoot"
 	metaKeyOriginalFilename           = "originalFilename"
 	metaKeyName                       = "name"
@@ -116,29 +116,19 @@ func (f FileMeta) Path() string {
 	return f.stringV(metaKeyPath)
 }
 
-// PathFile returns the relative file path for the file source. This
-// will in most cases be the same as Path.
+// PathFile returns the relative file path for the file source.
 func (f FileMeta) PathFile() string {
-	pf := f.stringV(metaKeyPathFile)
-	if f.isFileMount() {
-		return pf
+	base := f.stringV(metaKeyBaseDir)
+	if base == "" {
+		return ""
 	}
-	mountRoot := f.mountRoot()
-	if mountRoot == pf {
-		return f.Path()
-	}
-
-	return pf + (strings.TrimPrefix(f.Path(), mountRoot))
+	return strings.TrimPrefix(strings.TrimPrefix(f.Filename(), base), filepathSeparator)
 }
 
-func (f FileMeta) mountRoot() string {
+func (f FileMeta) MountRoot() string {
 	return f.stringV(metaKeyMountRoot)
 }
 
-func (f FileMeta) isFileMount() bool {
-	return f.GetBool(metaKeyIsFileMount)
-}
-
 func (f FileMeta) Weight() int {
 	return f.GetInt(metaKeyWeight)
 }

hugofs/rootmapping_fs.go

@@ -57,12 +57,8 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 		// Extract "blog" from "content/blog"
 		rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
 		if rm.Meta != nil {
-			rm.Meta[metaKeyIsFileMount] = !fi.IsDir()
+			rm.Meta[metaKeyBaseDir] = rm.ToBasedir
 			rm.Meta[metaKeyMountRoot] = rm.path
-			if rm.ToBasedir != "" {
-				pathFile := strings.TrimPrefix(strings.TrimPrefix(rm.To, rm.ToBasedir), filepathSeparator)
-				rm.Meta[metaKeyPathFile] = pathFile
-			}
 		}
 
 		meta := copyFileMeta(rm.Meta)

hugofs/rootmapping_fs_test.go

@@ -271,7 +271,7 @@ func TestRootMappingFsMount(t *testing.T) {
 	c.Assert(singles, qt.HasLen, 2)
 	for i, lang := range []string{"no", "sv"} {
 		fi := singles[i].(FileMetaInfo)
-		c.Assert(fi.Meta().PathFile(), qt.Equals, lang+".txt")
+		c.Assert(fi.Meta().PathFile(), qt.Equals, filepath.FromSlash("themes/a/singlefiles/"+lang+".txt"))
 		c.Assert(fi.Meta().Lang(), qt.Equals, lang)
 		c.Assert(fi.Name(), qt.Equals, "p1.md")
 	}

hugolib/content_render_hooks_test.go

@@ -14,7 +14,10 @@
 package hugolib
 
 import (
+	"fmt"
 	"testing"
+
+	qt "github.com/frankban/quicktest"
 )
 
 func TestRenderHooks(t *testing.T) {
@@ -118,7 +121,20 @@ title: With RenderString
 {{< myshortcode5 >}}Inner Link: [Inner Link](https://www.gohugo.io "Hugo's Homepage"){{< /myshortcode5 >}}
 
 `)
-	b.Build(BuildCfg{})
+
+	for i := 1; i <= 30; i++ {
+		// Add some content with no shortcodes or links, i.e no templates needed.
+		b.WithContent(fmt.Sprintf("blog/notempl%d.md", i), `---
+title: No Template
+---
+
+## Content
+`)
+	}
+	counters := &testCounters{}
+	b.Build(BuildCfg{testCounters: counters})
+	b.Assert(int(counters.contentRenderCounter), qt.Equals, 50)
+
 	b.AssertFileContent("public/blog/p1/index.html", `
 <p>Cool Page|https://www.google.com|Title: Google's Homepage|Text: First Link|END</p>
 Text: Second
@@ -149,7 +165,11 @@ SHORT3|
 		"layouts/shortcodes/myshortcode3.html", `SHORT3_EDITED|`,
 	)
 
-	b.Build(BuildCfg{})
+	counters = &testCounters{}
+	b.Build(BuildCfg{testCounters: counters})
+	// Make sure that only content using the changed templates are re-rendered.
+	b.Assert(int(counters.contentRenderCounter), qt.Equals, 7)
+
 	b.AssertFileContent("public/customview/p1/index.html", `.Render: myrender: Custom View|P4: PARTIAL4_EDITED`)
 	b.AssertFileContent("public/blog/p1/index.html", `<p>EDITED: https://www.google.com|</p>`, "SHORT3_EDITED|")
 	b.AssertFileContent("public/blog/p2/index.html", `PARTIAL1_EDITED`)

hugolib/filesystems/basefs.go

@@ -295,21 +295,16 @@ func (d *SourceFilesystem) Contains(filename string) bool {
 	return false
 }
 
-// Path returns the relative path to the given filename if it is a member of
+// Path returns the mount relative path to the given filename if it is a member of
 // of the current filesystem, an empty string if not.
 func (d *SourceFilesystem) Path(filename string) string {
 	for _, dir := range d.Dirs {
 		meta := dir.Meta()
-		if !dir.IsDir() {
-			if filename == meta.Filename() {
-				return meta.PathFile()
-			}
-			continue
-		}
-
 		if strings.HasPrefix(filename, meta.Filename()) {
 			p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator)
-			p = path.Join(meta.PathFile(), p)
+			if mountRoot := meta.MountRoot(); mountRoot != "" {
+				return filepath.Join(mountRoot, p)
+			}
 			return p
 		}
 	}

hugolib/hugo_modules_test.go

@@ -668,6 +668,126 @@ Readme Edit
 
 }
 
+func TestMountsPaths(t *testing.T) {
+	c := qt.New(t)
+
+	type test struct {
+		b          *sitesBuilder
+		clean      func()
+		workingDir string
+	}
+
+	prepare := func(c *qt.C, mounts string) test {
+		workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mounts-paths")
+		c.Assert(err, qt.IsNil)
+
+		configTemplate := `
+baseURL = "https://example.com"
+title = "My Modular Site"
+workingDir = %q
+
+%s
+
+`
+		config := fmt.Sprintf(configTemplate, workingDir, mounts)
+		config = strings.Replace(config, "WORKING_DIR", workingDir, -1)
+
+		b := newTestSitesBuilder(c).Running()
+
+		b.Fs = hugofs.NewDefault(viper.New())
+
+		os.MkdirAll(filepath.Join(workingDir, "content", "blog"), 0777)
+
+		b.WithWorkingDir(workingDir).WithConfigFile("toml", config)
+
+		return test{
+			b:          b,
+			clean:      clean,
+			workingDir: workingDir,
+		}
+
+	}
+
+	c.Run("Default", func(c *qt.C) {
+		mounts := ``
+
+		test := prepare(c, mounts)
+		b := test.b
+		defer test.clean()
+
+		b.WithContent("blog/p1.md", `---
+title: P1
+---`)
+
+		b.Build(BuildCfg{})
+
+		p := b.GetPage("blog/p1.md")
+		f := p.File().FileInfo().Meta()
+		b.Assert(filepath.ToSlash(f.Path()), qt.Equals, "blog/p1.md")
+		b.Assert(filepath.ToSlash(f.PathFile()), qt.Equals, "content/blog/p1.md")
+
+		b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(test.workingDir, "layouts", "_default", "single.html")), qt.Equals, filepath.FromSlash("_default/single.html"))
+
+	})
+
+	c.Run("Mounts", func(c *qt.C) {
+		absDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-mounts-paths-abs")
+		c.Assert(err, qt.IsNil)
+		defer clean()
+
+		mounts := `[module]
+  [[module.mounts]]
+    source = "README.md"
+    target = "content/_index.md"
+  [[module.mounts]]
+    source = "mycontent"
+    target = "content/blog"
+   [[module.mounts]]
+    source = "subdir/mypartials"
+    target = "layouts/partials"
+   [[module.mounts]]
+    source = %q
+    target = "layouts/shortcodes"
+`
+		mounts = fmt.Sprintf(mounts, filepath.Join(absDir, "/abs/myshortcodes"))
+
+		test := prepare(c, mounts)
+		b := test.b
+		defer test.clean()
+
+		subContentDir := filepath.Join(test.workingDir, "mycontent", "sub")
+		os.MkdirAll(subContentDir, 0777)
+		myPartialsDir := filepath.Join(test.workingDir, "subdir", "mypartials")
+		os.MkdirAll(myPartialsDir, 0777)
+
+		absShortcodesDir := filepath.Join(absDir, "abs", "myshortcodes")
+		os.MkdirAll(absShortcodesDir, 0777)
+
+		b.WithSourceFile("README.md", "---\ntitle: Readme\n---")
+		b.WithSourceFile("mycontent/sub/p1.md", "---\ntitle: P1\n---")
+
+		b.WithSourceFile(filepath.Join(absShortcodesDir, "myshort.html"), "MYSHORT")
+		b.WithSourceFile(filepath.Join(myPartialsDir, "mypartial.html"), "MYPARTIAL")
+
+		b.Build(BuildCfg{})
+
+		p1_1 := b.GetPage("/blog/sub/p1.md")
+		p1_2 := b.GetPage("/mycontent/sub/p1.md")
+		b.Assert(p1_1, qt.Not(qt.IsNil))
+		b.Assert(p1_2, qt.Equals, p1_1)
+
+		f := p1_1.File().FileInfo().Meta()
+		b.Assert(filepath.ToSlash(f.Path()), qt.Equals, "blog/sub/p1.md")
+		b.Assert(filepath.ToSlash(f.PathFile()), qt.Equals, "mycontent/sub/p1.md")
+		b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(myPartialsDir, "mypartial.html")), qt.Equals, filepath.FromSlash("partials/mypartial.html"))
+		b.Assert(b.H.BaseFs.Layouts.Path(filepath.Join(absShortcodesDir, "myshort.html")), qt.Equals, filepath.FromSlash("shortcodes/myshort.html"))
+		b.Assert(b.H.BaseFs.Content.Path(filepath.Join(subContentDir, "p1.md")), qt.Equals, filepath.FromSlash("blog/sub/p1.md"))
+		b.Assert(b.H.BaseFs.Content.Path(filepath.Join(test.workingDir, "README.md")), qt.Equals, filepath.FromSlash("_index.md"))
+
+	})
+
+}
+
 // https://github.com/gohugoio/hugo/issues/6299
 func TestSiteWithGoModButNoModules(t *testing.T) {
 	t.Parallel()

hugolib/hugo_sites.go

@@ -19,6 +19,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/gohugoio/hugo/identity"
 
@@ -82,6 +83,19 @@ type HugoSites struct {
 	init *hugoSitesInit
 
 	*fatalErrorHandler
+	*testCounters
+}
+
+// Only used in tests.
+type testCounters struct {
+	contentRenderCounter uint64
+}
+
+func (h *testCounters) IncrContentRender() {
+	if h == nil {
+		return
+	}
+	atomic.AddUint64(&h.contentRenderCounter, 1)
 }
 
 type fatalErrorHandler struct {
@@ -579,6 +593,8 @@ type BuildCfg struct {
 
 	// Recently visited URLs. This is used for partial re-rendering.
 	RecentlyVisited map[string]bool
+
+	testCounters *testCounters
 }
 
 // shouldRender is used in the Fast Render Mode to determine if we need to re-render

hugolib/hugo_sites_build.go

@@ -66,6 +66,8 @@ func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
 		h.Metrics.Reset()
 	}
 
+	h.testCounters = config.testCounters
+
 	// Need a pointer as this may be modified.
 	conf := &config
 

hugolib/page__per_output.go

@@ -80,6 +80,8 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err
 	}
 
 	initContent := func() (err error) {
+		p.s.h.IncrContentRender()
+
 		if p.cmap == nil {
 			// Nothing to do.
 			return nil

hugolib/testhelpers_test.go

@@ -224,6 +224,9 @@ func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder {
 
 func (s *sitesBuilder) absFilename(filename string) string {
 	filename = filepath.FromSlash(filename)
+	if filepath.IsAbs(filename) {
+		return filename
+	}
 	if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) {
 		filename = filepath.Join(s.workingDir, filename)
 	}
@@ -736,6 +739,12 @@ func (s *sitesBuilder) CheckExists(filename string) bool {
 	return destinationExists(s.Fs, filepath.Clean(filename))
 }
 
+func (s *sitesBuilder) GetPage(ref string) page.Page {
+	p, err := s.H.Sites[0].getPageNew(nil, ref)
+	s.Assert(err, qt.IsNil)
+	return p
+}
+
 func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper {
 	return testHelper{
 		Cfg: cfg,