hugofs: Fix mount with hole regression

Fixes #6854

hugofs/decorators.go

@@ -81,12 +81,28 @@ func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {
 // NewBaseFileDecorator decorates the given Fs to provide the real filename
 // and an Opener func.
 func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
-
 	ffs := &baseFileDecoratorFs{Fs: fs}
 
 	decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
 		// Store away the original in case it's a symlink.
 		meta := FileMeta{metaKeyName: fi.Name()}
+		if fi.IsDir() {
+			meta[metaKeyJoinStat] = func(name string) (FileMetaInfo, error) {
+				joinedFilename := filepath.Join(filename, name)
+				fi, _, err := lstatIfPossible(fs, joinedFilename)
+				if err != nil {
+					return nil, err
+				}
+
+				fi, err = ffs.decorate(fi, joinedFilename)
+				if err != nil {
+					return nil, err
+				}
+
+				return fi.(FileMetaInfo), nil
+			}
+		}
+
 		isSymlink := isSymlink(fi)
 		if isSymlink {
 			meta[metaKeyOriginalFilename] = filename

hugofs/fileinfo.go

@@ -50,6 +50,7 @@ const (
 	metaKeyOpener                     = "opener"
 	metaKeyIsOrdered                  = "isOrdered"
 	metaKeyIsSymlink                  = "isSymlink"
+	metaKeyJoinStat                   = "joinStat"
 	metaKeySkipDir                    = "skipDir"
 	metaKeyClassifier                 = "classifier"
 	metaKeyTranslationBaseName        = "translationBaseName"
@@ -177,6 +178,14 @@ func (f FileMeta) Open() (afero.File, error) {
 	return v.(func() (afero.File, error))()
 }
 
+func (f FileMeta) JoinStat(name string) (FileMetaInfo, error) {
+	v, found := f[metaKeyJoinStat]
+	if !found {
+		return nil, os.ErrNotExist
+	}
+	return v.(func(name string) (FileMetaInfo, error))(name)
+}
+
 func (f FileMeta) stringV(key string) string {
 	if v, found := f[key]; found {
 		return v.(string)

hugofs/rootmapping_fs.go

@@ -128,6 +128,11 @@ type RootMapping struct {
 
 }
 
+type keyRootMappings struct {
+	key   string
+	roots []RootMapping
+}
+
 func (rm *RootMapping) clean() {
 	rm.From = strings.Trim(filepath.Clean(rm.From), filepathSeparator)
 	rm.To = filepath.Clean(rm.To)
@@ -281,6 +286,21 @@ func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
 	return roots
 }
 
+func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
+	var roots []keyRootMappings
+	fs.rootMapToReal.WalkPath(prefix, func(s string, v interface{}) bool {
+		if strings.HasPrefix(prefix, s+filepathSeparator) {
+			roots = append(roots, keyRootMappings{
+				key:   s,
+				roots: v.([]RootMapping),
+			})
+		}
+		return false
+	})
+
+	return roots
+}
+
 func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
 	meta := fis[0].Meta()
 	f, err := meta.Open()
@@ -342,17 +362,15 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
 	seen := make(map[string]bool) // Prevent duplicate directories
 	level := strings.Count(prefix, filepathSeparator)
 
-	// First add any real files/directories.
-	rms := fs.getRoot(prefix)
-	for _, rm := range rms {
-		f, err := rm.fi.Meta().Open()
+	collectDir := func(rm RootMapping, fi FileMetaInfo) error {
+		f, err := fi.Meta().Open()
 		if err != nil {
-			return nil, err
+			return err
 		}
 		direntries, err := f.Readdir(-1)
 		if err != nil {
 			f.Close()
-			return nil, err
+			return err
 		}
 
 		for _, fi := range direntries {
@@ -374,6 +392,16 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
 		}
 
 		f.Close()
+
+		return nil
+	}
+
+	// First add any real files/directories.
+	rms := fs.getRoot(prefix)
+	for _, rm := range rms {
+		if err := collectDir(rm, rm.fi); err != nil {
+			return nil, err
+		}
 	}
 
 	// Next add any file mounts inside the given directory.
@@ -428,6 +456,22 @@ func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error)
 		return false
 	})
 
+	// Finally add any ancestor dirs with files in this directory.
+	ancestors := fs.getAncestors(prefix)
+	for _, root := range ancestors {
+		subdir := strings.TrimPrefix(prefix, root.key)
+		for _, rm := range root.roots {
+			if rm.fi.IsDir() {
+				fi, err := rm.fi.Meta().JoinStat(subdir)
+				if err == nil {
+					if err := collectDir(rm, fi); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	}
+
 	return fis, nil
 }
 

hugofs/rootmapping_fs_test.go

@@ -365,12 +365,18 @@ func TestRootMappingFsOs(t *testing.T) {
 
 	c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil)
 
+	// https://github.com/gohugoio/hugo/issues/6854
+	mystaticDir := filepath.Join(d, "mystatic", "a", "b", "c")
+	c.Assert(fs.MkdirAll(mystaticDir, 0755), qt.IsNil)
+	c.Assert(afero.WriteFile(fs, filepath.Join(mystaticDir, "ms-1.txt"), []byte("some content"), 0755), qt.IsNil)
+
 	rfs, err := newRootMappingFsFromFromTo(
 		d,
 		fs,
 		"static/bf1", filepath.Join(d, "f1t"),
 		"static/cf2", filepath.Join(d, "f2t"),
 		"static/af3", filepath.Join(d, "f3t"),
+		"static", filepath.Join(d, "mystatic"),
 		"static/a/b/c", filepath.Join(d, "d1", "d2", "d3"),
 		"layouts", filepath.Join(d, "d1"),
 	)
@@ -400,13 +406,13 @@ func TestRootMappingFsOs(t *testing.T) {
 	}
 
 	c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"})
-	c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"})
+	c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
 	c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"})
 
 	all, err := collectFilenames(rfs, "static", "static")
 	c.Assert(err, qt.IsNil)
 
-	c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"})
+	c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "a/b/c/ms-1.txt", "cf2/myfile.txt"})
 
 	fis, err := collectFileinfos(rfs, "static", "static")
 	c.Assert(err, qt.IsNil)
@@ -423,7 +429,7 @@ func TestRootMappingFsOs(t *testing.T) {
 	sortFileInfos(fileInfos)
 	i := 0
 	for _, fi := range fileInfos {
-		if fi.IsDir() {
+		if fi.IsDir() || fi.Name() == "ms-1.txt" {
 			continue
 		}
 		i++
@@ -437,3 +443,47 @@ func TestRootMappingFsOs(t *testing.T) {
 	_, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3"))
 	c.Assert(err, qt.IsNil)
 }
+
+func TestRootMappingFsOsBase(t *testing.T) {
+	c := qt.New(t)
+	fs := NewBaseFileDecorator(afero.NewOsFs())
+
+	d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os-base")
+	c.Assert(err, qt.IsNil)
+	defer clean()
+
+	// Deep structure
+	deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5")
+	c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil)
+	for i := 1; i <= 3; i++ {
+		c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil)
+		c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil)
+	}
+
+	mystaticDir := filepath.Join(d, "mystatic", "a", "b", "c")
+	c.Assert(fs.MkdirAll(mystaticDir, 0755), qt.IsNil)
+	c.Assert(afero.WriteFile(fs, filepath.Join(mystaticDir, "ms-1.txt"), []byte("some content"), 0755), qt.IsNil)
+
+	bfs := afero.NewBasePathFs(fs, d)
+
+	rfs, err := newRootMappingFsFromFromTo(
+		"",
+		bfs,
+		"static", "mystatic",
+		"static/a/b/c", filepath.Join("d1", "d2", "d3"),
+	)
+
+	getDirnames := func(dirname string) []string {
+		dirname = filepath.FromSlash(dirname)
+		f, err := rfs.Open(dirname)
+		c.Assert(err, qt.IsNil)
+		defer f.Close()
+		dirnames, err := f.Readdirnames(-1)
+		c.Assert(err, qt.IsNil)
+		sort.Strings(dirnames)
+		return dirnames
+	}
+
+	c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt", "ms-1.txt"})
+
+}