Fix module mount in sub folder

This addresses a specific issue, but is a also a major simplification of the filesystem file mounts.

Fixes #6730

hugofs/decorators.go

@@ -79,7 +79,7 @@ func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs {
 }
 
 // NewBaseFileDecorator decorates the given Fs to provide the real filename
-// and an Opener func. If
+// and an Opener func.
 func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
 
 	ffs := &baseFileDecoratorFs{Fs: fs}
@@ -102,7 +102,6 @@ func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
 
 		opener := func() (afero.File, error) {
 			return ffs.open(filename)
-
 		}
 
 		return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil

hugofs/fileinfo.go

@@ -18,6 +18,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"time"
 
@@ -271,13 +272,21 @@ func (fi *dirNameOnlyFileInfo) Sys() interface{} {
 	return nil
 }
 
-func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {
+func newDirNameOnlyFileInfo(name string, meta FileMeta, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {
 	name = normalizeFilename(name)
 	_, base := filepath.Split(name)
-	return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{
+
-		metaKeyFilename:  name,
+	m := copyFileMeta(meta)
-		metaKeyIsOrdered: isOrdered,
+	if _, found := m[metaKeyFilename]; !found {
-		metaKeyOpener:    fileOpener})
+		m.setIfNotZero(metaKeyFilename, name)
+	}
+	m[metaKeyOpener] = fileOpener
+	m[metaKeyIsOrdered] = isOrdered
+
+	return NewFileMetaInfo(
+		&dirNameOnlyFileInfo{name: base},
+		m,
+	)
 }
 
 func decorateFileInfo(
@@ -339,3 +348,18 @@ func fileInfosToNames(fis []os.FileInfo) []string {
 	}
 	return names
 }
+
+func fromSlash(filenames []string) []string {
+	for i, name := range filenames {
+		filenames[i] = filepath.FromSlash(name)
+	}
+	return filenames
+}
+
+func sortFileInfos(fis []os.FileInfo) {
+	sort.Slice(fis, func(i, j int) bool {
+		fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo)
+		return fimi.Meta().Filename() < fimj.Meta().Filename()
+
+	})
+}

hugofs/nosymlink_test.go

@@ -137,6 +137,7 @@ func TestNoSymlinkFs(t *testing.T) {
 			c.Assert(err, qt.IsNil)
 			// There is at least one unsported symlink inside workDir
 			_, err = f.Readdir(-1)
+			c.Assert(err, qt.IsNil)
 			f.Close()
 			c.Assert(logger.WarnCounter.Count(), qt.Equals, uint64(1))
 

hugofs/rootmapping_fs.go

@@ -27,15 +27,18 @@ import (
 	"github.com/spf13/afero"
 )
 
-var filepathSeparator = string(filepath.Separator)
+var (
+	filepathSeparator = string(filepath.Separator)
+)
 
 // NewRootMappingFs creates a new RootMappingFs on top of the provided with
-// of root mappings with some optional metadata about the root.
+// root mappings with some optional metadata about the root.
 // Note that From represents a virtual root that maps to the actual filename in To.
 func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 	rootMapToReal := radix.New()
+	var virtualRoots []RootMapping
 
-	for i, rm := range rms {
+	for _, rm := range rms {
 		(&rm).clean()
 
 		fromBase := files.ResolveComponentFolder(rm.From)
@@ -56,11 +59,13 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 		}
 		// Extract "blog" from "content/blog"
 		rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
-		if rm.Meta != nil {
+		if rm.Meta == nil {
-			rm.Meta[metaKeyBaseDir] = rm.ToBasedir
+			rm.Meta = make(FileMeta)
-			rm.Meta[metaKeyMountRoot] = rm.path
 		}
 
+		rm.Meta[metaKeyBaseDir] = rm.ToBasedir
+		rm.Meta[metaKeyMountRoot] = rm.path
+
 		meta := copyFileMeta(rm.Meta)
 
 		if !fi.IsDir() {
@@ -70,7 +75,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 
 		rm.fi = NewFileMetaInfo(fi, meta)
 
-		key := rm.rootKey()
+		key := filepathSeparator + rm.From
 		var mappings []RootMapping
 		v, found := rootMapToReal.Get(key)
 		if found {
@@ -80,30 +85,38 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 		mappings = append(mappings, rm)
 		rootMapToReal.Insert(key, mappings)
 
-		rms[i] = rm
+		virtualRoots = append(virtualRoots, rm)
 	}
 
-	rfs := &RootMappingFs{Fs: fs,
+	rootMapToReal.Insert(filepathSeparator, virtualRoots)
-		virtualRoots:  rms,
+
-		rootMapToReal: rootMapToReal}
+	rfs := &RootMappingFs{
+		Fs:            fs,
+		rootMapToReal: rootMapToReal,
+	}
 
 	return rfs, nil
 }
 
-// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking
+func newRootMappingFsFromFromTo(
-// From and To as string pairs.
+	baseDir string,
-func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {
+	fs afero.Fs,
+	fromTo ...string,
+) (*RootMappingFs, error) {
+
 	rms := make([]RootMapping, len(fromTo)/2)
 	for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 {
 		rms[i] = RootMapping{
-			From: fromTo[j],
+			From:      fromTo[j],
-			To:   fromTo[j+1],
+			To:        fromTo[j+1],
+			ToBasedir: baseDir,
 		}
 	}
 
 	return NewRootMappingFs(fs, rms...)
 }
 
+// RootMapping describes a virtual file or directory mount.
 type RootMapping struct {
 	From      string   // The virtual mount.
 	To        string   // The source directory or file.
@@ -127,21 +140,16 @@ func (r RootMapping) filename(name string) string {
 	return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
 }
 
-func (r RootMapping) rootKey() string {
-	return r.From
-}
-
 // A RootMappingFs maps several roots into one. Note that the root of this filesystem
 // is directories only, and they will be returned in Readdir and Readdirnames
 // in the order given.
 type RootMappingFs struct {
 	afero.Fs
 	rootMapToReal *radix.Tree
-	virtualRoots  []RootMapping
-	filter        func(r RootMapping) bool
 }
 
 func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
+	base = filepathSeparator + fs.cleanName(base)
 	roots := fs.getRootsWithPrefix(base)
 
 	if roots == nil {
@@ -176,138 +184,46 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
 	return fss, nil
 }
 
+// Filter creates a copy of this filesystem with only mappings matching a filter.
+func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
+	rootMapToReal := radix.New()
+	fs.rootMapToReal.Walk(func(b string, v interface{}) bool {
+		rms := v.([]RootMapping)
+		var nrms []RootMapping
+		for _, rm := range rms {
+			if f(rm) {
+				nrms = append(nrms, rm)
+			}
+		}
+		if len(nrms) != 0 {
+			rootMapToReal.Insert(b, nrms)
+		}
+		return false
+	})
+
+	fs.rootMapToReal = rootMapToReal
+
+	return &fs
+}
+
 // LstatIfPossible returns the os.FileInfo structure describing a given file.
 func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
-	fis, _, b, err := fs.doLstat(name, false)
+	fis, err := fs.doLstat(name)
 	if err != nil {
-		return nil, b, err
+		return nil, false, err
 	}
-	return fis[0], b, nil
+	return fis[0], false, nil
 }
 
-func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) {
+// Open opens the named file for reading.
-	return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil }
-}
-
-func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) {
-	if fs.isRoot(name) {
-		return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil
-	}
-
-	roots := fs.getRoots(name)
-	rootsWithPrefix := fs.getRootsWithPrefix(name)
-	hasRootMappingsBelow := len(rootsWithPrefix) != 0
-
-	if len(roots) == 0 {
-		if hasRootMappingsBelow {
-			// No exact matches, but we have root mappings below name,
-			// let's make it look like a directory.
-			return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, nil, false, nil
-		}
-
-		return nil, nil, false, os.ErrNotExist
-	}
-
-	// We may have a mapping for both static and static/subdir.
-	// These will not show in any Readdir so append them
-	// manually.
-	rootsInDir := fs.filterRootsBelow(rootsWithPrefix, name)
-
-	var (
-		fis  []FileMetaInfo
-		dirs []FileMetaInfo
-		b    bool
-		root RootMapping
-		err  error
-	)
-
-	for _, root = range roots {
-		var fi os.FileInfo
-		fi, b, err = fs.statRoot(root, name)
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-			return nil, nil, false, err
-		}
-		fim := fi.(FileMetaInfo)
-
-		fis = append(fis, fim)
-	}
-
-	for _, root = range rootsInDir {
-
-		fi, _, err := fs.statRoot(root, "")
-		if err != nil {
-			if os.IsNotExist(err) {
-				continue
-			}
-			return nil, nil, false, err
-		}
-		fim := fi.(FileMetaInfo)
-		dirs = append(dirs, fim)
-	}
-
-	if len(fis) == 0 && len(dirs) == 0 {
-		return nil, nil, false, os.ErrNotExist
-	}
-
-	if allowMultiple || len(fis) == 1 {
-		return fis, dirs, b, nil
-	}
-
-	if len(fis) == 0 {
-		return nil, nil, false, os.ErrNotExist
-	}
-
-	// Open it in this composite filesystem.
-	opener := func() (afero.File, error) {
-		return fs.Open(name)
-	}
-
-	return []FileMetaInfo{decorateFileInfo(fis[0], fs, opener, "", "", root.Meta)}, nil, b, nil
-
-}
-
-// Open opens the namedrootMappingFile file for reading.
 func (fs *RootMappingFs) Open(name string) (afero.File, error) {
-	if fs.isRoot(name) {
+	fis, err := fs.doLstat(name)
-		return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil
-	}
 
-	fis, dirs, _, err := fs.doLstat(name, true)
 	if err != nil {
 		return nil, err
 	}
 
-	if len(fis) == 1 {
+	return fs.newUnionFile(fis...)
-		fi := fis[0]
-		meta := fi.(FileMetaInfo).Meta()
-		f, err := meta.Open()
-		if err != nil {
-			return nil, err
-		}
-
-		f = &rootMappingFile{File: f, fs: fs, name: name, meta: meta}
-
-		if len(dirs) > 0 {
-			return &readDirDirsAppender{File: f, dirs: dirs}, nil
-		}
-
-		return f, nil
-	}
-
-	f, err := fs.newUnionFile(fis...)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(dirs) > 0 {
-		return &readDirDirsAppender{File: f, dirs: dirs}, nil
-	}
-
-	return f, nil
-
 }
 
 // Stat returns the os.FileInfo structure describing a given file.  If there is
@@ -318,80 +234,51 @@ func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {
 
 }
 
-// Filter creates a copy of this filesystem with the applied filter.
+func (fs *RootMappingFs) hasPrefix(prefix string) bool {
-func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
+	hasPrefix := false
-	fs.filter = f
+	fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
-	return &fs
+		hasPrefix = true
+		return true
+	})
+
+	return hasPrefix
 }
 
-func (fs *RootMappingFs) isRoot(name string) bool {
+func (fs *RootMappingFs) getRoot(key string) []RootMapping {
-	return name == "" || name == filepathSeparator
+	v, found := fs.rootMapToReal.Get(key)
-
-}
-
-func (fs *RootMappingFs) getRoots(name string) []RootMapping {
-	name = filepath.Clean(name)
-	_, v, found := fs.rootMapToReal.LongestPrefix(name)
 	if !found {
 		return nil
 	}
 
-	rm := v.([]RootMapping)
+	return v.([]RootMapping)
-
-	return fs.applyFilterToRoots(rm)
 }
 
-func (fs *RootMappingFs) applyFilterToRoots(rm []RootMapping) []RootMapping {
+func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) {
-	if fs.filter == nil {
+	s, v, found := fs.rootMapToReal.LongestPrefix(key)
-		return rm
+	if !found || (s == filepathSeparator && key != filepathSeparator) {
+		return "", nil
 	}
+	return s, v.([]RootMapping)
 
-	var filtered []RootMapping
+}
-	for _, m := range rm {
+
-		if fs.filter(m) {
+func (fs *RootMappingFs) debug() {
-			filtered = append(filtered, m)
+	fmt.Println("debug():")
-		}
+	fs.rootMapToReal.Walk(func(s string, v interface{}) bool {
-	}
+		fmt.Println("Key", s)
+		return false
+	})
 
-	return filtered
 }
 
 func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
-	if fs.isRoot(prefix) {
-		return fs.virtualRoots
-	}
-	prefix = filepath.Clean(prefix)
 	var roots []RootMapping
-
 	fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
 		roots = append(roots, v.([]RootMapping)...)
 		return false
 	})
 
-	return fs.applyFilterToRoots(roots)
+	return roots
-}
-
-// Filter out the mappings inside the name directory.
-func (fs *RootMappingFs) filterRootsBelow(roots []RootMapping, name string) []RootMapping {
-	if len(roots) == 0 {
-		return nil
-	}
-
-	sepCount := strings.Count(name, filepathSeparator)
-	var filtered []RootMapping
-	for _, x := range roots {
-		if name == x.From {
-			continue
-		}
-
-		if strings.Count(x.From, filepathSeparator)-sepCount != 1 {
-			continue
-		}
-
-		filtered = append(filtered, x)
-
-	}
-	return filtered
 }
 
 func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
@@ -400,6 +287,10 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
 	if err != nil {
 		return nil, err
 	}
+	if len(fis) == 1 {
+		return f, nil
+	}
+
 	rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta}
 	if len(fis) == 1 {
 		return rf, err
@@ -439,75 +330,215 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
 
 }
 
-func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) {
+func (fs *RootMappingFs) cleanName(name string) string {
+	return strings.Trim(filepath.Clean(name), filepathSeparator)
+}
+
+func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) {
+	prefix = filepathSeparator + fs.cleanName(prefix)
+
+	var fis []os.FileInfo
+
+	seen := make(map[string]bool) // Prevent duplicate directories
+	level := strings.Count(prefix, filepathSeparator)
+
+	// First add any real files/directories.
+	rms := fs.getRoot(prefix)
+	for _, rm := range rms {
+		f, err := rm.fi.Meta().Open()
+		if err != nil {
+			return nil, err
+		}
+		direntries, err := f.Readdir(-1)
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+
+		for _, fi := range direntries {
+			meta := fi.(FileMetaInfo).Meta()
+			mergeFileMeta(rm.Meta, meta)
+			if fi.IsDir() {
+				name := fi.Name()
+				if seen[name] {
+					continue
+				}
+				seen[name] = true
+				opener := func() (afero.File, error) {
+					return fs.Open(filepath.Join(rm.From, name))
+				}
+				fi = newDirNameOnlyFileInfo(name, meta, false, opener)
+			}
+
+			fis = append(fis, fi)
+		}
+
+		f.Close()
+	}
+
+	// Next add any file mounts inside the given directory.
+	prefixInside := prefix + filepathSeparator
+	fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool {
+
+		if (strings.Count(s, filepathSeparator) - level) != 1 {
+			// This directory is not part of the current, but we
+			// need to include the first name part to make it
+			// navigable.
+			path := strings.TrimPrefix(s, prefixInside)
+			parts := strings.Split(path, filepathSeparator)
+			name := parts[0]
+
+			if seen[name] {
+				return false
+			}
+			seen[name] = true
+			opener := func() (afero.File, error) {
+				return fs.Open(path)
+			}
+
+			fi := newDirNameOnlyFileInfo(name, nil, false, opener)
+			fis = append(fis, fi)
+
+			return false
+		}
+
+		rms := v.([]RootMapping)
+		for _, rm := range rms {
+			if !rm.fi.IsDir() {
+				// A single file mount
+				fis = append(fis, rm.fi)
+				continue
+			}
+			name := filepath.Base(rm.From)
+			if seen[name] {
+				continue
+			}
+			seen[name] = true
+
+			opener := func() (afero.File, error) {
+				return fs.Open(rm.From)
+			}
+
+			fi := newDirNameOnlyFileInfo(name, rm.Meta, false, opener)
+
+			fis = append(fis, fi)
+
+		}
+
+		return false
+	})
+
+	return fis, nil
+}
+
+func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
+	name = fs.cleanName(name)
+	key := filepathSeparator + name
+
+	roots := fs.getRoot(key)
+
+	if roots == nil {
+		if fs.hasPrefix(key) {
+			// We have directories mounted below this.
+			// Make it look like a directory.
+			return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, true, fs.virtualDirOpener(name))}, nil
+		}
+
+		// Find any real files or directories with this key.
+		_, roots := fs.getRoots(key)
+		if roots == nil {
+			return nil, os.ErrNotExist
+		}
+
+		var err error
+		var fis []FileMetaInfo
+
+		for _, rm := range roots {
+			var fi FileMetaInfo
+			fi, _, err = fs.statRoot(rm, name)
+			if err == nil {
+				fis = append(fis, fi)
+			}
+		}
+
+		if fis != nil {
+			return fis, nil
+		}
+
+		if err == nil {
+			err = os.ErrNotExist
+		}
+
+		return nil, err
+	}
+
+	fileCount := 0
+	for _, root := range roots {
+		if !root.fi.IsDir() {
+			fileCount++
+		}
+		if fileCount > 1 {
+			break
+		}
+	}
+
+	if fileCount == 0 {
+		// Dir only.
+		return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, true, fs.virtualDirOpener(name))}, nil
+	}
+
+	if fileCount > 1 {
+		// Not supported by this filesystem.
+		return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name)
+
+	}
+
+	return []FileMetaInfo{roots[0].fi}, nil
+
+}
+
+func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {
 	filename := root.filename(name)
 
-	var b bool
+	fi, b, err := lstatIfPossible(fs.Fs, filename)
-	var fi os.FileInfo
+	if err != nil {
-	var err error
+		return nil, b, err
-
-	if ls, ok := fs.Fs.(afero.Lstater); ok {
-		fi, b, err = ls.LstatIfPossible(filename)
-		if err != nil {
-			return nil, b, err
-		}
-
-	} else {
-		fi, err = fs.Fs.Stat(filename)
-		if err != nil {
-			return nil, b, err
-		}
-	}
-
-	// Opens the real directory/file.
-	opener := func() (afero.File, error) {
-		return fs.Fs.Open(filename)
 	}
 
+	var opener func() (afero.File, error)
 	if fi.IsDir() {
-		if name == "" {
+		// Make sure metadata gets applied in Readdir.
-			name = root.From
+		opener = fs.realDirOpener(filename, root.Meta)
+	} else {
+		// Opens the real file directly.
+		opener = func() (afero.File, error) {
+			return fs.Fs.Open(filename)
 		}
-		_, name = filepath.Split(name)
-		fi = newDirNameOnlyFileInfo(name, false, opener)
 	}
 
 	return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
 
 }
 
+func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) {
+	return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil }
+}
+
+func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) {
+	return func() (afero.File, error) {
+		f, err := fs.Fs.Open(name)
+		if err != nil {
+			return nil, err
+		}
+		return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil
+	}
+}
+
 type rootMappingFile struct {
 	afero.File
-	fs     *RootMappingFs
+	fs   *RootMappingFs
-	name   string
+	name string
-	meta   FileMeta
+	meta FileMeta
-	isRoot bool
-}
-
-type readDirDirsAppender struct {
-	afero.File
-	dirs []FileMetaInfo
-}
-
-func (f *readDirDirsAppender) Readdir(count int) ([]os.FileInfo, error) {
-	fis, err := f.File.Readdir(count)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, dir := range f.dirs {
-		fis = append(fis, dir)
-	}
-	return fis, nil
-
-}
-
-func (f *readDirDirsAppender) Readdirnames(count int) ([]string, error) {
-	fis, err := f.Readdir(count)
-	if err != nil {
-		return nil, err
-	}
-	return fileInfosToNames(fis), nil
 }
 
 func (f *rootMappingFile) Close() error {
@@ -522,65 +553,18 @@ func (f *rootMappingFile) Name() string {
 }
 
 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
-	if f.File == nil {
+	if f.File != nil {
-		filesn := make([]os.FileInfo, 0)
+		fis, err := f.File.Readdir(count)
-		roots := f.fs.getRootsWithPrefix(f.name)
+		if err != nil {
-		seen := make(map[string]bool) // Do not return duplicate directories
+			return nil, err
-
-		j := 0
-		for _, rm := range roots {
-			if count != -1 && j >= count {
-				break
-			}
-
-			if !rm.fi.IsDir() {
-				// A single file mount
-				filesn = append(filesn, rm.fi)
-				continue
-			}
-
-			from := rm.From
-			name := from
-			if !f.isRoot {
-				_, name = filepath.Split(from)
-			}
-
-			if seen[name] {
-				continue
-			}
-			seen[name] = true
-
-			opener := func() (afero.File, error) {
-				return f.fs.Open(from)
-			}
-
-			j++
-
-			fi := newDirNameOnlyFileInfo(name, false, opener)
-
-			if rm.Meta != nil {
-				mergeFileMeta(rm.Meta, fi.Meta())
-			}
-
-			filesn = append(filesn, fi)
 		}
-		return filesn, nil
-	}
 
-	if f.File == nil {
+		for i, fi := range fis {
-		panic(fmt.Sprintf("no File for %q", f.name))
+			fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+		}
+		return fis, nil
 	}
-
+	return f.fs.collectDirEntries(f.name)
-	fis, err := f.File.Readdir(count)
-	if err != nil {
-		return nil, err
-	}
-
-	for i, fi := range fis {
-		fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
-	}
-
-	return fis, nil
 }
 
 func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {

hugofs/rootmapping_fs_test.go

@@ -14,9 +14,10 @@
 package hugofs
 
 import (
+	"fmt"
 	"io/ioutil"
-	"os"
 	"path/filepath"
+	"sort"
 	"testing"
 
 	"github.com/spf13/viper"
@@ -34,8 +35,12 @@ func TestLanguageRootMapping(t *testing.T) {
 	fs := NewBaseFileDecorator(afero.NewMemMapFs())
 
 	c.Assert(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755), qt.IsNil)
+
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil)
+	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent/d1", "sv-d1-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil)
+	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent/d1", "en-d1-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil)
+
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755), qt.IsNil)
@@ -72,19 +77,30 @@ func TestLanguageRootMapping(t *testing.T) {
 
 	collected, err := collectFilenames(rfs, "content", "content")
 	c.Assert(err, qt.IsNil)
-	c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"})
+	c.Assert(collected, qt.DeepEquals,
-
+		[]string{"blog/d1/en-d1-f.txt", "blog/d1/sv-d1-f.txt", "blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, qt.Commentf("%#v", collected))
-	bfs := afero.NewBasePathFs(rfs, "content")
-	collected, err = collectFilenames(bfs, "", "")
-	c.Assert(err, qt.IsNil)
-	c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"})
 
 	dirs, err := rfs.Dirs(filepath.FromSlash("content/blog"))
 	c.Assert(err, qt.IsNil)
-
 	c.Assert(len(dirs), qt.Equals, 4)
+	for _, dir := range dirs {
+		f, err := dir.Meta().Open()
+		c.Assert(err, qt.IsNil)
+		f.Close()
+	}
+
+	blog, err := rfs.Open(filepath.FromSlash("content/blog"))
+	c.Assert(err, qt.IsNil)
+	fis, err := blog.Readdir(-1)
+	for _, fi := range fis {
+		f, err := fi.(FileMetaInfo).Meta().Open()
+		c.Assert(err, qt.IsNil)
+		f.Close()
+	}
+	blog.Close()
 
 	getDirnames := func(name string, rfs *RootMappingFs) []string {
+		c.Helper()
 		filename := filepath.FromSlash(name)
 		f, err := rfs.Open(filename)
 		c.Assert(err, qt.IsNil)
@@ -109,16 +125,16 @@ func TestLanguageRootMapping(t *testing.T) {
 		return rm.Meta.Lang() == "en"
 	})
 
-	c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"en-f.txt", "en-f2.txt"})
+	c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"})
 
 	rfsSv := rfs.Filter(func(rm RootMapping) bool {
 		return rm.Meta.Lang() == "sv"
 	})
 
-	c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"sv-f.txt", "svdir"})
+	c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"})
 
 	// Make sure we have not messed with the original
-	c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"})
+	c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"d1", "sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"})
 
 	c.Assert(getDirnames("content", rfsSv), qt.DeepEquals, []string{"blog", "docs"})
 	c.Assert(getDirnames("content", rfs), qt.DeepEquals, []string{"blog", "docs"})
@@ -135,7 +151,7 @@ func TestRootMappingFsDirnames(t *testing.T) {
 	c.Assert(fs.Mkdir("f3t", 0755), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755), qt.IsNil)
 
-	rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")
+	rfs, err := newRootMappingFsFromFromTo("", fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")
 	c.Assert(err, qt.IsNil)
 
 	fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))
@@ -144,12 +160,12 @@ func TestRootMappingFsDirnames(t *testing.T) {
 	fifm := fif.(FileMetaInfo).Meta()
 	c.Assert(fifm.Filename(), qt.Equals, filepath.FromSlash("f2t/myfile.txt"))
 
-	root, err := rfs.Open(filepathSeparator)
+	root, err := rfs.Open("static")
 	c.Assert(err, qt.IsNil)
 
 	dirnames, err := root.Readdirnames(-1)
 	c.Assert(err, qt.IsNil)
-	c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"})
+	c.Assert(dirnames, qt.DeepEquals, []string{"af3", "bf1", "cf2"})
 
 }
 
@@ -165,7 +181,7 @@ func TestRootMappingFsFilename(t *testing.T) {
 	c.Assert(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, testfilename, []byte("content"), 0666), qt.IsNil)
 
-	rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
+	rfs, err := newRootMappingFsFromFromTo(workDir, fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
 	c.Assert(err, qt.IsNil)
 
 	fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt"))
@@ -256,12 +272,9 @@ func TestRootMappingFsMount(t *testing.T) {
 	c.Assert(err, qt.IsNil)
 	c.Assert(string(b), qt.Equals, "some no content")
 
-	// Check file mappings
+	// Ambigous
-	single, err := rfs.Stat(filepath.FromSlash("content/singles/p1.md"))
+	_, err = rfs.Stat(filepath.FromSlash("content/singles/p1.md"))
-	c.Assert(err, qt.IsNil)
+	c.Assert(err, qt.Not(qt.IsNil))
-	c.Assert(single.IsDir(), qt.Equals, false)
-	singlem := single.(FileMetaInfo).Meta()
-	c.Assert(singlem.Lang(), qt.Equals, "no") // First match
 
 	singlesDir, err := rfs.Open(filepath.FromSlash("content/singles"))
 	c.Assert(err, qt.IsNil)
@@ -308,19 +321,20 @@ func TestRootMappingFsMountOverlap(t *testing.T) {
 	rfs, err := NewRootMappingFs(fs, rm...)
 	c.Assert(err, qt.IsNil)
 
-	getDirnames := func(name string) []string {
+	checkDirnames := func(name string, expect []string) {
+		c.Helper()
 		name = filepath.FromSlash(name)
 		f, err := rfs.Open(name)
 		c.Assert(err, qt.IsNil)
 		defer f.Close()
 		names, err := f.Readdirnames(-1)
 		c.Assert(err, qt.IsNil)
-		return names
+		c.Assert(names, qt.DeepEquals, expect, qt.Commentf(fmt.Sprintf("%#v", names)))
 	}
 
-	c.Assert(getDirnames("static"), qt.DeepEquals, []string{"a.txt", "b", "e"})
+	checkDirnames("static", []string{"a.txt", "b", "e"})
-	c.Assert(getDirnames("static/b"), qt.DeepEquals, []string{"b.txt", "c"})
+	checkDirnames("static/b", []string{"b.txt", "c"})
-	c.Assert(getDirnames("static/b/c"), qt.DeepEquals, []string{"c.txt"})
+	checkDirnames("static/b/c", []string{"c.txt"})
 
 	fi, err := rfs.Stat(filepath.FromSlash("static/b/b.txt"))
 	c.Assert(err, qt.IsNil)
@@ -330,32 +344,96 @@ func TestRootMappingFsMountOverlap(t *testing.T) {
 
 func TestRootMappingFsOs(t *testing.T) {
 	c := qt.New(t)
-	fs := afero.NewOsFs()
+	fs := NewBaseFileDecorator(afero.NewOsFs())
 
-	d, err := ioutil.TempDir("", "hugo-root-mapping")
+	d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os")
 	c.Assert(err, qt.IsNil)
-	defer func() {
+	defer clean()
-		os.RemoveAll(d)
-	}()
 
 	testfile := "myfile.txt"
 	c.Assert(fs.Mkdir(filepath.Join(d, "f1t"), 0755), qt.IsNil)
 	c.Assert(fs.Mkdir(filepath.Join(d, "f2t"), 0755), qt.IsNil)
 	c.Assert(fs.Mkdir(filepath.Join(d, "f3t"), 0755), qt.IsNil)
+
+	// Deep structure
+	deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5")
+	c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil)
+	for i := 1; i <= 3; i++ {
+		c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil)
+		c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil)
+	}
+
 	c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil)
 
-	rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t"))
+	rfs, err := newRootMappingFsFromFromTo(
+		d,
+		fs,
+		"static/bf1", filepath.Join(d, "f1t"),
+		"static/cf2", filepath.Join(d, "f2t"),
+		"static/af3", filepath.Join(d, "f3t"),
+		"static/a/b/c", filepath.Join(d, "d1", "d2", "d3"),
+		"layouts", filepath.Join(d, "d1"),
+	)
+
 	c.Assert(err, qt.IsNil)
 
 	fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))
 	c.Assert(err, qt.IsNil)
 	c.Assert(fif.Name(), qt.Equals, "myfile.txt")
 
-	root, err := rfs.Open(filepathSeparator)
+	root, err := rfs.Open("static")
 	c.Assert(err, qt.IsNil)
 
 	dirnames, err := root.Readdirnames(-1)
 	c.Assert(err, qt.IsNil)
-	c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"})
+	c.Assert(dirnames, qt.DeepEquals, []string{"a", "af3", "bf1", "cf2"}, qt.Commentf(fmt.Sprintf("%#v", dirnames)))
 
+	getDirnames := func(dirname string) []string {
+		dirname = filepath.FromSlash(dirname)
+		f, err := rfs.Open(dirname)
+		c.Assert(err, qt.IsNil)
+		defer f.Close()
+		dirnames, err := f.Readdirnames(-1)
+		c.Assert(err, qt.IsNil)
+		sort.Strings(dirnames)
+		return dirnames
+	}
+
+	c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"})
+	c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"})
+	c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"})
+
+	all, err := collectFilenames(rfs, "static", "static")
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"})
+
+	fis, err := collectFileinfos(rfs, "static", "static")
+	c.Assert(err, qt.IsNil)
+
+	c.Assert(fis[9].Meta().PathFile(), qt.Equals, filepath.FromSlash("d1/d2/d3/f-1.txt"))
+
+	dirc := fis[3].Meta()
+
+	f, err := dirc.Open()
+	c.Assert(err, qt.IsNil)
+	defer f.Close()
+	fileInfos, err := f.Readdir(-1)
+	c.Assert(err, qt.IsNil)
+	sortFileInfos(fileInfos)
+	i := 0
+	for _, fi := range fileInfos {
+		if fi.IsDir() {
+			continue
+		}
+		i++
+		meta := fi.(FileMetaInfo).Meta()
+		c.Assert(meta.Filename(), qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i)))
+		c.Assert(meta.PathFile(), qt.Equals, filepath.FromSlash(fmt.Sprintf("d1/d2/d3/f-%d.txt", i)))
+	}
+
+	_, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3/f-1.txt"))
+	c.Assert(err, qt.IsNil)
+	_, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3"))
+	c.Assert(err, qt.IsNil)
 }

hugofs/walk.go

@@ -124,7 +124,6 @@ func (w *Walkway) Walk() error {
 			if w.checkErr(w.root, err) {
 				return nil
 			}
-
 			return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root))
 		}
 		fi = info.(FileMetaInfo)
@@ -154,6 +153,15 @@ func (w *Walkway) checkErr(filename string, err error) bool {
 		logUnsupportedSymlink(filename, w.logger)
 		return true
 	}
+
+	if os.IsNotExist(err) {
+		// The file may be removed in process.
+		// This may be a ERROR situation, but it is not possible
+		// to determine as a general case.
+		w.logger.WARN.Printf("File %q not found, skipping.", filename)
+		return true
+	}
+
 	return false
 }
 

hugofs/walk_test.go

@@ -176,6 +176,27 @@ func collectFilenames(fs afero.Fs, base, root string) ([]string, error) {
 
 }
 
+func collectFileinfos(fs afero.Fs, base, root string) ([]FileMetaInfo, error) {
+	var fis []FileMetaInfo
+
+	walkFn := func(path string, info FileMetaInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		fis = append(fis, info)
+
+		return nil
+	}
+
+	w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn})
+
+	err := w.Walk()
+
+	return fis, err
+
+}
+
 func BenchmarkWalk(b *testing.B) {
 	c := qt.New(b)
 	fs := NewBaseFileDecorator(afero.NewMemMapFs())

hugolib/filesystems/basefs.go

@@ -258,6 +258,7 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
 // MakePathRelative creates a relative path from the given filename.
 // It will return an empty string if the filename is not a member of this filesystem.
 func (d *SourceFilesystem) MakePathRelative(filename string) string {
+
 	for _, dir := range d.Dirs {
 		meta := dir.(hugofs.FileMetaInfo).Meta()
 		currentPath := meta.Filename()

hugolib/filesystems/basefs_test.go

@@ -173,9 +173,7 @@ theme = ["atheme"]
 			filename = filepath.FromSlash(filename)
 			f, err := fs.Open(filename)
 			c.Assert(err, qt.IsNil)
-			name := f.Name()
 			f.Close()
-			c.Assert(name, qt.Equals, filename)
 		}
 	}
 }

hugolib/hugo_modules_test.go

@@ -38,6 +38,47 @@ import (
 	"github.com/spf13/viper"
 )
 
+// https://github.com/gohugoio/hugo/issues/6730
+func TestHugoModulesTargetInSubFolder(t *testing.T) {
+	config := `
+baseURL="https://example.org"
+workingDir = %q
+
+[module]
+[[module.imports]]
+path="github.com/gohugoio/hugoTestModule2"
+  [[module.imports.mounts]]
+    source = "templates/hooks"
+    target = "layouts/_default/_markup"
+    
+`
+
+	b := newTestSitesBuilder(t)
+	workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-target-in-subfolder-test")
+	b.Assert(err, qt.IsNil)
+	defer clean()
+	b.Fs = hugofs.NewDefault(viper.New())
+	b.WithWorkingDir(workingDir).WithConfigFile("toml", fmt.Sprintf(config, workingDir))
+	b.WithTemplates("_default/single.html", `{{ .Content }}`)
+	b.WithContent("p1.md", `---
+title: "Page"
+---
+
+[A link](https://bep.is)
+
+`)
+	b.WithSourceFile("go.mod", `
+module github.com/gohugoio/tests/testHugoModules
+
+
+`)
+
+	b.Build(BuildCfg{})
+
+	b.AssertFileContent("public/p1/index.html", `<p>Page|https://bep.is|Title: |Text: A link|END</p>`)
+
+}
+
 // TODO(bep) this fails when testmodBuilder is also building ...
 func TestHugoModules(t *testing.T) {
 	if !isCI() {
@@ -588,6 +629,9 @@ workingDir = %q
 
 {{ $mypage := .Site.GetPage "/blog/mypage.md" }}
 {{ with $mypage }}MYPAGE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}
+{{ $mybundle := .Site.GetPage "/blog/mybundle" }}
+{{ with $mybundle }}MYBUNDLE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}
+
 
 `, "_default/_markup/render-link.html", `
 {{ $link := .Destination }}
@@ -640,6 +684,7 @@ README: Readme Title
 /README.md|Path: _index.md|FilePath: README.md
 Readme Content.
 MYPAGE: My Page|Path: blog/mypage.md|FilePath: mycontent/mypage.md|
+MYBUNDLE: My Bundle|Path: blog/mybundle/index.md|FilePath: mycontent/mybundle/index.md|
 `)
 	b.AssertFileContent("public/blog/mypage/index.html", `
 <a href="https://example.com/blog/mybundle/">Relative Link From Page</a>