From 80dd6ddde27ce36f5432fb780e94d4974b5277c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Fri, 31 Jan 2020 17:15:14 +0100 Subject: [PATCH] Fix module mount in sub folder This addresses a specific issue, but is a also a major simplification of the filesystem file mounts. Fixes #6730 --- hugofs/decorators.go | 3 +- hugofs/fileinfo.go | 34 +- hugofs/nosymlink_test.go | 1 + hugofs/rootmapping_fs.go | 598 ++++++++++++++--------------- hugofs/rootmapping_fs_test.go | 146 +++++-- hugofs/walk.go | 10 +- hugofs/walk_test.go | 21 + hugolib/filesystems/basefs.go | 1 + hugolib/filesystems/basefs_test.go | 2 - hugolib/hugo_modules_test.go | 45 +++ 10 files changed, 510 insertions(+), 351 deletions(-) diff --git a/hugofs/decorators.go b/hugofs/decorators.go index e93f53aab..e1e3b9b51 100644 --- a/hugofs/decorators.go +++ b/hugofs/decorators.go @@ -79,7 +79,7 @@ func DecorateBasePathFs(base *afero.BasePathFs) afero.Fs { } // NewBaseFileDecorator decorates the given Fs to provide the real filename -// and an Opener func. If +// and an Opener func. func NewBaseFileDecorator(fs afero.Fs) afero.Fs { ffs := &baseFileDecoratorFs{Fs: fs} @@ -102,7 +102,6 @@ func NewBaseFileDecorator(fs afero.Fs) afero.Fs { opener := func() (afero.File, error) { return ffs.open(filename) - } return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil diff --git a/hugofs/fileinfo.go b/hugofs/fileinfo.go index c8a71bf21..0f20ec386 100644 --- a/hugofs/fileinfo.go +++ b/hugofs/fileinfo.go @@ -18,6 +18,7 @@ import ( "os" "path/filepath" "runtime" + "sort" "strings" "time" @@ -271,13 +272,21 @@ func (fi *dirNameOnlyFileInfo) Sys() interface{} { return nil } -func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo { +func newDirNameOnlyFileInfo(name string, meta FileMeta, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo { name = normalizeFilename(name) _, base := filepath.Split(name) - return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{ - metaKeyFilename: name, - metaKeyIsOrdered: isOrdered, - metaKeyOpener: fileOpener}) + + m := copyFileMeta(meta) + if _, found := m[metaKeyFilename]; !found { + m.setIfNotZero(metaKeyFilename, name) + } + m[metaKeyOpener] = fileOpener + m[metaKeyIsOrdered] = isOrdered + + return NewFileMetaInfo( + &dirNameOnlyFileInfo{name: base}, + m, + ) } func decorateFileInfo( @@ -339,3 +348,18 @@ func fileInfosToNames(fis []os.FileInfo) []string { } return names } + +func fromSlash(filenames []string) []string { + for i, name := range filenames { + filenames[i] = filepath.FromSlash(name) + } + return filenames +} + +func sortFileInfos(fis []os.FileInfo) { + sort.Slice(fis, func(i, j int) bool { + fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo) + return fimi.Meta().Filename() < fimj.Meta().Filename() + + }) +} diff --git a/hugofs/nosymlink_test.go b/hugofs/nosymlink_test.go index b3b364789..c938da006 100644 --- a/hugofs/nosymlink_test.go +++ b/hugofs/nosymlink_test.go @@ -137,6 +137,7 @@ func TestNoSymlinkFs(t *testing.T) { c.Assert(err, qt.IsNil) // There is at least one unsported symlink inside workDir _, err = f.Readdir(-1) + c.Assert(err, qt.IsNil) f.Close() c.Assert(logger.WarnCounter.Count(), qt.Equals, uint64(1)) diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go index 2196be8e0..ea8b7e04d 100644 --- a/hugofs/rootmapping_fs.go +++ b/hugofs/rootmapping_fs.go @@ -27,15 +27,18 @@ import ( "github.com/spf13/afero" ) -var filepathSeparator = string(filepath.Separator) +var ( + filepathSeparator = string(filepath.Separator) +) // NewRootMappingFs creates a new RootMappingFs on top of the provided with -// of root mappings with some optional metadata about the root. +// root mappings with some optional metadata about the root. // Note that From represents a virtual root that maps to the actual filename in To. func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rootMapToReal := radix.New() + var virtualRoots []RootMapping - for i, rm := range rms { + for _, rm := range rms { (&rm).clean() fromBase := files.ResolveComponentFolder(rm.From) @@ -56,11 +59,13 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { } // Extract "blog" from "content/blog" rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator) - if rm.Meta != nil { - rm.Meta[metaKeyBaseDir] = rm.ToBasedir - rm.Meta[metaKeyMountRoot] = rm.path + if rm.Meta == nil { + rm.Meta = make(FileMeta) } + rm.Meta[metaKeyBaseDir] = rm.ToBasedir + rm.Meta[metaKeyMountRoot] = rm.path + meta := copyFileMeta(rm.Meta) if !fi.IsDir() { @@ -70,7 +75,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { rm.fi = NewFileMetaInfo(fi, meta) - key := rm.rootKey() + key := filepathSeparator + rm.From var mappings []RootMapping v, found := rootMapToReal.Get(key) if found { @@ -80,30 +85,38 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) { mappings = append(mappings, rm) rootMapToReal.Insert(key, mappings) - rms[i] = rm + virtualRoots = append(virtualRoots, rm) } - rfs := &RootMappingFs{Fs: fs, - virtualRoots: rms, - rootMapToReal: rootMapToReal} + rootMapToReal.Insert(filepathSeparator, virtualRoots) + + rfs := &RootMappingFs{ + Fs: fs, + rootMapToReal: rootMapToReal, + } return rfs, nil } -// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking -// From and To as string pairs. -func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) { +func newRootMappingFsFromFromTo( + baseDir string, + fs afero.Fs, + fromTo ...string, +) (*RootMappingFs, error) { + rms := make([]RootMapping, len(fromTo)/2) for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { rms[i] = RootMapping{ - From: fromTo[j], - To: fromTo[j+1], + From: fromTo[j], + To: fromTo[j+1], + ToBasedir: baseDir, } } return NewRootMappingFs(fs, rms...) } +// RootMapping describes a virtual file or directory mount. type RootMapping struct { From string // The virtual mount. To string // The source directory or file. @@ -127,21 +140,16 @@ func (r RootMapping) filename(name string) string { return filepath.Join(r.To, strings.TrimPrefix(name, r.From)) } -func (r RootMapping) rootKey() string { - return r.From -} - // A RootMappingFs maps several roots into one. Note that the root of this filesystem // is directories only, and they will be returned in Readdir and Readdirnames // in the order given. type RootMappingFs struct { afero.Fs rootMapToReal *radix.Tree - virtualRoots []RootMapping - filter func(r RootMapping) bool } func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { + base = filepathSeparator + fs.cleanName(base) roots := fs.getRootsWithPrefix(base) if roots == nil { @@ -176,138 +184,46 @@ func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) { return fss, nil } +// Filter creates a copy of this filesystem with only mappings matching a filter. +func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { + rootMapToReal := radix.New() + fs.rootMapToReal.Walk(func(b string, v interface{}) bool { + rms := v.([]RootMapping) + var nrms []RootMapping + for _, rm := range rms { + if f(rm) { + nrms = append(nrms, rm) + } + } + if len(nrms) != 0 { + rootMapToReal.Insert(b, nrms) + } + return false + }) + + fs.rootMapToReal = rootMapToReal + + return &fs +} + // LstatIfPossible returns the os.FileInfo structure describing a given file. func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) { - fis, _, b, err := fs.doLstat(name, false) + fis, err := fs.doLstat(name) if err != nil { - return nil, b, err + return nil, false, err } - return fis[0], b, nil + return fis[0], false, nil } -func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) { - return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil } -} - -func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) { - if fs.isRoot(name) { - return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil - } - - roots := fs.getRoots(name) - rootsWithPrefix := fs.getRootsWithPrefix(name) - hasRootMappingsBelow := len(rootsWithPrefix) != 0 - - if len(roots) == 0 { - if hasRootMappingsBelow { - // No exact matches, but we have root mappings below name, - // let's make it look like a directory. - return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, nil, false, nil - } - - return nil, nil, false, os.ErrNotExist - } - - // We may have a mapping for both static and static/subdir. - // These will not show in any Readdir so append them - // manually. - rootsInDir := fs.filterRootsBelow(rootsWithPrefix, name) - - var ( - fis []FileMetaInfo - dirs []FileMetaInfo - b bool - root RootMapping - err error - ) - - for _, root = range roots { - var fi os.FileInfo - fi, b, err = fs.statRoot(root, name) - if err != nil { - if os.IsNotExist(err) { - continue - } - return nil, nil, false, err - } - fim := fi.(FileMetaInfo) - - fis = append(fis, fim) - } - - for _, root = range rootsInDir { - - fi, _, err := fs.statRoot(root, "") - if err != nil { - if os.IsNotExist(err) { - continue - } - return nil, nil, false, err - } - fim := fi.(FileMetaInfo) - dirs = append(dirs, fim) - } - - if len(fis) == 0 && len(dirs) == 0 { - return nil, nil, false, os.ErrNotExist - } - - if allowMultiple || len(fis) == 1 { - return fis, dirs, b, nil - } - - if len(fis) == 0 { - return nil, nil, false, os.ErrNotExist - } - - // Open it in this composite filesystem. - opener := func() (afero.File, error) { - return fs.Open(name) - } - - return []FileMetaInfo{decorateFileInfo(fis[0], fs, opener, "", "", root.Meta)}, nil, b, nil - -} - -// Open opens the namedrootMappingFile file for reading. +// Open opens the named file for reading. func (fs *RootMappingFs) Open(name string) (afero.File, error) { - if fs.isRoot(name) { - return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil - } + fis, err := fs.doLstat(name) - fis, dirs, _, err := fs.doLstat(name, true) if err != nil { return nil, err } - if len(fis) == 1 { - fi := fis[0] - meta := fi.(FileMetaInfo).Meta() - f, err := meta.Open() - if err != nil { - return nil, err - } - - f = &rootMappingFile{File: f, fs: fs, name: name, meta: meta} - - if len(dirs) > 0 { - return &readDirDirsAppender{File: f, dirs: dirs}, nil - } - - return f, nil - } - - f, err := fs.newUnionFile(fis...) - if err != nil { - return nil, err - } - - if len(dirs) > 0 { - return &readDirDirsAppender{File: f, dirs: dirs}, nil - } - - return f, nil - + return fs.newUnionFile(fis...) } // Stat returns the os.FileInfo structure describing a given file. If there is @@ -318,80 +234,51 @@ func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) { } -// Filter creates a copy of this filesystem with the applied filter. -func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs { - fs.filter = f - return &fs +func (fs *RootMappingFs) hasPrefix(prefix string) bool { + hasPrefix := false + fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool { + hasPrefix = true + return true + }) + + return hasPrefix } -func (fs *RootMappingFs) isRoot(name string) bool { - return name == "" || name == filepathSeparator - -} - -func (fs *RootMappingFs) getRoots(name string) []RootMapping { - name = filepath.Clean(name) - _, v, found := fs.rootMapToReal.LongestPrefix(name) +func (fs *RootMappingFs) getRoot(key string) []RootMapping { + v, found := fs.rootMapToReal.Get(key) if !found { return nil } - rm := v.([]RootMapping) - - return fs.applyFilterToRoots(rm) + return v.([]RootMapping) } -func (fs *RootMappingFs) applyFilterToRoots(rm []RootMapping) []RootMapping { - if fs.filter == nil { - return rm +func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) { + s, v, found := fs.rootMapToReal.LongestPrefix(key) + if !found || (s == filepathSeparator && key != filepathSeparator) { + return "", nil } + return s, v.([]RootMapping) - var filtered []RootMapping - for _, m := range rm { - if fs.filter(m) { - filtered = append(filtered, m) - } - } +} + +func (fs *RootMappingFs) debug() { + fmt.Println("debug():") + fs.rootMapToReal.Walk(func(s string, v interface{}) bool { + fmt.Println("Key", s) + return false + }) - return filtered } func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping { - if fs.isRoot(prefix) { - return fs.virtualRoots - } - prefix = filepath.Clean(prefix) var roots []RootMapping - fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool { roots = append(roots, v.([]RootMapping)...) return false }) - return fs.applyFilterToRoots(roots) -} - -// Filter out the mappings inside the name directory. -func (fs *RootMappingFs) filterRootsBelow(roots []RootMapping, name string) []RootMapping { - if len(roots) == 0 { - return nil - } - - sepCount := strings.Count(name, filepathSeparator) - var filtered []RootMapping - for _, x := range roots { - if name == x.From { - continue - } - - if strings.Count(x.From, filepathSeparator)-sepCount != 1 { - continue - } - - filtered = append(filtered, x) - - } - return filtered + return roots } func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { @@ -400,6 +287,10 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { if err != nil { return nil, err } + if len(fis) == 1 { + return f, nil + } + rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta} if len(fis) == 1 { return rf, err @@ -439,75 +330,215 @@ func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) { } -func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) { +func (fs *RootMappingFs) cleanName(name string) string { + return strings.Trim(filepath.Clean(name), filepathSeparator) +} + +func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) { + prefix = filepathSeparator + fs.cleanName(prefix) + + var fis []os.FileInfo + + seen := make(map[string]bool) // Prevent duplicate directories + level := strings.Count(prefix, filepathSeparator) + + // First add any real files/directories. + rms := fs.getRoot(prefix) + for _, rm := range rms { + f, err := rm.fi.Meta().Open() + if err != nil { + return nil, err + } + direntries, err := f.Readdir(-1) + if err != nil { + f.Close() + return nil, err + } + + for _, fi := range direntries { + meta := fi.(FileMetaInfo).Meta() + mergeFileMeta(rm.Meta, meta) + if fi.IsDir() { + name := fi.Name() + if seen[name] { + continue + } + seen[name] = true + opener := func() (afero.File, error) { + return fs.Open(filepath.Join(rm.From, name)) + } + fi = newDirNameOnlyFileInfo(name, meta, false, opener) + } + + fis = append(fis, fi) + } + + f.Close() + } + + // Next add any file mounts inside the given directory. + prefixInside := prefix + filepathSeparator + fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool { + + if (strings.Count(s, filepathSeparator) - level) != 1 { + // This directory is not part of the current, but we + // need to include the first name part to make it + // navigable. + path := strings.TrimPrefix(s, prefixInside) + parts := strings.Split(path, filepathSeparator) + name := parts[0] + + if seen[name] { + return false + } + seen[name] = true + opener := func() (afero.File, error) { + return fs.Open(path) + } + + fi := newDirNameOnlyFileInfo(name, nil, false, opener) + fis = append(fis, fi) + + return false + } + + rms := v.([]RootMapping) + for _, rm := range rms { + if !rm.fi.IsDir() { + // A single file mount + fis = append(fis, rm.fi) + continue + } + name := filepath.Base(rm.From) + if seen[name] { + continue + } + seen[name] = true + + opener := func() (afero.File, error) { + return fs.Open(rm.From) + } + + fi := newDirNameOnlyFileInfo(name, rm.Meta, false, opener) + + fis = append(fis, fi) + + } + + return false + }) + + return fis, nil +} + +func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) { + name = fs.cleanName(name) + key := filepathSeparator + name + + roots := fs.getRoot(key) + + if roots == nil { + if fs.hasPrefix(key) { + // We have directories mounted below this. + // Make it look like a directory. + return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, true, fs.virtualDirOpener(name))}, nil + } + + // Find any real files or directories with this key. + _, roots := fs.getRoots(key) + if roots == nil { + return nil, os.ErrNotExist + } + + var err error + var fis []FileMetaInfo + + for _, rm := range roots { + var fi FileMetaInfo + fi, _, err = fs.statRoot(rm, name) + if err == nil { + fis = append(fis, fi) + } + } + + if fis != nil { + return fis, nil + } + + if err == nil { + err = os.ErrNotExist + } + + return nil, err + } + + fileCount := 0 + for _, root := range roots { + if !root.fi.IsDir() { + fileCount++ + } + if fileCount > 1 { + break + } + } + + if fileCount == 0 { + // Dir only. + return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, true, fs.virtualDirOpener(name))}, nil + } + + if fileCount > 1 { + // Not supported by this filesystem. + return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name) + + } + + return []FileMetaInfo{roots[0].fi}, nil + +} + +func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) { filename := root.filename(name) - var b bool - var fi os.FileInfo - var err error - - if ls, ok := fs.Fs.(afero.Lstater); ok { - fi, b, err = ls.LstatIfPossible(filename) - if err != nil { - return nil, b, err - } - - } else { - fi, err = fs.Fs.Stat(filename) - if err != nil { - return nil, b, err - } - } - - // Opens the real directory/file. - opener := func() (afero.File, error) { - return fs.Fs.Open(filename) + fi, b, err := lstatIfPossible(fs.Fs, filename) + if err != nil { + return nil, b, err } + var opener func() (afero.File, error) if fi.IsDir() { - if name == "" { - name = root.From + // Make sure metadata gets applied in Readdir. + opener = fs.realDirOpener(filename, root.Meta) + } else { + // Opens the real file directly. + opener = func() (afero.File, error) { + return fs.Fs.Open(filename) } - _, name = filepath.Split(name) - fi = newDirNameOnlyFileInfo(name, false, opener) } return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil } +func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) { + return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil } +} + +func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) { + return func() (afero.File, error) { + f, err := fs.Fs.Open(name) + if err != nil { + return nil, err + } + return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil + } +} + type rootMappingFile struct { afero.File - fs *RootMappingFs - name string - meta FileMeta - isRoot bool -} - -type readDirDirsAppender struct { - afero.File - dirs []FileMetaInfo -} - -func (f *readDirDirsAppender) Readdir(count int) ([]os.FileInfo, error) { - fis, err := f.File.Readdir(count) - if err != nil { - return nil, err - } - - for _, dir := range f.dirs { - fis = append(fis, dir) - } - return fis, nil - -} - -func (f *readDirDirsAppender) Readdirnames(count int) ([]string, error) { - fis, err := f.Readdir(count) - if err != nil { - return nil, err - } - return fileInfosToNames(fis), nil + fs *RootMappingFs + name string + meta FileMeta } func (f *rootMappingFile) Close() error { @@ -522,65 +553,18 @@ func (f *rootMappingFile) Name() string { } func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) { - if f.File == nil { - filesn := make([]os.FileInfo, 0) - roots := f.fs.getRootsWithPrefix(f.name) - seen := make(map[string]bool) // Do not return duplicate directories - - j := 0 - for _, rm := range roots { - if count != -1 && j >= count { - break - } - - if !rm.fi.IsDir() { - // A single file mount - filesn = append(filesn, rm.fi) - continue - } - - from := rm.From - name := from - if !f.isRoot { - _, name = filepath.Split(from) - } - - if seen[name] { - continue - } - seen[name] = true - - opener := func() (afero.File, error) { - return f.fs.Open(from) - } - - j++ - - fi := newDirNameOnlyFileInfo(name, false, opener) - - if rm.Meta != nil { - mergeFileMeta(rm.Meta, fi.Meta()) - } - - filesn = append(filesn, fi) + if f.File != nil { + fis, err := f.File.Readdir(count) + if err != nil { + return nil, err } - return filesn, nil - } - if f.File == nil { - panic(fmt.Sprintf("no File for %q", f.name)) + for i, fi := range fis { + fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) + } + return fis, nil } - - fis, err := f.File.Readdir(count) - if err != nil { - return nil, err - } - - for i, fi := range fis { - fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta) - } - - return fis, nil + return f.fs.collectDirEntries(f.name) } func (f *rootMappingFile) Readdirnames(count int) ([]string, error) { diff --git a/hugofs/rootmapping_fs_test.go b/hugofs/rootmapping_fs_test.go index f7637a61f..44b957f18 100644 --- a/hugofs/rootmapping_fs_test.go +++ b/hugofs/rootmapping_fs_test.go @@ -14,9 +14,10 @@ package hugofs import ( + "fmt" "io/ioutil" - "os" "path/filepath" + "sort" "testing" "github.com/spf13/viper" @@ -34,8 +35,12 @@ func TestLanguageRootMapping(t *testing.T) { fs := NewBaseFileDecorator(afero.NewMemMapFs()) c.Assert(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent/d1", "sv-d1-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent/d1", "en-d1-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755), qt.IsNil) @@ -72,19 +77,30 @@ func TestLanguageRootMapping(t *testing.T) { collected, err := collectFilenames(rfs, "content", "content") c.Assert(err, qt.IsNil) - c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}) - - bfs := afero.NewBasePathFs(rfs, "content") - collected, err = collectFilenames(bfs, "", "") - c.Assert(err, qt.IsNil) - c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}) + c.Assert(collected, qt.DeepEquals, + []string{"blog/d1/en-d1-f.txt", "blog/d1/sv-d1-f.txt", "blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, qt.Commentf("%#v", collected)) dirs, err := rfs.Dirs(filepath.FromSlash("content/blog")) c.Assert(err, qt.IsNil) - c.Assert(len(dirs), qt.Equals, 4) + for _, dir := range dirs { + f, err := dir.Meta().Open() + c.Assert(err, qt.IsNil) + f.Close() + } + + blog, err := rfs.Open(filepath.FromSlash("content/blog")) + c.Assert(err, qt.IsNil) + fis, err := blog.Readdir(-1) + for _, fi := range fis { + f, err := fi.(FileMetaInfo).Meta().Open() + c.Assert(err, qt.IsNil) + f.Close() + } + blog.Close() getDirnames := func(name string, rfs *RootMappingFs) []string { + c.Helper() filename := filepath.FromSlash(name) f, err := rfs.Open(filename) c.Assert(err, qt.IsNil) @@ -109,16 +125,16 @@ func TestLanguageRootMapping(t *testing.T) { return rm.Meta.Lang() == "en" }) - c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"en-f.txt", "en-f2.txt"}) + c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"}) rfsSv := rfs.Filter(func(rm RootMapping) bool { return rm.Meta.Lang() == "sv" }) - c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"sv-f.txt", "svdir"}) + c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"}) // Make sure we have not messed with the original - c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}) + c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"d1", "sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}) c.Assert(getDirnames("content", rfsSv), qt.DeepEquals, []string{"blog", "docs"}) c.Assert(getDirnames("content", rfs), qt.DeepEquals, []string{"blog", "docs"}) @@ -135,7 +151,7 @@ func TestRootMappingFsDirnames(t *testing.T) { c.Assert(fs.Mkdir("f3t", 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755), qt.IsNil) - rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t") + rfs, err := newRootMappingFsFromFromTo("", fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t") c.Assert(err, qt.IsNil) fif, err := rfs.Stat(filepath.Join("static/cf2", testfile)) @@ -144,12 +160,12 @@ func TestRootMappingFsDirnames(t *testing.T) { fifm := fif.(FileMetaInfo).Meta() c.Assert(fifm.Filename(), qt.Equals, filepath.FromSlash("f2t/myfile.txt")) - root, err := rfs.Open(filepathSeparator) + root, err := rfs.Open("static") c.Assert(err, qt.IsNil) dirnames, err := root.Readdirnames(-1) c.Assert(err, qt.IsNil) - c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"}) + c.Assert(dirnames, qt.DeepEquals, []string{"af3", "bf1", "cf2"}) } @@ -165,7 +181,7 @@ func TestRootMappingFsFilename(t *testing.T) { c.Assert(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777), qt.IsNil) c.Assert(afero.WriteFile(fs, testfilename, []byte("content"), 0666), qt.IsNil) - rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t")) + rfs, err := newRootMappingFsFromFromTo(workDir, fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t")) c.Assert(err, qt.IsNil) fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt")) @@ -256,12 +272,9 @@ func TestRootMappingFsMount(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(string(b), qt.Equals, "some no content") - // Check file mappings - single, err := rfs.Stat(filepath.FromSlash("content/singles/p1.md")) - c.Assert(err, qt.IsNil) - c.Assert(single.IsDir(), qt.Equals, false) - singlem := single.(FileMetaInfo).Meta() - c.Assert(singlem.Lang(), qt.Equals, "no") // First match + // Ambigous + _, err = rfs.Stat(filepath.FromSlash("content/singles/p1.md")) + c.Assert(err, qt.Not(qt.IsNil)) singlesDir, err := rfs.Open(filepath.FromSlash("content/singles")) c.Assert(err, qt.IsNil) @@ -308,19 +321,20 @@ func TestRootMappingFsMountOverlap(t *testing.T) { rfs, err := NewRootMappingFs(fs, rm...) c.Assert(err, qt.IsNil) - getDirnames := func(name string) []string { + checkDirnames := func(name string, expect []string) { + c.Helper() name = filepath.FromSlash(name) f, err := rfs.Open(name) c.Assert(err, qt.IsNil) defer f.Close() names, err := f.Readdirnames(-1) c.Assert(err, qt.IsNil) - return names + c.Assert(names, qt.DeepEquals, expect, qt.Commentf(fmt.Sprintf("%#v", names))) } - c.Assert(getDirnames("static"), qt.DeepEquals, []string{"a.txt", "b", "e"}) - c.Assert(getDirnames("static/b"), qt.DeepEquals, []string{"b.txt", "c"}) - c.Assert(getDirnames("static/b/c"), qt.DeepEquals, []string{"c.txt"}) + checkDirnames("static", []string{"a.txt", "b", "e"}) + checkDirnames("static/b", []string{"b.txt", "c"}) + checkDirnames("static/b/c", []string{"c.txt"}) fi, err := rfs.Stat(filepath.FromSlash("static/b/b.txt")) c.Assert(err, qt.IsNil) @@ -330,32 +344,96 @@ func TestRootMappingFsMountOverlap(t *testing.T) { func TestRootMappingFsOs(t *testing.T) { c := qt.New(t) - fs := afero.NewOsFs() + fs := NewBaseFileDecorator(afero.NewOsFs()) - d, err := ioutil.TempDir("", "hugo-root-mapping") + d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os") c.Assert(err, qt.IsNil) - defer func() { - os.RemoveAll(d) - }() + defer clean() testfile := "myfile.txt" c.Assert(fs.Mkdir(filepath.Join(d, "f1t"), 0755), qt.IsNil) c.Assert(fs.Mkdir(filepath.Join(d, "f2t"), 0755), qt.IsNil) c.Assert(fs.Mkdir(filepath.Join(d, "f3t"), 0755), qt.IsNil) + + // Deep structure + deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5") + c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil) + for i := 1; i <= 3; i++ { + c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil) + c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil) + } + c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil) - rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t")) + rfs, err := newRootMappingFsFromFromTo( + d, + fs, + "static/bf1", filepath.Join(d, "f1t"), + "static/cf2", filepath.Join(d, "f2t"), + "static/af3", filepath.Join(d, "f3t"), + "static/a/b/c", filepath.Join(d, "d1", "d2", "d3"), + "layouts", filepath.Join(d, "d1"), + ) + c.Assert(err, qt.IsNil) fif, err := rfs.Stat(filepath.Join("static/cf2", testfile)) c.Assert(err, qt.IsNil) c.Assert(fif.Name(), qt.Equals, "myfile.txt") - root, err := rfs.Open(filepathSeparator) + root, err := rfs.Open("static") c.Assert(err, qt.IsNil) dirnames, err := root.Readdirnames(-1) c.Assert(err, qt.IsNil) - c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"}) + c.Assert(dirnames, qt.DeepEquals, []string{"a", "af3", "bf1", "cf2"}, qt.Commentf(fmt.Sprintf("%#v", dirnames))) + getDirnames := func(dirname string) []string { + dirname = filepath.FromSlash(dirname) + f, err := rfs.Open(dirname) + c.Assert(err, qt.IsNil) + defer f.Close() + dirnames, err := f.Readdirnames(-1) + c.Assert(err, qt.IsNil) + sort.Strings(dirnames) + return dirnames + } + + c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"}) + c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"}) + c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"}) + + all, err := collectFilenames(rfs, "static", "static") + c.Assert(err, qt.IsNil) + + c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"}) + + fis, err := collectFileinfos(rfs, "static", "static") + c.Assert(err, qt.IsNil) + + c.Assert(fis[9].Meta().PathFile(), qt.Equals, filepath.FromSlash("d1/d2/d3/f-1.txt")) + + dirc := fis[3].Meta() + + f, err := dirc.Open() + c.Assert(err, qt.IsNil) + defer f.Close() + fileInfos, err := f.Readdir(-1) + c.Assert(err, qt.IsNil) + sortFileInfos(fileInfos) + i := 0 + for _, fi := range fileInfos { + if fi.IsDir() { + continue + } + i++ + meta := fi.(FileMetaInfo).Meta() + c.Assert(meta.Filename(), qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i))) + c.Assert(meta.PathFile(), qt.Equals, filepath.FromSlash(fmt.Sprintf("d1/d2/d3/f-%d.txt", i))) + } + + _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3/f-1.txt")) + c.Assert(err, qt.IsNil) + _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3")) + c.Assert(err, qt.IsNil) } diff --git a/hugofs/walk.go b/hugofs/walk.go index 6947660c8..da6983f11 100644 --- a/hugofs/walk.go +++ b/hugofs/walk.go @@ -124,7 +124,6 @@ func (w *Walkway) Walk() error { if w.checkErr(w.root, err) { return nil } - return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root)) } fi = info.(FileMetaInfo) @@ -154,6 +153,15 @@ func (w *Walkway) checkErr(filename string, err error) bool { logUnsupportedSymlink(filename, w.logger) return true } + + if os.IsNotExist(err) { + // The file may be removed in process. + // This may be a ERROR situation, but it is not possible + // to determine as a general case. + w.logger.WARN.Printf("File %q not found, skipping.", filename) + return true + } + return false } diff --git a/hugofs/walk_test.go b/hugofs/walk_test.go index 4effa8000..0c08968c6 100644 --- a/hugofs/walk_test.go +++ b/hugofs/walk_test.go @@ -176,6 +176,27 @@ func collectFilenames(fs afero.Fs, base, root string) ([]string, error) { } +func collectFileinfos(fs afero.Fs, base, root string) ([]FileMetaInfo, error) { + var fis []FileMetaInfo + + walkFn := func(path string, info FileMetaInfo, err error) error { + if err != nil { + return err + } + + fis = append(fis, info) + + return nil + } + + w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn}) + + err := w.Walk() + + return fis, err + +} + func BenchmarkWalk(b *testing.B) { c := qt.New(b) fs := NewBaseFileDecorator(afero.NewMemMapFs()) diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index 5cede88d0..34770520f 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -258,6 +258,7 @@ func (s SourceFilesystems) MakeStaticPathRelative(filename string) string { // MakePathRelative creates a relative path from the given filename. // It will return an empty string if the filename is not a member of this filesystem. func (d *SourceFilesystem) MakePathRelative(filename string) string { + for _, dir := range d.Dirs { meta := dir.(hugofs.FileMetaInfo).Meta() currentPath := meta.Filename() diff --git a/hugolib/filesystems/basefs_test.go b/hugolib/filesystems/basefs_test.go index 3cac4f11a..a0e9f0020 100644 --- a/hugolib/filesystems/basefs_test.go +++ b/hugolib/filesystems/basefs_test.go @@ -173,9 +173,7 @@ theme = ["atheme"] filename = filepath.FromSlash(filename) f, err := fs.Open(filename) c.Assert(err, qt.IsNil) - name := f.Name() f.Close() - c.Assert(name, qt.Equals, filename) } } } diff --git a/hugolib/hugo_modules_test.go b/hugolib/hugo_modules_test.go index 4b71a54c8..65d232208 100644 --- a/hugolib/hugo_modules_test.go +++ b/hugolib/hugo_modules_test.go @@ -38,6 +38,47 @@ import ( "github.com/spf13/viper" ) +// https://github.com/gohugoio/hugo/issues/6730 +func TestHugoModulesTargetInSubFolder(t *testing.T) { + config := ` +baseURL="https://example.org" +workingDir = %q + +[module] +[[module.imports]] +path="github.com/gohugoio/hugoTestModule2" + [[module.imports.mounts]] + source = "templates/hooks" + target = "layouts/_default/_markup" + +` + + b := newTestSitesBuilder(t) + workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-target-in-subfolder-test") + b.Assert(err, qt.IsNil) + defer clean() + b.Fs = hugofs.NewDefault(viper.New()) + b.WithWorkingDir(workingDir).WithConfigFile("toml", fmt.Sprintf(config, workingDir)) + b.WithTemplates("_default/single.html", `{{ .Content }}`) + b.WithContent("p1.md", `--- +title: "Page" +--- + +[A link](https://bep.is) + +`) + b.WithSourceFile("go.mod", ` +module github.com/gohugoio/tests/testHugoModules + + +`) + + b.Build(BuildCfg{}) + + b.AssertFileContent("public/p1/index.html", `

Page|https://bep.is|Title: |Text: A link|END

`) + +} + // TODO(bep) this fails when testmodBuilder is also building ... func TestHugoModules(t *testing.T) { if !isCI() { @@ -588,6 +629,9 @@ workingDir = %q {{ $mypage := .Site.GetPage "/blog/mypage.md" }} {{ with $mypage }}MYPAGE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }} +{{ $mybundle := .Site.GetPage "/blog/mybundle" }} +{{ with $mybundle }}MYBUNDLE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }} + `, "_default/_markup/render-link.html", ` {{ $link := .Destination }} @@ -640,6 +684,7 @@ README: Readme Title /README.md|Path: _index.md|FilePath: README.md Readme Content. MYPAGE: My Page|Path: blog/mypage.md|FilePath: mycontent/mypage.md| +MYBUNDLE: My Bundle|Path: blog/mybundle/index.md|FilePath: mycontent/mybundle/index.md| `) b.AssertFileContent("public/blog/mypage/index.html", ` Relative Link From Page