Replace all usage of CopyOnWriteFs with OverlayFs

Fixes #9761

create/content_test.go

@@ -82,7 +82,6 @@ func TestNewContentFromFile(t *testing.T) {
 			cfg, fs := newTestCfg(c, mm)
 			h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
 			c.Assert(err, qt.IsNil)
-
 			err = create.NewContent(h, cas.kind, cas.path)
 
 			if b, ok := cas.expected.(bool); ok && !b {
@@ -98,6 +97,7 @@ func TestNewContentFromFile(t *testing.T) {
 			if !strings.HasPrefix(fname, "content") {
 				fname = filepath.Join("content", fname)
 			}
+
 			content := readFileFromFs(c, fs.Source, fname)
 
 			for _, v := range cas.expected.([]string) {

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/bep/godartsass v0.14.0
 	github.com/bep/golibsass v1.0.0
 	github.com/bep/gowebp v0.1.0
-	github.com/bep/overlayfs v0.1.0
+	github.com/bep/overlayfs v0.4.0
 	github.com/bep/tmc v0.5.1
 	github.com/clbanning/mxj/v2 v2.5.5
 	github.com/cli/safeexec v1.0.0

go.sum

@@ -188,6 +188,12 @@ github.com/bep/gowebp v0.1.0 h1:4/iQpfnxHyXs3x/aTxMMdOpLEQQhFmF6G7EieWPTQyo=
 github.com/bep/gowebp v0.1.0/go.mod h1:ZhFodwdiFp8ehGJpF4LdPl6unxZm9lLFjxD3z2h2AgI=
 github.com/bep/overlayfs v0.1.0 h1:1hOCrvS4E5Hf0qwxM7m+9oitqClD9mRjQ1d4pECsVcU=
 github.com/bep/overlayfs v0.1.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
+github.com/bep/overlayfs v0.2.0 h1:JSJbbXLi0FRHtadJCmUNvaFWEAZhpDbX1nLNiKviECM=
+github.com/bep/overlayfs v0.2.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
+github.com/bep/overlayfs v0.3.0 h1:Vufu7kg4ehDJcjOaLTSiZI0F6tSF0Aqt0AWRi44DzSg=
+github.com/bep/overlayfs v0.3.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
+github.com/bep/overlayfs v0.4.0 h1:J/G5YltfU2BxO2KV/VcFzJo94jpRMjtthRNEZ+7V7uA=
+github.com/bep/overlayfs v0.4.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
 github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI=
 github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
 github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=

hugofs/language_composite_fs.go

@@ -1,81 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"os"
-
-	"github.com/spf13/afero"
-)
-
-var (
-	_ afero.Fs             = (*languageCompositeFs)(nil)
-	_ afero.Lstater        = (*languageCompositeFs)(nil)
-	_ FilesystemsUnwrapper = (*languageCompositeFs)(nil)
-)
-
-type languageCompositeFs struct {
-	base    afero.Fs
-	overlay afero.Fs
-	*afero.CopyOnWriteFs
-}
-
-// NewLanguageCompositeFs creates a composite and language aware filesystem.
-// This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename
-// to the target filesystem. This information is available in Readdir, Stat etc. via the
-// special LanguageFileInfo FileInfo implementation.
-func NewLanguageCompositeFs(base, overlay afero.Fs) afero.Fs {
-	return &languageCompositeFs{base, overlay, afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)}
-}
-
-func (fs *languageCompositeFs) UnwrapFilesystems() []afero.Fs {
-	return []afero.Fs{fs.base, fs.overlay}
-}
-
-// Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged
-// using the language as a weight.
-func (fs *languageCompositeFs) Open(name string) (afero.File, error) {
-	f, err := fs.CopyOnWriteFs.Open(name)
-	if err != nil {
-		return nil, err
-	}
-
-	fu, ok := f.(*afero.UnionFile)
-	if ok {
-		// This is a directory: Merge it.
-		fu.Merger = LanguageDirsMerger
-	}
-	return f, nil
-}
-
-// LanguageDirsMerger implements the afero.DirsMerger interface, which is used
-// to merge two directories.
-var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
-	for _, fi1 := range bofi {
-		fim1 := fi1.(FileMetaInfo)
-		var found bool
-		for _, fi2 := range lofi {
-			fim2 := fi2.(FileMetaInfo)
-			if fi1.Name() == fi2.Name() && fim1.Meta().Lang == fim2.Meta().Lang {
-				found = true
-				break
-			}
-		}
-		if !found {
-			lofi = append(lofi, fi1)
-		}
-	}
-
-	return lofi, nil
-}

hugofs/language_merge.go

@@ -0,0 +1,39 @@
+// Copyright 2022 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+)
+
+// LanguageDirsMerger implements the overlayfs.DirsMerger func, which is used
+// to merge two directories.
+var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) []os.FileInfo {
+	for _, fi1 := range bofi {
+		fim1 := fi1.(FileMetaInfo)
+		var found bool
+		for _, fi2 := range lofi {
+			fim2 := fi2.(FileMetaInfo)
+			if fi1.Name() == fi2.Name() && fim1.Meta().Lang == fim2.Meta().Lang {
+				found = true
+				break
+			}
+		}
+		if !found {
+			lofi = append(lofi, fi1)
+		}
+	}
+
+	return lofi
+}

hugofs/noop_fs.go

@@ -38,11 +38,11 @@ func (fs noOpFs) Create(name string) (afero.File, error) {
 }
 
 func (fs noOpFs) Mkdir(name string, perm os.FileMode) error {
-	return errNoOp
+	return nil
 }
 
 func (fs noOpFs) MkdirAll(path string, perm os.FileMode) error {
-	return errNoOp
+	return nil
 }
 
 func (fs noOpFs) Open(name string) (afero.File, error) {
@@ -54,11 +54,11 @@ func (fs noOpFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File,
 }
 
 func (fs noOpFs) Remove(name string) error {
-	return errNoOp
+	return nil
 }
 
 func (fs noOpFs) RemoveAll(path string) error {
-	return errNoOp
+	return nil
 }
 
 func (fs noOpFs) Rename(oldname string, newname string) error {

hugolib/content_factory.go

@@ -112,6 +112,7 @@ func (f ContentFactory) SectionFromFilename(filename string) (string, error) {
 func (f ContentFactory) CreateContentPlaceHolder(filename string) (string, error) {
 	filename = filepath.Clean(filename)
 	_, abs, err := f.h.AbsProjectContentDir(filename)
+
 	if err != nil {
 		return "", err
 	}

hugolib/datafiles_test.go

@@ -27,6 +27,38 @@ import (
 	qt "github.com/frankban/quicktest"
 )
 
+func TestDataFromTheme(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- config.toml --
+[module]
+[[module.imports]]
+path = "mytheme"
+-- data/a.toml --
+d1 = "d1main"
+d2 = "d2main"
+-- themes/mytheme/data/a.toml --
+d1 = "d1theme"
+d2 = "d2theme"
+d3 = "d3theme"
+-- layouts/index.html --
+d1: {{ site.Data.a.d1  }}|d2: {{ site.Data.a.d2 }}|d3: {{ site.Data.a.d3  }}
+
+`
+
+	b := NewIntegrationTestBuilder(
+		IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	).Build()
+
+	b.AssertFileContent("public/index.html", `
+d1: d1main|d2: d2main|d3: d3theme
+	`)
+}
+
 func TestDataDir(t *testing.T) {
 	t.Parallel()
 	equivDataDirs := make([]dataDir, 3)

hugolib/filesystems/basefs.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/bep/overlayfs"
 	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/hugofs/glob"
 
@@ -145,12 +146,14 @@ func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
 		if !meta.IsProject {
 			continue
 		}
+
 		if isAbs {
 			if strings.HasPrefix(filename, meta.Filename) {
 				return strings.TrimPrefix(filename, meta.Filename), filename, nil
 			}
 		} else {
-			contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator)
+			contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator) + filePathSeparator
+
 			if strings.HasPrefix(filename, contentDir) {
 				relFilename := strings.TrimPrefix(filename, contentDir)
 				absFilename := filepath.Join(meta.Filename, relFilename)
@@ -163,14 +166,14 @@ func (b *BaseFs) AbsProjectContentDir(filename string) (string, string, error) {
 	if !isAbs {
 		// A filename on the form "posts/mypage.md", put it inside
 		// the first content folder, usually <workDir>/content.
-		// Pick the last project dir (which is probably the most important one).
+		// Pick the first project dir (which is probably the most important one).
-		contentDirs := b.SourceFilesystems.Content.Dirs
+		for _, dir := range b.SourceFilesystems.Content.Dirs {
-		for i := len(contentDirs) - 1; i >= 0; i-- {
+			meta := dir.Meta()
-			meta := contentDirs[i].Meta()
 			if meta.IsProject {
 				return filename, filepath.Join(meta.Filename, filename), nil
 			}
 		}
+
 	}
 
 	return "", "", errors.Errorf("could not determine content directory for %q", filename)
@@ -260,10 +263,16 @@ type SourceFilesystem struct {
 // The order is content, static and then assets.
 // TODO(bep) check usage
 func (s SourceFilesystems) ContentStaticAssetFs(lang string) afero.Fs {
-	staticFs := s.StaticFs(lang)
+	return overlayfs.New(
+		overlayfs.Options{
+			Fss: []afero.Fs{
+				s.Content.Fs,
+				s.StaticFs(lang),
+				s.Assets.Fs,
+			},
+		},
+	)
 
-	base := afero.NewCopyOnWriteFs(s.Assets.Fs, staticFs)
-	return afero.NewCopyOnWriteFs(base, s.Content.Fs)
 }
 
 // StaticFs returns the static filesystem for the given language.
@@ -491,7 +500,6 @@ func (b *sourceFilesystemsBuilder) newSourceFilesystem(name string, fs afero.Fs,
 
 func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
 	if b.theBigFs == nil {
-
 		theBigFs, err := b.createMainOverlayFs(b.p)
 		if err != nil {
 			return nil, errors.Wrap(err, "create main fs")
@@ -510,8 +518,6 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
 		return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs)
 	}
 
-	b.theBigFs.finalizeDirs()
-
 	b.result.Archetypes = createView(files.ComponentFolderArchetypes)
 	b.result.Layouts = createView(files.ComponentFolderLayouts)
 	b.result.Assets = createView(files.ComponentFolderAssets)
@@ -566,9 +572,12 @@ func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
 }
 
 func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) {
-	var staticFsMap map[string]afero.Fs
+	var staticFsMap map[string]*overlayfs.OverlayFs
 	if b.p.Cfg.GetBool("multihost") {
-		staticFsMap = make(map[string]afero.Fs)
+		staticFsMap = make(map[string]*overlayfs.OverlayFs)
+		for _, l := range b.p.Languages {
+			staticFsMap[l.Lang] = overlayfs.New(overlayfs.Options{})
+		}
 	}
 
 	collector := &filesystemsCollector{
@@ -576,35 +585,33 @@ func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesys
 		sourceModules:     hugofs.NewNoSymlinkFs(b.sourceFs, b.logger, false),
 		overlayDirs:       make(map[string][]hugofs.FileMetaInfo),
 		staticPerLanguage: staticFsMap,
+
+		overlayMounts:        overlayfs.New(overlayfs.Options{}),
+		overlayMountsContent: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}),
+		overlayMountsStatic:  overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}),
+		overlayFull:          overlayfs.New(overlayfs.Options{}),
+		overlayResources:     overlayfs.New(overlayfs.Options{FirstWritable: true}),
 	}
 
 	mods := p.AllModules
 
-	if len(mods) == 0 {
+	mounts := make([]mountsDescriptor, len(mods))
-		return collector, nil
-	}
 
-	modsReversed := make([]mountsDescriptor, len(mods))
+	for i := 0; i < len(mods); i++ {
-
-	// The theme components are ordered from left to right.
-	// We need to revert it to get the
-	// overlay logic below working as expected, with the project on top.
-	j := 0
-	for i := len(mods) - 1; i >= 0; i-- {
 		mod := mods[i]
 		dir := mod.Dir()
 
 		isMainProject := mod.Owner() == nil
-		modsReversed[j] = mountsDescriptor{
+		mounts[i] = mountsDescriptor{
 			Module:        mod,
 			dir:           dir,
 			isMainProject: isMainProject,
-			ordinal:       j,
+			ordinal:       i,
 		}
-		j++
+
 	}
 
-	err := b.createOverlayFs(collector, modsReversed)
+	err := b.createOverlayFs(collector, mounts)
 
 	return collector, err
 }
@@ -617,137 +624,143 @@ func (b *sourceFilesystemsBuilder) isStaticMount(mnt modules.Mount) bool {
 	return strings.HasPrefix(mnt.Target, files.ComponentFolderStatic)
 }
 
-func (b *sourceFilesystemsBuilder) createModFs(
+func (b *sourceFilesystemsBuilder) createOverlayFs(
 	collector *filesystemsCollector,
-	md mountsDescriptor) error {
+	mounts []mountsDescriptor) error {
-	var (
-		fromTo        []hugofs.RootMapping
-		fromToContent []hugofs.RootMapping
-		fromToStatic  []hugofs.RootMapping
-	)
 
-	absPathify := func(path string) (string, string) {
+	if len(mounts) == 0 {
-		if filepath.IsAbs(path) {
+		appendNopIfEmpty := func(ofs *overlayfs.OverlayFs) *overlayfs.OverlayFs {
-			return "", path
+			if ofs.NumFilesystems() > 0 {
+				return ofs
+			}
+			return ofs.Append(hugofs.NoOpFs)
 		}
-		return md.dir, hpaths.AbsPathify(md.dir, path)
+		collector.overlayMounts = appendNopIfEmpty(collector.overlayMounts)
+		collector.overlayMountsContent = appendNopIfEmpty(collector.overlayMountsContent)
+		collector.overlayMountsStatic = appendNopIfEmpty(collector.overlayMountsStatic)
+		collector.overlayFull = appendNopIfEmpty(collector.overlayFull)
+		collector.overlayResources = appendNopIfEmpty(collector.overlayResources)
+
+		return nil
 	}
 
-	for i, mount := range md.Mounts() {
+	for _, md := range mounts {
-
+		var (
-		// Add more weight to early mounts.
+			fromTo        []hugofs.RootMapping
-		// When two mounts contain the same filename,
+			fromToContent []hugofs.RootMapping
-		// the first entry wins.
+			fromToStatic  []hugofs.RootMapping
-		mountWeight := (10 + md.ordinal) * (len(md.Mounts()) - i)
-
-		inclusionFilter, err := glob.NewFilenameFilter(
-			types.ToStringSlicePreserveString(mount.IncludeFiles),
-			types.ToStringSlicePreserveString(mount.ExcludeFiles),
 		)
+
+		absPathify := func(path string) (string, string) {
+			if filepath.IsAbs(path) {
+				return "", path
+			}
+			return md.dir, hpaths.AbsPathify(md.dir, path)
+		}
+
+		for i, mount := range md.Mounts() {
+
+			// Add more weight to early mounts.
+			// When two mounts contain the same filename,
+			// the first entry wins.
+			mountWeight := (10 + md.ordinal) * (len(md.Mounts()) - i)
+
+			inclusionFilter, err := glob.NewFilenameFilter(
+				types.ToStringSlicePreserveString(mount.IncludeFiles),
+				types.ToStringSlicePreserveString(mount.ExcludeFiles),
+			)
+			if err != nil {
+				return err
+			}
+
+			base, filename := absPathify(mount.Source)
+
+			rm := hugofs.RootMapping{
+				From:      mount.Target,
+				To:        filename,
+				ToBasedir: base,
+				Module:    md.Module.Path(),
+				IsProject: md.isMainProject,
+				Meta: &hugofs.FileMeta{
+					Watch:           md.Watch(),
+					Weight:          mountWeight,
+					Classifier:      files.ContentClassContent,
+					InclusionFilter: inclusionFilter,
+				},
+			}
+
+			isContentMount := b.isContentMount(mount)
+
+			lang := mount.Lang
+			if lang == "" && isContentMount {
+				lang = b.p.DefaultContentLanguage
+			}
+
+			rm.Meta.Lang = lang
+
+			if isContentMount {
+				fromToContent = append(fromToContent, rm)
+			} else if b.isStaticMount(mount) {
+				fromToStatic = append(fromToStatic, rm)
+			} else {
+				fromTo = append(fromTo, rm)
+			}
+		}
+
+		modBase := collector.sourceProject
+		if !md.isMainProject {
+			modBase = collector.sourceModules
+		}
+		sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true)
+
+		rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
+		if err != nil {
+			return err
+		}
+		rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)
+		if err != nil {
+			return err
+		}
+		rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...)
 		if err != nil {
 			return err
 		}
 
-		base, filename := absPathify(mount.Source)
+		// We need to keep the ordered list of directories for watching and
+		// some special merge operations (data, i18n).
+		collector.addDirs(rmfs)
+		collector.addDirs(rmfsContent)
+		collector.addDirs(rmfsStatic)
 
-		rm := hugofs.RootMapping{
+		if collector.staticPerLanguage != nil {
-			From:      mount.Target,
+			for _, l := range b.p.Languages {
-			To:        filename,
+				lang := l.Lang
-			ToBasedir: base,
-			Module:    md.Module.Path(),
-			IsProject: md.isMainProject,
-			Meta: &hugofs.FileMeta{
-				Watch:           md.Watch(),
-				Weight:          mountWeight,
-				Classifier:      files.ContentClassContent,
-				InclusionFilter: inclusionFilter,
-			},
-		}
 
-		isContentMount := b.isContentMount(mount)
+				lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool {
+					rlang := rm.Meta.Lang
+					return rlang == "" || rlang == lang
+				})
 
-		lang := mount.Lang
+				bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic)
-		if lang == "" && isContentMount {
+				collector.staticPerLanguage[lang] = collector.staticPerLanguage[lang].Append(bfs)
-			lang = b.p.DefaultContentLanguage
-		}
 
-		rm.Meta.Lang = lang
-
-		if isContentMount {
-			fromToContent = append(fromToContent, rm)
-		} else if b.isStaticMount(mount) {
-			fromToStatic = append(fromToStatic, rm)
-		} else {
-			fromTo = append(fromTo, rm)
-		}
-	}
-
-	modBase := collector.sourceProject
-	if !md.isMainProject {
-		modBase = collector.sourceModules
-	}
-	sourceStatic := hugofs.NewNoSymlinkFs(modBase, b.logger, true)
-
-	rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
-	if err != nil {
-		return err
-	}
-	rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)
-	if err != nil {
-		return err
-	}
-	rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...)
-	if err != nil {
-		return err
-	}
-
-	// We need to keep the ordered list of directories for watching and
-	// some special merge operations (data, i18n).
-	collector.addDirs(rmfs)
-	collector.addDirs(rmfsContent)
-	collector.addDirs(rmfsStatic)
-
-	if collector.staticPerLanguage != nil {
-		for _, l := range b.p.Languages {
-			lang := l.Lang
-
-			lfs := rmfsStatic.Filter(func(rm hugofs.RootMapping) bool {
-				rlang := rm.Meta.Lang
-				return rlang == "" || rlang == lang
-			})
-
-			bfs := afero.NewBasePathFs(lfs, files.ComponentFolderStatic)
-
-			sfs, found := collector.staticPerLanguage[lang]
-			if found {
-				collector.staticPerLanguage[lang] = afero.NewCopyOnWriteFs(sfs, bfs)
-			} else {
-				collector.staticPerLanguage[lang] = bfs
 			}
 		}
-	}
 
-	getResourcesDir := func() string {
+		getResourcesDir := func() string {
-		if md.isMainProject {
+			if md.isMainProject {
-			return b.p.AbsResourcesDir
+				return b.p.AbsResourcesDir
+			}
+			_, filename := absPathify(files.FolderResources)
+			return filename
 		}
-		_, filename := absPathify(files.FolderResources)
-		return filename
-	}
 
-	if collector.overlayMounts == nil {
+		collector.overlayMounts = collector.overlayMounts.Append(rmfs)
-		collector.overlayMounts = rmfs
+		collector.overlayMountsContent = collector.overlayMountsContent.Append(rmfsContent)
-		collector.overlayMountsContent = rmfsContent
+		collector.overlayMountsStatic = collector.overlayMountsStatic.Append(rmfsStatic)
-		collector.overlayMountsStatic = rmfsStatic
+		collector.overlayFull = collector.overlayFull.Append(afero.NewBasePathFs(modBase, md.dir))
-		collector.overlayFull = afero.NewBasePathFs(modBase, md.dir)
+		collector.overlayResources = collector.overlayResources.Append(afero.NewBasePathFs(modBase, getResourcesDir()))
-		collector.overlayResources = afero.NewBasePathFs(modBase, getResourcesDir())
-	} else {
 
-		collector.overlayMounts = afero.NewCopyOnWriteFs(collector.overlayMounts, rmfs)
-		collector.overlayMountsContent = hugofs.NewLanguageCompositeFs(collector.overlayMountsContent, rmfsContent)
-		collector.overlayMountsStatic = hugofs.NewLanguageCompositeFs(collector.overlayMountsStatic, rmfsStatic)
-		collector.overlayFull = afero.NewCopyOnWriteFs(collector.overlayFull, afero.NewBasePathFs(modBase, md.dir))
-		collector.overlayResources = afero.NewCopyOnWriteFs(collector.overlayResources, afero.NewBasePathFs(modBase, getResourcesDir()))
 	}
 
 	return nil
@@ -777,18 +790,18 @@ type filesystemsCollector struct {
 	sourceProject afero.Fs // Source for project folders
 	sourceModules afero.Fs // Source for modules/themes
 
-	overlayMounts        afero.Fs
+	overlayMounts        *overlayfs.OverlayFs
-	overlayMountsContent afero.Fs
+	overlayMountsContent *overlayfs.OverlayFs
-	overlayMountsStatic  afero.Fs
+	overlayMountsStatic  *overlayfs.OverlayFs
-	overlayFull          afero.Fs
+	overlayFull          *overlayfs.OverlayFs
-	overlayResources     afero.Fs
+	overlayResources     *overlayfs.OverlayFs
 
 	// Maps component type (layouts, static, content etc.) an ordered list of
 	// directories representing the overlay filesystems above.
 	overlayDirs map[string][]hugofs.FileMetaInfo
 
 	// Set if in multihost mode
-	staticPerLanguage map[string]afero.Fs
+	staticPerLanguage map[string]*overlayfs.OverlayFs
 
 	finalizerInit sync.Once
 }
@@ -807,15 +820,6 @@ func (c *filesystemsCollector) addDir(rfs *hugofs.RootMappingFs, componentFolder
 	}
 }
 
-func (c *filesystemsCollector) finalizeDirs() {
-	c.finalizerInit.Do(func() {
-		// Order the directories from top to bottom (project, theme a, theme ...).
-		for _, dirs := range c.overlayDirs {
-			c.reverseFis(dirs)
-		}
-	})
-}
-
 func (c *filesystemsCollector) reverseFis(fis []hugofs.FileMetaInfo) {
 	for i := len(fis)/2 - 1; i >= 0; i-- {
 		opp := len(fis) - 1 - i
@@ -829,20 +833,3 @@ type mountsDescriptor struct {
 	isMainProject bool
 	ordinal       int
 }
-
-func (b *sourceFilesystemsBuilder) createOverlayFs(collector *filesystemsCollector, mounts []mountsDescriptor) error {
-	if len(mounts) == 0 {
-		return nil
-	}
-
-	err := b.createModFs(collector, mounts[0])
-	if err != nil {
-		return err
-	}
-
-	if len(mounts) == 1 {
-		return nil
-	}
-
-	return b.createOverlayFs(collector, mounts[1:])
-}

hugolib/hugo_modules_test.go

@@ -777,6 +777,8 @@ weight = 2
 				}
 			}
 
+			c.Logf("Checking %d:%d %q", i, j, id)
+
 			statCheck(componentFs, fmt.Sprintf("realsym%s", id), true)
 			statCheck(componentFs, fmt.Sprintf("real/datasym%s.toml", id), false)
 

hugolib/paths/paths.go

@@ -53,9 +53,6 @@ type Paths struct {
 	// pagination path handling
 	PaginatePath string
 
-	// TODO1 check usage
-	PublishDir string
-
 	// When in multihost mode, this returns a list of base paths below PublishDir
 	// for each language.
 	MultihostTargetBasePaths []string
@@ -185,9 +182,6 @@ func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) {
 		p.ModulesClient = cfg.Get("modulesClient").(*modules.Client)
 	}
 
-	// TODO(bep) remove this, eventually
-	p.PublishDir = absPublishDir
-
 	return p, nil
 }
 

langs/i18n/integration_test.go

@@ -0,0 +1,57 @@
+// Copyright 2022 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package i18n_test
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+func TestI18nFromTheme(t *testing.T) {
+	t.Parallel()
+
+	files := `
+-- config.toml --
+[module]
+[[module.imports]]
+path = "mytheme"
+-- i18n/en.toml --
+[l1]
+other = 'l1main'
+[l2]
+other = 'l2main'
+-- themes/mytheme/i18n/en.toml --
+[l1]
+other = 'l1theme'
+[l2]
+other = 'l2theme'
+[l3]
+other = 'l3theme'
+-- layouts/index.html --
+l1: {{ i18n "l1"  }}|l2: {{ i18n "l2"  }}|l3: {{ i18n "l3"  }}
+
+`
+
+	b := hugolib.NewIntegrationTestBuilder(
+		hugolib.IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	).Build()
+
+	b.AssertFileContent("public/index.html", `
+l1: l1main|l2: l2main|l3: l3theme
+	`)
+}

tpl/os/os.go

@@ -21,6 +21,7 @@ import (
 	_os "os"
 	"path/filepath"
 
+	"github.com/bep/overlayfs"
 	"github.com/gohugoio/hugo/deps"
 	"github.com/spf13/afero"
 	"github.com/spf13/cast"
@@ -32,7 +33,12 @@ func New(d *deps.Deps) *Namespace {
 
 	// The docshelper script does not have or need all the dependencies set up.
 	if d.PathSpec != nil {
-		readFileFs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(d.PathSpec.BaseFs.Content.Fs, d.PathSpec.BaseFs.Work))
+		readFileFs = overlayfs.New(overlayfs.Options{
+			Fss: []afero.Fs{
+				d.PathSpec.BaseFs.Work,
+				d.PathSpec.BaseFs.Content.Fs,
+			},
+		})
 		// See #9599
 		workFs = d.PathSpec.BaseFs.WorkDir
 	}