Add support for a content dir set per language

A sample config:

```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true

[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"

[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```

The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.

The content files will be assigned a language by

1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.

The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.

Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.

If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.

Fixes #4523
Fixes #4552
Fixes #4553

Gopkg.lock

@@ -293,8 +293,8 @@
     ".",
     "mem"
   ]
-  revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c"
-  version = "v1.0.2"
+  revision = "63644898a8da0bc22138abf860edaf5277b6102e"
+  version = "v1.1.0"
 
 [[projects]]
   name = "github.com/spf13/cast"
@@ -424,6 +424,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "13ab39f8bfafadc12c05726e565ee3f3d94bf7d6c0e8adf04056de0691bf2dd6"
+  inputs-digest = "edb250b53926de21df1740c379c76351b7e9b110c96a77078a10ba69bf31a2d4"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -78,7 +78,7 @@
 
 [[constraint]]
   name = "github.com/spf13/afero"
-  version = "^1.0.1"
+  version = "^1.1.0"
 
 [[constraint]]
   name = "github.com/spf13/cast"

commands/hugo.go

@@ -705,7 +705,7 @@ func (c *commandeer) getDirList() ([]string, error) {
 					c.Logger.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", path, err)
 					return nil
 				}
-				linkfi, err := helpers.LstatIfOs(c.Fs.Source, link)
+				linkfi, err := helpers.LstatIfPossible(c.Fs.Source, link)
 				if err != nil {
 					c.Logger.ERROR.Printf("Cannot stat %q: %s", link, err)
 					return nil
@@ -743,9 +743,13 @@ func (c *commandeer) getDirList() ([]string, error) {
 
 	// SymbolicWalk will log anny ERRORs
 	_ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker)
-	_ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), symLinkWalker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker)
+
+	for _, contentDir := range c.PathSpec().ContentDirs() {
+		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker)
+	}
+
 	for _, staticDir := range staticDirs {
 		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
 	}

common/types/types.go

@@ -20,6 +20,12 @@ import (
 	"github.com/spf13/cast"
 )
 
+// KeyValueStr is a string tuple.
+type KeyValueStr struct {
+	Key   string
+	Value string
+}
+
 // KeyValues holds an key and a slice of values.
 type KeyValues struct {
 	Key    interface{}

create/content.go

@@ -63,7 +63,22 @@ func NewContent(
 		return err
 	}
 
-	contentPath := s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+	// The site may have multiple content dirs, and we currently do not know which contentDir the
+	// user wants to create this content in. We should improve on this, but we start by testing if the
+	// provided path points to an existing dir. If so, use it as is.
+	var contentPath string
+	var exists bool
+	targetDir := filepath.Dir(targetPath)
+
+	if targetDir != "" && targetDir != "." {
+		exists, _ = helpers.Exists(targetDir, ps.Fs.Source)
+	}
+
+	if exists {
+		contentPath = targetPath
+	} else {
+		contentPath = s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+	}
 
 	if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
 		return err

create/content_template_handler.go

@@ -88,10 +88,15 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 		err               error
 	)
 
-	sp := source.NewSourceSpec(s.Deps.Cfg, s.Deps.Fs)
+	ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg)
+	sp := source.NewSourceSpec(ps, ps.Fs.Source)
+	if err != nil {
+		return nil, err
+	}
 	f := sp.NewFileInfo("", targetPath, false, nil)
 
 	name := f.TranslationBaseName()
+
 	if name == "index" || name == "_index" {
 		// Page bundles; the directory name will hopefully have a better name.
 		dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator)

create/content_test.go

@@ -75,7 +75,7 @@ func TestNewContent(t *testing.T) {
 		for i, v := range c.expected {
 			found := strings.Contains(content, v)
 			if !found {
-				t.Errorf("[%d] %q missing from output:\n%q", i, v, content)
+				t.Fatalf("[%d] %q missing from output:\n%q", i, v, content)
 			}
 		}
 	}

deps/deps.go

@@ -126,7 +126,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
-	sp := source.NewSourceSpec(cfg.Language, fs)
+	sp := source.NewSourceSpec(ps, fs.Source)
 
 	d := &Deps{
 		Fs:                  fs,

helpers/language.go

@@ -41,6 +41,14 @@ type Language struct {
 	Title        string
 	Weight       int
 
+	Disabled bool
+
+	// If set per language, this tells Hugo that all content files without any
+	// language indicator (e.g. my-page.en.md) is in this language.
+	// This is usually a path relative to the working dir, but it can be an
+	// absolute directory referenece. It is what we get.
+	ContentDir string
+
 	Cfg config.Provider
 
 	// These are params declared in the [params] section of the language merged with the
@@ -66,7 +74,13 @@ func NewLanguage(lang string, cfg config.Provider) *Language {
 		params[k] = v
 	}
 	ToLowerMap(params)
-	l := &Language{Lang: lang, Cfg: cfg, params: params, settings: make(map[string]interface{})}
+
+	defaultContentDir := cfg.GetString("contentDir")
+	if defaultContentDir == "" {
+		panic("contentDir not set")
+	}
+
+	l := &Language{Lang: lang, ContentDir: defaultContentDir, Cfg: cfg, params: params, settings: make(map[string]interface{})}
 	return l
 }
 

helpers/language_test.go

@@ -22,11 +22,12 @@ import (
 
 func TestGetGlobalOnlySetting(t *testing.T) {
 	v := viper.New()
+	v.Set("defaultContentLanguageInSubdir", true)
+	v.Set("contentDir", "content")
+	v.Set("paginatePath", "page")
 	lang := NewDefaultLanguage(v)
 	lang.Set("defaultContentLanguageInSubdir", false)
 	lang.Set("paginatePath", "side")
-	v.Set("defaultContentLanguageInSubdir", true)
-	v.Set("paginatePath", "page")
 
 	require.True(t, lang.GetBool("defaultContentLanguageInSubdir"))
 	require.Equal(t, "side", lang.GetString("paginatePath"))
@@ -37,6 +38,7 @@ func TestLanguageParams(t *testing.T) {
 
 	v := viper.New()
 	v.Set("p1", "p1cfg")
+	v.Set("contentDir", "content")
 
 	lang := NewDefaultLanguage(v)
 	lang.SetParam("p1", "p1p")

helpers/path.go

@@ -33,7 +33,7 @@ var (
 	ErrThemeUndefined = errors.New("no theme set")
 
 	// ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters.
-	ErrWalkRootTooShort = errors.New("Path too short. Stop walking.")
+	ErrPathTooShort = errors.New("file path is too short")
 )
 
 // filepathPathBridge is a bridge for common functionality in filepath vs path
@@ -446,7 +446,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 
 	// Sanity check
 	if len(root) < 4 {
-		return ErrWalkRootTooShort
+		return ErrPathTooShort
 	}
 
 	// Handle the root first
@@ -481,7 +481,7 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 }
 
 func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
-	fileInfo, err := LstatIfOs(fs, path)
+	fileInfo, err := LstatIfPossible(fs, path)
 	realPath := path
 
 	if err != nil {
@@ -493,7 +493,7 @@ func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot read symbolic link '%s', error was: %s", path, err)
 		}
-		fileInfo, err = LstatIfOs(fs, link)
+		fileInfo, err = LstatIfPossible(fs, link)
 		if err != nil {
 			return nil, "", fmt.Errorf("Cannot stat '%s', error was: %s", link, err)
 		}
@@ -514,16 +514,14 @@ func GetRealPath(fs afero.Fs, path string) (string, error) {
 	return realPath, nil
 }
 
-// Code copied from Afero's path.go
-// if the filesystem is OsFs use Lstat, else use fs.Stat
-func LstatIfOs(fs afero.Fs, path string) (info os.FileInfo, err error) {
-	_, ok := fs.(*afero.OsFs)
-	if ok {
-		info, err = os.Lstat(path)
-	} else {
-		info, err = fs.Stat(path)
+// LstatIfPossible can be used to call Lstat if possible, else Stat.
+func LstatIfPossible(fs afero.Fs, path string) (os.FileInfo, error) {
+	if lstater, ok := fs.(afero.Lstater); ok {
+		fi, _, err := lstater.LstatIfPossible(path)
+		return fi, err
 	}
-	return
+
+	return fs.Stat(path)
 }
 
 // SafeWriteToDisk is the same as WriteToDisk

helpers/path_test.go

@@ -57,8 +57,10 @@ func TestMakePath(t *testing.T) {
 
 	for _, test := range tests {
 		v := viper.New()
-		l := NewDefaultLanguage(v)
+		v.Set("contentDir", "content")
 		v.Set("removePathAccents", test.removeAccents)
+
+		l := NewDefaultLanguage(v)
 		p, err := NewPathSpec(hugofs.NewMem(v), l)
 		require.NoError(t, err)
 
@@ -71,6 +73,8 @@ func TestMakePath(t *testing.T) {
 
 func TestMakePathSanitized(t *testing.T) {
 	v := viper.New()
+	v.Set("contentDir", "content")
+
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -98,6 +102,7 @@ func TestMakePathSanitizedDisablePathToLower(t *testing.T) {
 	v := viper.New()
 
 	v.Set("disablePathToLower", true)
+	v.Set("contentDir", "content")
 
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)

helpers/pathspec.go

@@ -17,6 +17,9 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/cast"
@@ -49,6 +52,8 @@ type PathSpec struct {
 	layoutDir      string
 	workingDir     string
 	staticDirs     []string
+	absContentDirs []types.KeyValueStr
+
 	PublishDir string
 
 	// The PathSpec looks up its config settings in both the current language
@@ -65,6 +70,9 @@ type PathSpec struct {
 	// The file systems to use
 	Fs *hugofs.Fs
 
+	// The fine grained filesystems in play (resources, content etc.).
+	BaseFs *hugofs.BaseFs
+
 	// The config provider to use
 	Cfg config.Provider
 }
@@ -105,8 +113,65 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		languages = l
 	}
 
+	defaultContentLanguage := cfg.GetString("defaultContentLanguage")
+
+	// We will eventually pull out this badly placed path logic.
+	contentDir := cfg.GetString("contentDir")
+	workingDir := cfg.GetString("workingDir")
+	resourceDir := cfg.GetString("resourceDir")
+	publishDir := cfg.GetString("publishDir")
+
+	if len(languages) == 0 {
+		// We have some old tests that does not test the entire chain, hence
+		// they have no languages. So create one so we get the proper filesystem.
+		languages = Languages{&Language{Lang: "en", ContentDir: contentDir}}
+	}
+
+	absPuslishDir := AbsPathify(workingDir, publishDir)
+	if !strings.HasSuffix(absPuslishDir, FilePathSeparator) {
+		absPuslishDir += FilePathSeparator
+	}
+	// If root, remove the second '/'
+	if absPuslishDir == "//" {
+		absPuslishDir = FilePathSeparator
+	}
+	absResourcesDir := AbsPathify(workingDir, resourceDir)
+	if !strings.HasSuffix(absResourcesDir, FilePathSeparator) {
+		absResourcesDir += FilePathSeparator
+	}
+	if absResourcesDir == "//" {
+		absResourcesDir = FilePathSeparator
+	}
+
+	contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages)
+	if err != nil {
+		return nil, err
+	}
+
+	// Make sure we don't have any overlapping content dirs. That will never work.
+	for i, d1 := range absContentDirs {
+		for j, d2 := range absContentDirs {
+			if i == j {
+				continue
+			}
+			if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
+				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
+			}
+		}
+	}
+
+	resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir)
+	publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir)
+
+	baseFs := &hugofs.BaseFs{
+		ContentFs:   contentFs,
+		ResourcesFs: resourcesFs,
+		PublishFs:   publishFs,
+	}
+
 	ps := &PathSpec{
 		Fs:                             fs,
+		BaseFs:                         baseFs,
 		Cfg:                            cfg,
 		disablePathToLower:             cfg.GetBool("disablePathToLower"),
 		removePathAccents:              cfg.GetBool("removePathAccents"),
@@ -116,14 +181,15 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		Language:                       language,
 		Languages:                      languages,
 		defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),
-		defaultContentLanguage:         cfg.GetString("defaultContentLanguage"),
+		defaultContentLanguage:         defaultContentLanguage,
 		paginatePath:                   cfg.GetString("paginatePath"),
 		BaseURL:                        baseURL,
-		contentDir:                     cfg.GetString("contentDir"),
+		contentDir:                     contentDir,
 		themesDir:                      cfg.GetString("themesDir"),
 		layoutDir:                      cfg.GetString("layoutDir"),
-		workingDir:                     cfg.GetString("workingDir"),
+		workingDir:                     workingDir,
 		staticDirs:                     staticDirs,
+		absContentDirs:                 absContentDirs,
 		theme:                          cfg.GetString("theme"),
 		ProcessingStats:                NewProcessingStats(lang),
 	}
@@ -135,13 +201,8 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		}
 	}
 
-	publishDir := ps.AbsPathify(cfg.GetString("publishDir")) + FilePathSeparator
-	// If root, remove the second '/'
-	if publishDir == "//" {
-		publishDir = FilePathSeparator
-	}
-
-	ps.PublishDir = publishDir
+	// TODO(bep) remove this, eventually
+	ps.PublishDir = absPuslishDir
 
 	return ps, nil
 }
@@ -165,6 +226,107 @@ func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
 	return out
 }
 
+func createContentFs(fs afero.Fs,
+	workingDir,
+	defaultContentLanguage string,
+	languages Languages) (afero.Fs, []types.KeyValueStr, error) {
+
+	var contentLanguages Languages
+	var contentDirSeen = make(map[string]bool)
+	languageSet := make(map[string]bool)
+
+	// The default content language needs to be first.
+	for _, language := range languages {
+		if language.Lang == defaultContentLanguage {
+			contentLanguages = append(contentLanguages, language)
+			contentDirSeen[language.ContentDir] = true
+		}
+		languageSet[language.Lang] = true
+	}
+
+	for _, language := range languages {
+		if contentDirSeen[language.ContentDir] {
+			continue
+		}
+		if language.ContentDir == "" {
+			language.ContentDir = defaultContentLanguage
+		}
+		contentDirSeen[language.ContentDir] = true
+		contentLanguages = append(contentLanguages, language)
+
+	}
+
+	var absContentDirs []types.KeyValueStr
+
+	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
+	return fs, absContentDirs, err
+
+}
+
+func createContentOverlayFs(source afero.Fs,
+	workingDir string,
+	languages Languages,
+	languageSet map[string]bool,
+	absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
+	if len(languages) == 0 {
+		return source, nil
+	}
+
+	language := languages[0]
+
+	contentDir := language.ContentDir
+	if contentDir == "" {
+		panic("missing contentDir")
+	}
+
+	absContentDir := AbsPathify(workingDir, language.ContentDir)
+	if !strings.HasSuffix(absContentDir, FilePathSeparator) {
+		absContentDir += FilePathSeparator
+	}
+
+	// If root, remove the second '/'
+	if absContentDir == "//" {
+		absContentDir = FilePathSeparator
+	}
+
+	if len(absContentDir) < 6 {
+		return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort)
+	}
+
+	*absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
+
+	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
+	if len(languages) == 1 {
+		return overlay, nil
+	}
+
+	base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
+	if err != nil {
+		return nil, err
+	}
+
+	return hugofs.NewLanguageCompositeFs(base, overlay), nil
+
+}
+
+// RelContentDir tries to create a path relative to the content root from
+// the given filename. The return value is the path and language code.
+func (p *PathSpec) RelContentDir(filename string) (string, string) {
+	for _, dir := range p.absContentDirs {
+		if strings.HasPrefix(filename, dir.Value) {
+			rel := strings.TrimPrefix(filename, dir.Value)
+			return strings.TrimPrefix(rel, FilePathSeparator), dir.Key
+		}
+	}
+	// Either not a content dir or already relative.
+	return filename, ""
+}
+
+// ContentDirs returns all the content dirs (absolute paths).
+func (p *PathSpec) ContentDirs() []types.KeyValueStr {
+	return p.absContentDirs
+}
+
 // PaginatePath returns the configured root path used for paginator pages.
 func (p *PathSpec) PaginatePath() string {
 	return p.paginatePath

helpers/pathspec_test.go

@@ -24,6 +24,7 @@ import (
 
 func TestNewPathSpecFromConfig(t *testing.T) {
 	v := viper.New()
+	v.Set("contentDir", "content")
 	l := NewLanguage("no", v)
 	v.Set("disablePathToLower", true)
 	v.Set("removePathAccents", true)

helpers/testhelpers_test.go

@@ -25,6 +25,7 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec {
 
 func newTestCfg(fs *hugofs.Fs) *viper.Viper {
 	v := viper.New()
+	v.Set("contentDir", "content")
 
 	v.SetFs(fs.Source)
 

helpers/url_test.go

@@ -27,6 +27,7 @@ import (
 func TestURLize(t *testing.T) {
 
 	v := viper.New()
+	v.Set("contentDir", "content")
 	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -88,6 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool,
 
 	for _, test := range tests {
 		v.Set("baseURL", test.baseURL)
+		v.Set("contentDir", "content")
 		l := NewLanguage(lang, v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -166,6 +168,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool,
 	for i, test := range tests {
 		v.Set("baseURL", test.baseURL)
 		v.Set("canonifyURLs", test.canonify)
+		v.Set("contentDir", "content")
 		l := NewLanguage(lang, v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
@@ -254,6 +257,7 @@ func TestURLPrep(t *testing.T) {
 	for i, d := range data {
 		v := viper.New()
 		v.Set("uglyURLs", d.ugly)
+		v.Set("contentDir", "content")
 		l := NewDefaultLanguage(v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 

hugofs/base_fs.go

@@ -0,0 +1,35 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"github.com/spf13/afero"
+)
+
+// BaseFs contains the core base filesystems used by Hugo. The name "base" is used
+// to underline that even if they can be composites, they all have a base path set to a specific
+// resource folder, e.g "/my-project/content". So, no absolute filenames needed.
+type BaseFs struct {
+	// The filesystem used to capture content. This can be a composite and
+	// language aware file system.
+	ContentFs afero.Fs
+
+	// The filesystem used to store resources (processed images etc.).
+	// This usually maps to /my-project/resources.
+	ResourcesFs afero.Fs
+
+	// The filesystem used to publish the rendered site.
+	// This usually maps to /my-project/public.
+	PublishFs afero.Fs
+}

hugofs/language_composite_fs.go

@@ -0,0 +1,51 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"github.com/spf13/afero"
+)
+
+var (
+	_ afero.Fs      = (*languageCompositeFs)(nil)
+	_ afero.Lstater = (*languageCompositeFs)(nil)
+)
+
+type languageCompositeFs struct {
+	*afero.CopyOnWriteFs
+}
+
+// NewLanguageCompositeFs creates a composite and language aware filesystem.
+// This is a hybrid filesystem. To get a specific file in Open, Stat etc., use the full filename
+// to the target filesystem. This information is available in Readdir, Stat etc. via the
+// special LanguageFileInfo FileInfo implementation.
+func NewLanguageCompositeFs(base afero.Fs, overlay *LanguageFs) afero.Fs {
+	return afero.NewReadOnlyFs(&languageCompositeFs{afero.NewCopyOnWriteFs(base, overlay).(*afero.CopyOnWriteFs)})
+}
+
+// Open takes the full path to the file in the target filesystem. If it is a directory, it gets merged
+// using the language as a weight.
+func (fs *languageCompositeFs) Open(name string) (afero.File, error) {
+	f, err := fs.CopyOnWriteFs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	fu, ok := f.(*afero.UnionFile)
+	if ok {
+		// This is a directory: Merge it.
+		fu.Merger = LanguageDirsMerger
+	}
+	return f, nil
+}

hugofs/language_composite_fs_test.go

@@ -0,0 +1,106 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"path/filepath"
+
+	"strings"
+
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompositeLanguagFsTest(t *testing.T) {
+	assert := require.New(t)
+
+	languages := map[string]bool{
+		"sv": true,
+		"en": true,
+		"nn": true,
+	}
+	msv := afero.NewMemMapFs()
+	baseSv := "/content/sv"
+	lfssv := NewLanguageFs("sv", languages, afero.NewBasePathFs(msv, baseSv))
+	mnn := afero.NewMemMapFs()
+	baseNn := "/content/nn"
+	lfsnn := NewLanguageFs("nn", languages, afero.NewBasePathFs(mnn, baseNn))
+	men := afero.NewMemMapFs()
+	baseEn := "/content/en"
+	lfsen := NewLanguageFs("en", languages, afero.NewBasePathFs(men, baseEn))
+
+	// The order will be sv, en, nn
+	composite := NewLanguageCompositeFs(lfsnn, lfsen)
+	composite = NewLanguageCompositeFs(composite, lfssv)
+
+	afero.WriteFile(msv, filepath.Join(baseSv, "f1.txt"), []byte("some sv"), 0755)
+	afero.WriteFile(mnn, filepath.Join(baseNn, "f1.txt"), []byte("some nn"), 0755)
+	afero.WriteFile(men, filepath.Join(baseEn, "f1.txt"), []byte("some en"), 0755)
+
+	// Swedish is the top layer.
+	assertLangFile(t, composite, "f1.txt", "sv")
+
+	afero.WriteFile(msv, filepath.Join(baseSv, "f2.en.txt"), []byte("some sv"), 0755)
+	afero.WriteFile(mnn, filepath.Join(baseNn, "f2.en.txt"), []byte("some nn"), 0755)
+	afero.WriteFile(men, filepath.Join(baseEn, "f2.en.txt"), []byte("some en"), 0755)
+
+	// English is in the middle, but the most specific language match wins.
+	//assertLangFile(t, composite, "f2.en.txt", "en")
+
+	// Fetch some specific language versions
+	assertLangFile(t, composite, filepath.Join(baseNn, "f2.en.txt"), "nn")
+	assertLangFile(t, composite, filepath.Join(baseEn, "f2.en.txt"), "en")
+	assertLangFile(t, composite, filepath.Join(baseSv, "f2.en.txt"), "sv")
+
+	// Read the root
+	f, err := composite.Open("/")
+	assert.NoError(err)
+	defer f.Close()
+	files, err := f.Readdir(-1)
+	assert.Equal(4, len(files))
+	expected := map[string]bool{
+		filepath.FromSlash("/content/en/f1.txt"):    true,
+		filepath.FromSlash("/content/nn/f1.txt"):    true,
+		filepath.FromSlash("/content/sv/f1.txt"):    true,
+		filepath.FromSlash("/content/en/f2.en.txt"): true,
+	}
+	got := make(map[string]bool)
+
+	for _, fi := range files {
+		fil, ok := fi.(*LanguageFileInfo)
+		assert.True(ok)
+		got[fil.Filename()] = true
+	}
+	assert.Equal(expected, got)
+}
+
+func assertLangFile(t testing.TB, fs afero.Fs, filename, match string) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	b, err := afero.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := string(b)
+	if !strings.Contains(s, match) {
+		t.Fatalf("got %q expected it to contain %q", s, match)
+
+	}
+}

hugofs/language_fs.go

@@ -0,0 +1,328 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/afero"
+)
+
+const hugoFsMarker = "__hugofs"
+
+var (
+	_ LanguageAnnouncer = (*LanguageFileInfo)(nil)
+	_ FilePather        = (*LanguageFileInfo)(nil)
+	_ afero.Lstater     = (*LanguageFs)(nil)
+)
+
+// LanguageAnnouncer is aware of its language.
+type LanguageAnnouncer interface {
+	Lang() string
+	TranslationBaseName() string
+}
+
+// FilePather is aware of its file's location.
+type FilePather interface {
+	// Filename gets the full path and filename to the file.
+	Filename() string
+
+	// Path gets the content relative path including file name and extension.
+	// The directory is relative to the content root where "content" is a broad term.
+	Path() string
+
+	// RealName is FileInfo.Name in its original form.
+	RealName() string
+
+	BaseDir() string
+}
+
+var LanguageDirsMerger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
+	m := make(map[string]*LanguageFileInfo)
+
+	for _, fi := range lofi {
+		fil, ok := fi.(*LanguageFileInfo)
+		if !ok {
+			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
+		}
+		m[fil.virtualName] = fil
+	}
+
+	for _, fi := range bofi {
+		fil, ok := fi.(*LanguageFileInfo)
+		if !ok {
+			return nil, fmt.Errorf("received %T, expected *LanguageFileInfo", fi)
+		}
+		existing, found := m[fil.virtualName]
+
+		if !found || existing.weight < fil.weight {
+			m[fil.virtualName] = fil
+		}
+	}
+
+	merged := make([]os.FileInfo, len(m))
+	i := 0
+	for _, v := range m {
+		merged[i] = v
+		i++
+	}
+
+	return merged, nil
+}
+
+type LanguageFileInfo struct {
+	os.FileInfo
+	lang                string
+	baseDir             string
+	realFilename        string
+	relFilename         string
+	name                string
+	realName            string
+	virtualName         string
+	translationBaseName string
+
+	// We add some weight to the files in their own language's content directory.
+	weight int
+}
+
+func (fi *LanguageFileInfo) Filename() string {
+	return fi.realFilename
+}
+
+func (fi *LanguageFileInfo) Path() string {
+	return fi.relFilename
+}
+
+func (fi *LanguageFileInfo) RealName() string {
+	return fi.realName
+}
+
+func (fi *LanguageFileInfo) BaseDir() string {
+	return fi.baseDir
+}
+
+func (fi *LanguageFileInfo) Lang() string {
+	return fi.lang
+}
+
+// TranslationBaseName returns the base filename without any extension or language
+// identificator.
+func (fi *LanguageFileInfo) TranslationBaseName() string {
+	return fi.translationBaseName
+}
+
+// Name is the name of the file within this filesystem without any path info.
+// It will be marked with language information so we can identify it as ours.
+func (fi *LanguageFileInfo) Name() string {
+	return fi.name
+}
+
+type languageFile struct {
+	afero.File
+	fs *LanguageFs
+}
+
+// Readdir creates FileInfo entries by calling Lstat if possible.
+func (l *languageFile) Readdir(c int) (ofi []os.FileInfo, err error) {
+	names, err := l.File.Readdirnames(c)
+	if err != nil {
+		return nil, err
+	}
+
+	fis := make([]os.FileInfo, len(names))
+
+	for i, name := range names {
+		fi, _, err := l.fs.LstatIfPossible(filepath.Join(l.Name(), name))
+
+		if err != nil {
+			return nil, err
+		}
+		fis[i] = fi
+	}
+
+	return fis, err
+}
+
+type LanguageFs struct {
+	// This Fs is usually created with a BasePathFs
+	basePath   string
+	lang       string
+	nameMarker string
+	languages  map[string]bool
+	afero.Fs
+}
+
+func NewLanguageFs(lang string, languages map[string]bool, fs afero.Fs) *LanguageFs {
+	if lang == "" {
+		panic("no lang set for the language fs")
+	}
+	var basePath string
+
+	if bfs, ok := fs.(*afero.BasePathFs); ok {
+		basePath, _ = bfs.RealPath("")
+	}
+
+	marker := hugoFsMarker + "_" + lang + "_"
+
+	return &LanguageFs{lang: lang, languages: languages, basePath: basePath, Fs: fs, nameMarker: marker}
+}
+
+func (fs *LanguageFs) Lang() string {
+	return fs.lang
+}
+
+func (fs *LanguageFs) Stat(name string) (os.FileInfo, error) {
+	name, err := fs.realName(name)
+	if err != nil {
+		return nil, err
+	}
+
+	fi, err := fs.Fs.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return fs.newLanguageFileInfo(name, fi)
+}
+
+func (fs *LanguageFs) Open(name string) (afero.File, error) {
+	name, err := fs.realName(name)
+	if err != nil {
+		return nil, err
+	}
+	f, err := fs.Fs.Open(name)
+
+	if err != nil {
+		return nil, err
+	}
+	return &languageFile{File: f, fs: fs}, nil
+}
+
+func (fs *LanguageFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	name, err := fs.realName(name)
+	if err != nil {
+		return nil, false, err
+	}
+
+	var fi os.FileInfo
+	var b bool
+
+	if lif, ok := fs.Fs.(afero.Lstater); ok {
+		fi, b, err = lif.LstatIfPossible(name)
+	} else {
+		fi, err = fs.Fs.Stat(name)
+	}
+
+	if err != nil {
+		return nil, b, err
+	}
+
+	lfi, err := fs.newLanguageFileInfo(name, fi)
+
+	return lfi, b, err
+}
+
+func (fs *LanguageFs) realPath(name string) (string, error) {
+	if baseFs, ok := fs.Fs.(*afero.BasePathFs); ok {
+		return baseFs.RealPath(name)
+	}
+	return name, nil
+}
+
+func (fs *LanguageFs) realName(name string) (string, error) {
+	if strings.Contains(name, hugoFsMarker) {
+		if !strings.Contains(name, fs.nameMarker) {
+			return "", os.ErrNotExist
+		}
+		return strings.Replace(name, fs.nameMarker, "", 1), nil
+	}
+
+	if fs.basePath == "" {
+		return name, nil
+	}
+
+	return strings.TrimPrefix(name, fs.basePath), nil
+}
+
+func (fs *LanguageFs) newLanguageFileInfo(filename string, fi os.FileInfo) (*LanguageFileInfo, error) {
+	filename = filepath.Clean(filename)
+	_, name := filepath.Split(filename)
+
+	realName := name
+	virtualName := name
+
+	realPath, err := fs.realPath(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	lang := fs.Lang()
+
+	baseNameNoExt := ""
+
+	if !fi.IsDir() {
+
+		// Try to extract the language from the file name.
+		// Any valid language identificator in the name will win over the
+		// language set on the file system, e.g. "mypost.en.md".
+		baseName := filepath.Base(name)
+		ext := filepath.Ext(baseName)
+		baseNameNoExt = baseName
+
+		if ext != "" {
+			baseNameNoExt = strings.TrimSuffix(baseNameNoExt, ext)
+		}
+
+		fileLangExt := filepath.Ext(baseNameNoExt)
+		fileLang := strings.TrimPrefix(fileLangExt, ".")
+
+		if fs.languages[fileLang] {
+			lang = fileLang
+		}
+
+		baseNameNoExt = strings.TrimSuffix(baseNameNoExt, fileLangExt)
+
+		// This connects the filename to the filesystem, not the language.
+		virtualName = baseNameNoExt + "." + lang + ext
+
+		name = fs.nameMarker + name
+	}
+
+	weight := 1
+	// If this file's language belongs in this directory, add some weight to it
+	// to make it more important.
+	if lang == fs.Lang() {
+		weight = 2
+	}
+
+	if fi.IsDir() {
+		// For directories we always want to start from the union view.
+		realPath = strings.TrimPrefix(realPath, fs.basePath)
+	}
+
+	return &LanguageFileInfo{
+		lang:                lang,
+		weight:              weight,
+		realFilename:        realPath,
+		realName:            realName,
+		relFilename:         strings.TrimPrefix(strings.TrimPrefix(realPath, fs.basePath), string(os.PathSeparator)),
+		name:                name,
+		virtualName:         virtualName,
+		translationBaseName: baseNameNoExt,
+		baseDir:             fs.basePath,
+		FileInfo:            fi}, nil
+}

hugofs/language_fs_test.go

@@ -0,0 +1,54 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLanguagFs(t *testing.T) {
+	languages := map[string]bool{
+		"sv": true,
+	}
+	base := filepath.FromSlash("/my/base")
+	assert := require.New(t)
+	m := afero.NewMemMapFs()
+	bfs := afero.NewBasePathFs(m, base)
+	lfs := NewLanguageFs("sv", languages, bfs)
+	assert.NotNil(lfs)
+	assert.Equal("sv", lfs.Lang())
+	err := afero.WriteFile(lfs, filepath.FromSlash("sect/page.md"), []byte("abc"), 0777)
+	assert.NoError(err)
+	fi, err := lfs.Stat(filepath.FromSlash("sect/page.md"))
+	assert.NoError(err)
+	assert.Equal("__hugofs_sv_page.md", fi.Name())
+
+	languager, ok := fi.(LanguageAnnouncer)
+	assert.True(ok)
+
+	assert.Equal("sv", languager.Lang())
+
+	lfi, ok := fi.(*LanguageFileInfo)
+	assert.True(ok)
+	assert.Equal(filepath.FromSlash("/my/base/sect/page.md"), lfi.Filename())
+	assert.Equal(filepath.FromSlash("sect/page.md"), lfi.Path())
+	assert.Equal("page.sv.md", lfi.virtualName)
+	assert.Equal("__hugofs_sv_page.md", lfi.Name())
+	assert.Equal("page.md", lfi.RealName())
+
+}

hugolib/config.go

@@ -130,22 +130,18 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 	} else {
 		languages = make(map[string]interface{})
 		for k, v := range languagesFromConfig {
-			isDisabled := false
 			for _, disabled := range disableLanguages {
 				if disabled == defaultLang {
 					return fmt.Errorf("cannot disable default language %q", defaultLang)
 				}
 
 				if strings.EqualFold(k, disabled) {
-					isDisabled = true
+					v.(map[string]interface{})["disabled"] = true
 					break
 				}
 			}
-			if !isDisabled {
 			languages[k] = v
 		}
-
-		}
 	}
 
 	var (

hugolib/disableKinds_test.go

@@ -104,8 +104,8 @@ categories:
 
 	writeSource(t, fs, "content/sect/p1.md", fmt.Sprintf(pageTemplate, "P1", "- tag1"))
 
-	writeNewContentFile(t, fs, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
-	writeNewContentFile(t, fs, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10)
+	writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
+	writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/tag1/_index.md", 10)
 
 	h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})
 

hugolib/fileInfo.go

@@ -14,7 +14,6 @@
 package hugolib
 
 import (
-	"os"
 	"strings"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -25,11 +24,22 @@ import (
 var (
 	_ source.File         = (*fileInfo)(nil)
 	_ source.ReadableFile = (*fileInfo)(nil)
+	_ pathLangFile        = (*fileInfo)(nil)
 )
 
+// A partial interface to prevent ambigous compiler error.
+type basePather interface {
+	Filename() string
+	RealName() string
+	BaseDir() string
+}
+
 type fileInfo struct {
 	bundleTp bundleDirType
+
 	source.ReadableFile
+	basePather
+
 	overriddenLang string
 
 	// Set if the content language for this file is disabled.
@@ -43,6 +53,10 @@ func (fi *fileInfo) Lang() string {
 	return fi.ReadableFile.Lang()
 }
 
+func (fi *fileInfo) Filename() string {
+	return fi.basePather.Filename()
+}
+
 func (fi *fileInfo) isOwner() bool {
 	return fi.bundleTp > bundleNot
 }
@@ -55,12 +69,13 @@ func (fi *fileInfo) isContentFile() bool {
 	return contentFileExtensionsSet[fi.Ext()]
 }
 
-func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi os.FileInfo, tp bundleDirType) *fileInfo {
+func newFileInfo(sp *source.SourceSpec, baseDir, filename string, fi pathLangFileFi, tp bundleDirType) *fileInfo {
 
 	baseFi := sp.NewFileInfo(baseDir, filename, tp == bundleLeaf, fi)
 	f := &fileInfo{
 		bundleTp:     tp,
 		ReadableFile: baseFi,
+		basePather:   fi,
 	}
 
 	lang := f.Lang()

hugolib/fileInfo_test.go

@@ -1,61 +0,0 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugolib
-
-import (
-	"testing"
-
-	"path/filepath"
-
-	"github.com/gohugoio/hugo/source"
-	"github.com/stretchr/testify/require"
-)
-
-func TestBundleFileInfo(t *testing.T) {
-	t.Parallel()
-
-	assert := require.New(t)
-	cfg, fs := newTestBundleSourcesMultilingual(t)
-	sourceSpec := source.NewSourceSpec(cfg, fs)
-
-	for _, this := range []struct {
-		filename string
-		check    func(f *fileInfo)
-	}{
-		{"/path/to/file.md", func(fi *fileInfo) {
-			assert.Equal("md", fi.Ext())
-			assert.Equal("en", fi.Lang())
-			assert.False(fi.isOwner())
-			assert.True(fi.isContentFile())
-		}},
-		{"/path/to/file.JPG", func(fi *fileInfo) {
-			assert.Equal("jpg", fi.Ext())
-			assert.False(fi.isContentFile())
-		}},
-		{"/path/to/file.nn.png", func(fi *fileInfo) {
-			assert.Equal("png", fi.Ext())
-			assert.Equal("nn", fi.Lang())
-			assert.Equal("file", fi.TranslationBaseName())
-			assert.False(fi.isContentFile())
-		}},
-	} {
-		fi := newFileInfo(
-			sourceSpec,
-			filepath.FromSlash("/work/base"),
-			filepath.FromSlash(this.filename),
-			nil, bundleNot)
-		this.check(fi)
-	}
-
-}

hugolib/hugo_sites.go

@@ -75,19 +75,8 @@ func (h *HugoSites) langSite() map[string]*Site {
 // GetContentPage finds a Page with content given the absolute filename.
 // Returns nil if none found.
 func (h *HugoSites) GetContentPage(filename string) *Page {
-	s := h.Sites[0]
-	contendDir := filepath.Join(s.PathSpec.AbsPathify(s.Cfg.GetString("contentDir")))
-	if !strings.HasPrefix(filename, contendDir) {
-		return nil
-	}
-
-	rel := strings.TrimPrefix(filename, contendDir)
-	rel = strings.TrimPrefix(rel, helpers.FilePathSeparator)
-
 	for _, s := range h.Sites {
-
-		pos := s.rawAllPages.findPagePosByFilePath(rel)
-
+		pos := s.rawAllPages.findPagePosByFilename(filename)
 		if pos == -1 {
 			continue
 		}
@@ -95,19 +84,16 @@ func (h *HugoSites) GetContentPage(filename string) *Page {
 	}
 
 	// If not found already, this may be bundled in another content file.
-	rel = filepath.Dir(rel)
+	dir := filepath.Dir(filename)
+
 	for _, s := range h.Sites {
-
-		pos := s.rawAllPages.findFirstPagePosByFilePathPrefix(rel)
-
+		pos := s.rawAllPages.findPagePosByFilnamePrefix(dir)
 		if pos == -1 {
 			continue
 		}
 		return s.rawAllPages[pos]
 	}
-
 	return nil
-
 }
 
 // NewHugoSites creates a new collection of sites given the input sites, building
@@ -126,17 +112,10 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 
 	var contentChangeTracker *contentChangeMap
 
-	// Only needed in server mode.
-	// TODO(bep) clean up the running vs watching terms
-	if cfg.Running {
-		contentChangeTracker = &contentChangeMap{symContent: make(map[string]map[string]bool)}
-	}
-
 	h := &HugoSites{
 		running:      cfg.Running,
 		multilingual: langConfig,
 		multihost:    cfg.Cfg.GetBool("multihost"),
-		ContentChanges: contentChangeTracker,
 		Sites:        sites}
 
 	for _, s := range sites {
@@ -149,6 +128,13 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 
 	h.Deps = sites[0].Deps
 
+	// Only needed in server mode.
+	// TODO(bep) clean up the running vs watching terms
+	if cfg.Running {
+		contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)}
+		h.ContentChanges = contentChangeTracker
+	}
+
 	if err := h.initGitInfo(); err != nil {
 		return nil, err
 	}
@@ -212,6 +198,7 @@ func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error {
 			d.OutputFormatsConfig = s.outputFormatsConfig
 			s.Deps = d
 		}
+
 		s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig)
 		if err != nil {
 			return err
@@ -260,6 +247,9 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
 	languages := getLanguages(cfg.Cfg)
 
 	for _, lang := range languages {
+		if lang.Disabled {
+			continue
+		}
 		var s *Site
 		var err error
 		cfg.Language = lang
@@ -517,9 +507,9 @@ func (h *HugoSites) createMissingPages() error {
 	return nil
 }
 
-func (h *HugoSites) removePageByPath(path string) {
+func (h *HugoSites) removePageByFilename(filename string) {
 	for _, s := range h.Sites {
-		s.removePageByPath(path)
+		s.removePageFilename(filename)
 	}
 }
 
@@ -671,6 +661,8 @@ type contentChangeMap struct {
 	branches []string
 	leafs    []string
 
+	pathSpec *helpers.PathSpec
+
 	// Hugo supports symlinked content (both directories and files). This
 	// can lead to situations where the same file can be referenced from several
 	// locations in /content -- which is really cool, but also means we have to
@@ -683,7 +675,7 @@ type contentChangeMap struct {
 
 func (m *contentChangeMap) add(filename string, tp bundleDirType) {
 	m.mu.Lock()
-	dir := filepath.Dir(filename)
+	dir := filepath.Dir(filename) + helpers.FilePathSeparator
 	switch tp {
 	case bundleBranch:
 		m.branches = append(m.branches, dir)
@@ -698,7 +690,7 @@ func (m *contentChangeMap) add(filename string, tp bundleDirType) {
 // Track the addition of bundle dirs.
 func (m *contentChangeMap) handleBundles(b *bundleDirs) {
 	for _, bd := range b.bundles {
-		m.add(bd.fi.Filename(), bd.tp)
+		m.add(bd.fi.Path(), bd.tp)
 	}
 }
 
@@ -709,21 +701,21 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 
-	dir, name := filepath.Split(filename)
-	dir = strings.TrimSuffix(dir, helpers.FilePathSeparator)
-	fileTp, isContent := classifyBundledFile(name)
-
-	// If the file itself is a bundle, no need to look further:
-	if fileTp > bundleNot {
-		return dir, dir, fileTp
+	// Bundles share resources, so we need to start from the virtual root.
+	relPath, _ := m.pathSpec.RelContentDir(filename)
+	dir, name := filepath.Split(relPath)
+	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
+		dir += helpers.FilePathSeparator
 	}
 
+	fileTp, _ := classifyBundledFile(name)
+
 	// This may be a member of a bundle. Start with branch bundles, the most specific.
-	if !isContent {
+	if fileTp != bundleLeaf {
 		for i, b := range m.branches {
 			if b == dir {
 				m.branches = append(m.branches[:i], m.branches[i+1:]...)
-				return dir, dir, bundleBranch
+				return dir, b, bundleBranch
 			}
 		}
 	}
@@ -732,7 +724,7 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu
 	for i, l := range m.leafs {
 		if strings.HasPrefix(dir, l) {
 			m.leafs = append(m.leafs[:i], m.leafs[i+1:]...)
-			return dir, dir, bundleLeaf
+			return dir, l, bundleLeaf
 		}
 	}
 

hugolib/hugo_sites_build_test.go

@@ -3,6 +3,7 @@ package hugolib
 import (
 	"bytes"
 	"fmt"
+	"io"
 	"strings"
 	"testing"
 
@@ -433,7 +434,7 @@ func TestMultiSitesRebuild(t *testing.T) {
 	// t.Parallel() not supported, see https://github.com/fortytw2/leaktest/issues/4
 	// This leaktest seems to be a little bit shaky on Travis.
 	if !isCI() {
-		defer leaktest.CheckTimeout(t, 30*time.Second)()
+		defer leaktest.CheckTimeout(t, 10*time.Second)()
 	}
 
 	assert := require.New(t)
@@ -459,6 +460,8 @@ func TestMultiSitesRebuild(t *testing.T) {
 	b.AssertFileContent("public/fr/sect/doc1/index.html", "Single", "Shortcode: Bonjour")
 	b.AssertFileContent("public/en/sect/doc1-slug/index.html", "Single", "Shortcode: Hello")
 
+	contentFs := b.H.BaseFs.ContentFs
+
 	for i, this := range []struct {
 		preFunc    func(t *testing.T)
 		events     []fsnotify.Event
@@ -490,9 +493,9 @@ func TestMultiSitesRebuild(t *testing.T) {
 		},
 		{
 			func(t *testing.T) {
-				writeNewContentFile(t, fs, "new_en_1", "2016-07-31", "content/new1.en.md", -5)
-				writeNewContentFile(t, fs, "new_en_2", "1989-07-30", "content/new2.en.md", -10)
-				writeNewContentFile(t, fs, "new_fr_1", "2016-07-30", "content/new1.fr.md", 10)
+				writeNewContentFile(t, contentFs, "new_en_1", "2016-07-31", "new1.en.md", -5)
+				writeNewContentFile(t, contentFs, "new_en_2", "1989-07-30", "new2.en.md", -10)
+				writeNewContentFile(t, contentFs, "new_fr_1", "2016-07-30", "new1.fr.md", 10)
 			},
 			[]fsnotify.Event{
 				{Name: filepath.FromSlash("content/new1.en.md"), Op: fsnotify.Create},
@@ -513,10 +516,10 @@ func TestMultiSitesRebuild(t *testing.T) {
 		},
 		{
 			func(t *testing.T) {
-				p := "content/sect/doc1.en.md"
-				doc1 := readSource(t, fs, p)
+				p := "sect/doc1.en.md"
+				doc1 := readFileFromFs(t, contentFs, p)
 				doc1 += "CHANGED"
-				writeSource(t, fs, p, doc1)
+				writeToFs(t, contentFs, p, doc1)
 			},
 			[]fsnotify.Event{{Name: filepath.FromSlash("content/sect/doc1.en.md"), Op: fsnotify.Write}},
 			func(t *testing.T) {
@@ -529,7 +532,7 @@ func TestMultiSitesRebuild(t *testing.T) {
 		// Rename a file
 		{
 			func(t *testing.T) {
-				if err := fs.Source.Rename("content/new1.en.md", "content/new1renamed.en.md"); err != nil {
+				if err := contentFs.Rename("new1.en.md", "new1renamed.en.md"); err != nil {
 					t.Fatalf("Rename failed: %s", err)
 				}
 			},
@@ -650,7 +653,7 @@ weight = 15
 title = "Svenska"
 `
 
-	writeNewContentFile(t, fs, "Swedish Contentfile", "2016-01-01", "content/sect/doc1.sv.md", 10)
+	writeNewContentFile(t, fs.Source, "Swedish Contentfile", "2016-01-01", "content/sect/doc1.sv.md", 10)
 	// replace the config
 	b.WithNewConfig(newConfig)
 
@@ -1038,18 +1041,31 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
 	if err != nil {
 		// Print some debug info
 		root := strings.Split(filename, helpers.FilePathSeparator)[0]
-		afero.Walk(fs, root, func(path string, info os.FileInfo, err error) error {
-			if info != nil && !info.IsDir() {
-				fmt.Println("    ", path)
-			}
-
-			return nil
-		})
+		printFs(fs, root, os.Stdout)
 		Fatalf(t, "Failed to read file: %s", err)
 	}
 	return string(b)
 }
 
+func printFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+		if info != nil && !info.IsDir() {
+			s := path
+			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+				s = s + "\tLANG: " + lang.Lang()
+			}
+			if fp, ok := info.(hugofs.FilePather); ok {
+				s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir()
+			}
+			fmt.Fprintln(w, "    ", s)
+		}
+		return nil
+	})
+}
+
 const testPageTemplate = `---
 title: "%s"
 publishdate: "%s"
@@ -1062,9 +1078,9 @@ func newTestPage(title, date string, weight int) string {
 	return fmt.Sprintf(testPageTemplate, title, date, weight, title)
 }
 
-func writeNewContentFile(t *testing.T, fs *hugofs.Fs, title, date, filename string, weight int) {
+func writeNewContentFile(t *testing.T, fs afero.Fs, title, date, filename string, weight int) {
 	content := newTestPage(title, date, weight)
-	writeSource(t, fs, filename, content)
+	writeToFs(t, fs, filename, content)
 }
 
 type multiSiteTestBuilder struct {

hugolib/language_content_dir_test.go

@@ -0,0 +1,253 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+/*
+
+/en/p1.md
+/nn/p1.md
+
+.Readdir
+
+- Name() => p1.en.md, p1.nn.md
+
+.Stat(name)
+
+.Open() --- real file name
+
+
+*/
+
+func TestLanguageContentRoot(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	config := `
+baseURL = "https://example.org/"
+
+defaultContentLanguage = "en"
+defaultContentLanguageInSubdir = true
+
+contentDir = "content/main"
+workingDir = "/my/project"
+
+[Languages]
+[Languages.en]
+weight = 10
+title = "In English"
+languageName = "English"
+
+[Languages.nn]
+weight = 20
+title = "På Norsk"
+languageName = "Norsk"
+# This tells Hugo that all content in this directory is in the Norwegian language.
+# It does not have to have the "my-page.nn.md" format. It can, but that is optional.
+contentDir = "content/norsk"
+
+[Languages.sv]
+weight = 30
+title = "På Svenska"
+languageName = "Svensk"
+contentDir = "content/svensk"
+`
+
+	pageTemplate := `
+---
+title: %s
+slug: %s
+weight: %d
+---
+
+Content.
+
+`
+
+	pageBundleTemplate := `
+---
+title: %s
+weight: %d
+---
+
+Content.
+
+`
+	var contentFiles []string
+	section := "sect"
+
+	var contentRoot = func(lang string) string {
+		contentRoot := "content/main"
+
+		switch lang {
+		case "nn":
+			contentRoot = "content/norsk"
+		case "sv":
+			contentRoot = "content/svensk"
+		}
+		return contentRoot + "/" + section
+	}
+
+	for _, lang := range []string{"en", "nn", "sv"} {
+		for j := 1; j <= 10; j++ {
+			if (lang == "nn" || lang == "en") && j%4 == 0 {
+				// Skip 4 and 8 for nn
+				// We also skip it for en, but that is added to the Swedish directory below.
+				continue
+			}
+
+			if lang == "sv" && j%5 == 0 {
+				// Skip 5 and 10 for sv
+				continue
+			}
+
+			base := fmt.Sprintf("p-%s-%d", lang, j)
+			slug := fmt.Sprintf("%s", base)
+			langID := ""
+
+			if lang == "sv" && j%4 == 0 {
+				// Put an English page in the Swedish content dir.
+				langID = ".en"
+			}
+
+			if lang == "en" && j == 8 {
+				// This should win over the sv variant above.
+				langID = ".en"
+			}
+
+			slug += langID
+
+			contentRoot := contentRoot(lang)
+
+			filename := filepath.Join(contentRoot, fmt.Sprintf("page%d%s.md", j, langID))
+			contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, j))
+		}
+	}
+
+	// Put common translations in all of them
+	for i, lang := range []string{"en", "nn", "sv"} {
+		contentRoot := contentRoot(lang)
+
+		slug := fmt.Sprintf("common_%s", lang)
+
+		filename := filepath.Join(contentRoot, "common.md")
+		contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, slug, slug, 100+i))
+
+		for j, lang2 := range []string{"en", "nn", "sv"} {
+			filename := filepath.Join(contentRoot, fmt.Sprintf("translated_all.%s.md", lang2))
+			langSlug := slug + "_translated_all_" + lang2
+			contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, langSlug, langSlug, 200+i+j))
+		}
+
+		for j, lang2 := range []string{"sv", "nn"} {
+			if lang == "en" {
+				continue
+			}
+			filename := filepath.Join(contentRoot, fmt.Sprintf("translated_some.%s.md", lang2))
+			langSlug := slug + "_translated_some_" + lang2
+			contentFiles = append(contentFiles, filename, fmt.Sprintf(pageTemplate, langSlug, langSlug, 300+i+j))
+		}
+	}
+
+	// Add a bundle with some images
+	for i, lang := range []string{"en", "nn", "sv"} {
+		contentRoot := contentRoot(lang)
+		slug := fmt.Sprintf("bundle_%s", lang)
+		filename := filepath.Join(contentRoot, "mybundle", "index.md")
+		contentFiles = append(contentFiles, filename, fmt.Sprintf(pageBundleTemplate, slug, 400+i))
+		if lang == "en" {
+			imageFilename := filepath.Join(contentRoot, "mybundle", "logo.png")
+			contentFiles = append(contentFiles, imageFilename, "PNG Data")
+		}
+		imageFilename := filepath.Join(contentRoot, "mybundle", "featured.png")
+		contentFiles = append(contentFiles, imageFilename, fmt.Sprintf("PNG Data for %s", lang))
+
+		// Add some bundled pages
+		contentFiles = append(contentFiles, filepath.Join(contentRoot, "mybundle", "p1.md"), fmt.Sprintf(pageBundleTemplate, slug, 401+i))
+		contentFiles = append(contentFiles, filepath.Join(contentRoot, "mybundle", "sub", "p1.md"), fmt.Sprintf(pageBundleTemplate, slug, 402+i))
+
+	}
+
+	b := newTestSitesBuilder(t)
+	b.WithWorkingDir("/my/project").WithConfigFile("toml", config).WithContent(contentFiles...).CreateSites()
+
+	_ = os.Stdout
+	//printFs(b.H.BaseFs.ContentFs, "/", os.Stdout)
+
+	b.Build(BuildCfg{})
+
+	assert.Equal(3, len(b.H.Sites))
+
+	enSite := b.H.Sites[0]
+	nnSite := b.H.Sites[1]
+	svSite := b.H.Sites[2]
+
+	//dumpPages(nnSite.RegularPages...)
+	assert.Equal(12, len(nnSite.RegularPages))
+	assert.Equal(13, len(enSite.RegularPages))
+
+	assert.Equal(10, len(svSite.RegularPages))
+
+	for i, p := range enSite.RegularPages {
+		j := i + 1
+		msg := fmt.Sprintf("Test %d", j)
+		assert.Equal("en", p.Lang(), msg)
+		assert.Equal("sect", p.Section())
+		if j < 9 {
+			if j%4 == 0 {
+				assert.Contains(p.Title(), fmt.Sprintf("p-sv-%d.en", i+1), msg)
+			} else {
+				assert.Contains(p.Title(), "p-en", msg)
+			}
+		}
+	}
+
+	// Check bundles
+	bundleEn := enSite.RegularPages[len(enSite.RegularPages)-1]
+	bundleNn := nnSite.RegularPages[len(nnSite.RegularPages)-1]
+	bundleSv := svSite.RegularPages[len(svSite.RegularPages)-1]
+
+	assert.Equal("/en/sect/mybundle/", bundleEn.RelPermalink())
+	assert.Equal("/sv/sect/mybundle/", bundleSv.RelPermalink())
+
+	assert.Equal(4, len(bundleEn.Resources))
+	assert.Equal(4, len(bundleNn.Resources))
+	assert.Equal(4, len(bundleSv.Resources))
+
+	assert.Equal("/en/sect/mybundle/logo.png", bundleEn.Resources.GetMatch("logo*").RelPermalink())
+	assert.Equal("/nn/sect/mybundle/logo.png", bundleNn.Resources.GetMatch("logo*").RelPermalink())
+	assert.Equal("/sv/sect/mybundle/logo.png", bundleSv.Resources.GetMatch("logo*").RelPermalink())
+
+	b.AssertFileContent("/my/project/public/sv/sect/mybundle/featured.png", "PNG Data for sv")
+	b.AssertFileContent("/my/project/public/nn/sect/mybundle/featured.png", "PNG Data for nn")
+	b.AssertFileContent("/my/project/public/en/sect/mybundle/featured.png", "PNG Data for en")
+	b.AssertFileContent("/my/project/public/en/sect/mybundle/logo.png", "PNG Data")
+	b.AssertFileContent("/my/project/public/sv/sect/mybundle/logo.png", "PNG Data")
+	b.AssertFileContent("/my/project/public/nn/sect/mybundle/logo.png", "PNG Data")
+
+	nnSect := nnSite.getPage(KindSection, "sect")
+	assert.NotNil(nnSect)
+	assert.Equal(12, len(nnSect.Pages))
+	nnHome, _ := nnSite.Info.Home()
+	assert.Equal("/nn/", nnHome.RelPermalink())
+
+}

hugolib/menu_test.go

@@ -74,8 +74,8 @@ Menu Main:  {{ partial "menu.html" (dict "page" . "menu" "main") }}`,
 	writeSource(t, fs, "content/sect2/p4.md", fmt.Sprintf(menuPageTemplate, "p4", 4, "main", "atitle4", 10))
 	writeSource(t, fs, "content/sect3/p5.md", fmt.Sprintf(menuPageTemplate, "p5", 5, "main", "atitle5", 5))
 
-	writeNewContentFile(t, fs, "Section One", "2017-01-01", "content/sect1/_index.md", 100)
-	writeNewContentFile(t, fs, "Section Five", "2017-01-01", "content/sect5/_index.md", 10)
+	writeNewContentFile(t, fs.Source, "Section One", "2017-01-01", "content/sect1/_index.md", 100)
+	writeNewContentFile(t, fs.Source, "Section Five", "2017-01-01", "content/sect5/_index.md", 10)
 
 	err := h.Build(BuildCfg{})
 

hugolib/multilingual.go

@@ -111,6 +111,10 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L
 				language.LanguageName = cast.ToString(v)
 			case "weight":
 				language.Weight = cast.ToInt(v)
+			case "contentdir":
+				language.ContentDir = cast.ToString(v)
+			case "disabled":
+				language.Disabled = cast.ToBool(v)
 			case "params":
 				m := cast.ToStringMap(v)
 				// Needed for case insensitive fetching of params values

hugolib/page.go

@@ -388,9 +388,9 @@ func (ps Pages) String() string {
 	return fmt.Sprintf("Pages(%d)", len(ps))
 }
 
-func (ps Pages) findPagePosByFilePath(inPath string) int {
+func (ps Pages) findPagePosByFilename(filename string) int {
 	for i, x := range ps {
-		if x.Source.Path() == inPath {
+		if x.Source.Filename() == filename {
 			return i
 		}
 	}
@@ -412,16 +412,26 @@ func (ps Pages) removeFirstIfFound(p *Page) Pages {
 	return ps
 }
 
-func (ps Pages) findFirstPagePosByFilePathPrefix(prefix string) int {
+func (ps Pages) findPagePosByFilnamePrefix(prefix string) int {
 	if prefix == "" {
 		return -1
 	}
+
+	lenDiff := -1
+	currPos := -1
+	prefixLen := len(prefix)
+
+	// Find the closest match
 	for i, x := range ps {
-		if strings.HasPrefix(x.Source.Path(), prefix) {
-			return i
+		if strings.HasPrefix(x.Source.Filename(), prefix) {
+			diff := len(x.Source.Filename()) - prefixLen
+			if lenDiff == -1 || diff < lenDiff {
+				lenDiff = diff
+				currPos = i
 			}
 		}
-	return -1
+	}
+	return currPos
 }
 
 // findPagePos Given a page, it will find the position in Pages

hugolib/page_bundler.go

@@ -17,7 +17,6 @@ import (
 	"fmt"
 	"math"
 	"runtime"
-	"strings"
 
 	// Use this until errgroup gets ported to context
 	// See https://github.com/golang/go/issues/19781
@@ -26,8 +25,6 @@ import (
 )
 
 type siteContentProcessor struct {
-	baseDir string
-
 	site *Site
 
 	handleContent contentHandler
@@ -41,7 +38,7 @@ type siteContentProcessor struct {
 	fileSinglesChan chan *fileInfo
 
 	// These assets should be just copied to destination.
-	fileAssetsChan chan []string
+	fileAssetsChan chan []pathLangFile
 
 	numWorkers int
 
@@ -67,14 +64,14 @@ func (s *siteContentProcessor) processSingle(fi *fileInfo) {
 	}
 }
 
-func (s *siteContentProcessor) processAssets(assets []string) {
+func (s *siteContentProcessor) processAssets(assets []pathLangFile) {
 	select {
 	case s.fileAssetsChan <- assets:
 	case <-s.ctx.Done():
 	}
 }
 
-func newSiteContentProcessor(ctx context.Context, baseDir string, partialBuild bool, s *Site) *siteContentProcessor {
+func newSiteContentProcessor(ctx context.Context, partialBuild bool, s *Site) *siteContentProcessor {
 	numWorkers := 12
 	if n := runtime.NumCPU() * 3; n > numWorkers {
 		numWorkers = n
@@ -85,12 +82,11 @@ func newSiteContentProcessor(ctx context.Context, baseDir string, partialBuild b
 	return &siteContentProcessor{
 		ctx:             ctx,
 		partialBuild:    partialBuild,
-		baseDir:         baseDir,
 		site:            s,
 		handleContent:   newHandlerChain(s),
 		fileBundlesChan: make(chan *bundleDir, numWorkers),
 		fileSinglesChan: make(chan *fileInfo, numWorkers),
-		fileAssetsChan:  make(chan []string, numWorkers),
+		fileAssetsChan:  make(chan []pathLangFile, numWorkers),
 		numWorkers:      numWorkers,
 		pagesChan:       make(chan *Page, numWorkers),
 	}
@@ -143,18 +139,16 @@ func (s *siteContentProcessor) process(ctx context.Context) error {
 		g2.Go(func() error {
 			for {
 				select {
-				case filenames, ok := <-s.fileAssetsChan:
+				case files, ok := <-s.fileAssetsChan:
 					if !ok {
 						return nil
 					}
-					for _, filename := range filenames {
-						name := strings.TrimPrefix(filename, s.baseDir)
-						f, err := s.site.Fs.Source.Open(filename)
+					for _, file := range files {
+						f, err := s.site.BaseFs.ContentFs.Open(file.Filename())
 						if err != nil {
-							return err
+							return fmt.Errorf("failed to open assets file: %s", err)
 						}
-
-						err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, name, f)
+						err = s.site.publish(&s.site.PathSpec.ProcessingStats.Files, file.Path(), f)
 						f.Close()
 						if err != nil {
 							return err
@@ -204,11 +198,11 @@ func (s *siteContentProcessor) process(ctx context.Context) error {
 }
 
 func (s *siteContentProcessor) readAndConvertContentFile(file *fileInfo) error {
-	ctx := &handlerContext{source: file, baseDir: s.baseDir, pages: s.pagesChan}
+	ctx := &handlerContext{source: file, pages: s.pagesChan}
 	return s.handleContent(ctx).err
 }
 
 func (s *siteContentProcessor) readAndConvertContentBundle(bundle *bundleDir) error {
-	ctx := &handlerContext{bundle: bundle, baseDir: s.baseDir, pages: s.pagesChan}
+	ctx := &handlerContext{bundle: bundle, pages: s.pagesChan}
 	return s.handleContent(ctx).err
 }

hugolib/page_bundler_capture.go

@@ -17,17 +17,21 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
 
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/gohugoio/hugo/helpers"
 
 	"golang.org/x/sync/errgroup"
 
 	"github.com/gohugoio/hugo/source"
-	"github.com/spf13/afero"
 	jww "github.com/spf13/jwalterweatherman"
 )
 
@@ -44,8 +48,6 @@ type capturer struct {
 	fs         afero.Fs
 	logger     *jww.Notepad
 
-	baseDir string
-
 	// Filenames limits the content to process to a list of filenames/directories.
 	// This is used for partial building in server mode.
 	filenames []string
@@ -62,7 +64,7 @@ func newCapturer(
 	sourceSpec *source.SourceSpec,
 	handler captureResultHandler,
 	contentChanges *contentChangeMap,
-	baseDir string, filenames ...string) *capturer {
+	filenames ...string) *capturer {
 
 	numWorkers := 4
 	if n := runtime.NumCPU(); n > numWorkers {
@@ -73,9 +75,10 @@ func newCapturer(
 		sem:            make(chan bool, numWorkers),
 		handler:        handler,
 		sourceSpec:     sourceSpec,
+		fs:             sourceSpec.Fs,
 		logger:         logger,
 		contentChanges: contentChanges,
-		fs:             sourceSpec.Fs.Source, baseDir: baseDir, seen: make(map[string]bool),
+		seen:           make(map[string]bool),
 		filenames:      filenames}
 
 	return c
@@ -85,7 +88,7 @@ func newCapturer(
 // these channels.
 type captureResultHandler interface {
 	handleSingles(fis ...*fileInfo)
-	handleCopyFiles(filenames ...string)
+	handleCopyFiles(fis ...pathLangFile)
 	captureBundlesHandler
 }
 
@@ -110,10 +113,10 @@ func (c *captureResultHandlerChain) handleBundles(b *bundleDirs) {
 	}
 }
 
-func (c *captureResultHandlerChain) handleCopyFiles(filenames ...string) {
+func (c *captureResultHandlerChain) handleCopyFiles(files ...pathLangFile) {
 	for _, h := range c.handlers {
 		if hh, ok := h.(captureResultHandler); ok {
-			hh.handleCopyFiles(filenames...)
+			hh.handleCopyFiles(files...)
 		}
 	}
 }
@@ -139,7 +142,7 @@ func (c *capturer) capturePartial(filenames ...string) error {
 				return err
 			}
 		default:
-			fi, _, err := c.getRealFileInfo(resolvedFilename)
+			fi, err := c.resolveRealPath(resolvedFilename)
 			if os.IsNotExist(err) {
 				// File has been deleted.
 				continue
@@ -147,9 +150,9 @@ func (c *capturer) capturePartial(filenames ...string) error {
 
 			// Just in case the owning dir is a new symlink -- this will
 			// create the proper mapping for it.
-			c.getRealFileInfo(dir)
+			c.resolveRealPath(dir)
 
-			f, active := c.newFileInfo(resolvedFilename, fi, tp)
+			f, active := c.newFileInfo(fi, tp)
 			if active {
 				c.copyOrHandleSingle(f)
 			}
@@ -164,7 +167,7 @@ func (c *capturer) capture() error {
 		return c.capturePartial(c.filenames...)
 	}
 
-	err := c.handleDir(c.baseDir)
+	err := c.handleDir(helpers.FilePathSeparator)
 	if err != nil {
 		return err
 	}
@@ -196,6 +199,7 @@ func (c *capturer) handleNestedDir(dirname string) error {
 func (c *capturer) handleBranchDir(dirname string) error {
 	files, err := c.readDir(dirname)
 	if err != nil {
+
 		return err
 	}
 
@@ -205,7 +209,7 @@ func (c *capturer) handleBranchDir(dirname string) error {
 
 	for _, fi := range files {
 		if !fi.IsDir() {
-			tp, _ := classifyBundledFile(fi.Name())
+			tp, _ := classifyBundledFile(fi.RealName())
 			if dirType == bundleNot {
 				dirType = tp
 			}
@@ -222,25 +226,35 @@ func (c *capturer) handleBranchDir(dirname string) error {
 
 	dirs := newBundleDirs(bundleBranch, c)
 
-	for _, fi := range files {
+	var secondPass []*fileInfo
 
+	// Handle potential bundle headers first.
+	for _, fi := range files {
 		if fi.IsDir() {
 			continue
 		}
 
-		tp, isContent := classifyBundledFile(fi.Name())
+		tp, isContent := classifyBundledFile(fi.RealName())
+
+		f, active := c.newFileInfo(fi, tp)
 
-		f, active := c.newFileInfo(fi.filename, fi.FileInfo, tp)
 		if !active {
 			continue
 		}
-		if f.isOwner() {
-			dirs.addBundleHeader(f)
-		} else if !isContent {
+
+		if !f.isOwner() {
+			if !isContent {
 				// This is a partial update -- we only care about the files that
 				// is in this bundle.
-			dirs.addBundleFiles(f)
+				secondPass = append(secondPass, f)
 			}
+			continue
+		}
+		dirs.addBundleHeader(f)
+	}
+
+	for _, f := range secondPass {
+		dirs.addBundleFiles(f)
 	}
 
 	c.handler.handleBundles(dirs)
@@ -250,6 +264,7 @@ func (c *capturer) handleBranchDir(dirname string) error {
 }
 
 func (c *capturer) handleDir(dirname string) error {
+
 	files, err := c.readDir(dirname)
 	if err != nil {
 		return err
@@ -290,7 +305,8 @@ func (c *capturer) handleDir(dirname string) error {
 
 	for i, fi := range files {
 		if !fi.IsDir() {
-			tp, isContent := classifyBundledFile(fi.Name())
+			tp, isContent := classifyBundledFile(fi.RealName())
+
 			fileBundleTypes[i] = tp
 			if !isBranch {
 				isBranch = tp == bundleBranch
@@ -317,6 +333,7 @@ func (c *capturer) handleDir(dirname string) error {
 	var fileInfos = make([]*fileInfo, 0, len(files))
 
 	for i, fi := range files {
+
 		currentType := bundleNot
 
 		if !fi.IsDir() {
@@ -329,7 +346,9 @@ func (c *capturer) handleDir(dirname string) error {
 		if bundleType == bundleNot && currentType != bundleNot {
 			bundleType = currentType
 		}
-		f, active := c.newFileInfo(fi.filename, fi.FileInfo, currentType)
+
+		f, active := c.newFileInfo(fi, currentType)
+
 		if !active {
 			continue
 		}
@@ -343,8 +362,7 @@ func (c *capturer) handleDir(dirname string) error {
 		for _, fi := range fileInfos {
 			if fi.FileInfo().IsDir() {
 				// Handle potential nested bundles.
-				filename := fi.Filename()
-				if err := c.handleNestedDir(filename); err != nil {
+				if err := c.handleNestedDir(fi.Path()); err != nil {
 					return err
 				}
 			} else if bundleType == bundleNot || (!fi.isOwner() && fi.isContentFile()) {
@@ -376,23 +394,23 @@ func (c *capturer) handleDir(dirname string) error {
 
 func (c *capturer) handleNonBundle(
 	dirname string,
-	fileInfos []fileInfoName,
+	fileInfos pathLangFileFis,
 	singlesOnly bool) error {
 
 	for _, fi := range fileInfos {
 		if fi.IsDir() {
-			if err := c.handleNestedDir(fi.filename); err != nil {
+			if err := c.handleNestedDir(fi.Filename()); err != nil {
 				return err
 			}
 		} else {
 			if singlesOnly {
-				f, active := c.newFileInfo(fi.filename, fi, bundleNot)
+				f, active := c.newFileInfo(fi, bundleNot)
 				if !active {
 					continue
 				}
 				c.handler.handleSingles(f)
 			} else {
-				c.handler.handleCopyFiles(fi.filename)
+				c.handler.handleCopyFiles(fi)
 			}
 		}
 	}
@@ -405,7 +423,7 @@ func (c *capturer) copyOrHandleSingle(fi *fileInfo) {
 		c.handler.handleSingles(fi)
 	} else {
 		// These do not currently need any further processing.
-		c.handler.handleCopyFiles(fi.Filename())
+		c.handler.handleCopyFiles(fi)
 	}
 }
 
@@ -430,7 +448,7 @@ func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirT
 					fileInfos = append(fileInfos, fis...)
 				}
 			}
-			err := c.collectFiles(fi.Filename(), collector)
+			err := c.collectFiles(fi.Path(), collector)
 			if err != nil {
 				return nil, err
 			}
@@ -462,6 +480,7 @@ func (c *capturer) createBundleDirs(fileInfos []*fileInfo, bundleType bundleDirT
 }
 
 func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInfo)) error {
+
 	filesInDir, err := c.readDir(dirname)
 	if err != nil {
 		return err
@@ -469,12 +488,12 @@ func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInf
 
 	for _, fi := range filesInDir {
 		if fi.IsDir() {
-			err := c.collectFiles(fi.filename, handleFiles)
+			err := c.collectFiles(fi.Filename(), handleFiles)
 			if err != nil {
 				return err
 			}
 		} else {
-			f, active := c.newFileInfo(fi.filename, fi.FileInfo, bundleNot)
+			f, active := c.newFileInfo(fi, bundleNot)
 			if active {
 				handleFiles(f)
 			}
@@ -484,27 +503,29 @@ func (c *capturer) collectFiles(dirname string, handleFiles func(fis ...*fileInf
 	return nil
 }
 
-func (c *capturer) readDir(dirname string) ([]fileInfoName, error) {
+func (c *capturer) readDir(dirname string) (pathLangFileFis, error) {
 	if c.sourceSpec.IgnoreFile(dirname) {
 		return nil, nil
 	}
 
 	dir, err := c.fs.Open(dirname)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("readDir: %s", err)
 	}
 	defer dir.Close()
-	names, err := dir.Readdirnames(-1)
+	fis, err := dir.Readdir(-1)
 	if err != nil {
 		return nil, err
 	}
 
-	fis := make([]fileInfoName, 0, len(names))
+	pfis := make(pathLangFileFis, 0, len(fis))
 
-	for _, name := range names {
-		filename := filepath.Join(dirname, name)
-		if !c.sourceSpec.IgnoreFile(filename) {
-			fi, _, err := c.getRealFileInfo(filename)
+	for _, fi := range fis {
+		fip := fi.(pathLangFileFi)
+
+		if !c.sourceSpec.IgnoreFile(fip.Filename()) {
+
+			err := c.resolveRealPathIn(fip)
 
 			if err != nil {
 				// It may have been deleted in the meantime.
@@ -514,23 +535,30 @@ func (c *capturer) readDir(dirname string) ([]fileInfoName, error) {
 				return nil, err
 			}
 
-			fis = append(fis, fileInfoName{filename: filename, FileInfo: fi})
+			pfis = append(pfis, fip)
 		}
 	}
 
-	return fis, nil
+	return pfis, nil
 }
 
-func (c *capturer) newFileInfo(filename string, fi os.FileInfo, tp bundleDirType) (*fileInfo, bool) {
-	f := newFileInfo(c.sourceSpec, c.baseDir, filename, fi, tp)
+func (c *capturer) newFileInfo(fi pathLangFileFi, tp bundleDirType) (*fileInfo, bool) {
+	f := newFileInfo(c.sourceSpec, "", "", fi, tp)
 	return f, !f.disabled
 }
 
-type fileInfoName struct {
-	os.FileInfo
-	filename string
+type pathLangFile interface {
+	hugofs.LanguageAnnouncer
+	hugofs.FilePather
 }
 
+type pathLangFileFi interface {
+	os.FileInfo
+	pathLangFile
+}
+
+type pathLangFileFis []pathLangFileFi
+
 type bundleDirs struct {
 	tp bundleDirType
 	// Maps languages to bundles.
@@ -589,16 +617,17 @@ func (b *bundleDirs) addBundleContentFile(fi *fileInfo) {
 		b.bundles[fi.Lang()] = dir
 	}
 
-	dir.resources[fi.Filename()] = fi
+	dir.resources[fi.Path()] = fi
 }
 
 func (b *bundleDirs) addBundleFiles(fi *fileInfo) {
 	dir := filepath.ToSlash(fi.Dir())
 	p := dir + fi.TranslationBaseName() + "." + fi.Ext()
 	for lang, bdir := range b.bundles {
-		key := lang + p
+		key := path.Join(lang, p)
+
 		// Given mypage.de.md (German translation) and mypage.md we pick the most
-		// the specific for that language.
+		// specific for that language.
 		if fi.Lang() == lang || !b.langOverrides[key] {
 			bdir.resources[key] = fi
 		}
@@ -623,40 +652,53 @@ func (c *capturer) isSeen(dirname string) bool {
 	return false
 }
 
-func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) {
-	fileInfo, err := c.lstatIfOs(path)
-	realPath := path
-
+func (c *capturer) resolveRealPath(path string) (pathLangFileFi, error) {
+	fileInfo, err := c.lstatIfPossible(path)
 	if err != nil {
-		return nil, "", err
+		return nil, err
 	}
+	return fileInfo, c.resolveRealPathIn(fileInfo)
+}
+
+func (c *capturer) resolveRealPathIn(fileInfo pathLangFileFi) error {
+
+	basePath := fileInfo.BaseDir()
+	path := fileInfo.Filename()
+
+	realPath := path
 
 	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {
 		link, err := filepath.EvalSymlinks(path)
 		if err != nil {
-			return nil, "", fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err)
+			return fmt.Errorf("Cannot read symbolic link %q, error was: %s", path, err)
 		}
 
-		fileInfo, err = c.lstatIfOs(link)
+		// This is a file on the outside of any base fs, so we have to use the os package.
+		sfi, err := os.Stat(link)
 		if err != nil {
-			return nil, "", fmt.Errorf("Cannot stat  %q, error was: %s", link, err)
+			return fmt.Errorf("Cannot stat  %q, error was: %s", link, err)
+		}
+
+		// TODO(bep) improve all of this.
+		if a, ok := fileInfo.(*hugofs.LanguageFileInfo); ok {
+			a.FileInfo = sfi
 		}
 
 		realPath = link
 
-		if realPath != path && fileInfo.IsDir() && c.isSeen(realPath) {
+		if realPath != path && sfi.IsDir() && c.isSeen(realPath) {
 			// Avoid cyclic symlinks.
 			// Note that this may prevent some uses that isn't cyclic and also
 			// potential useful, but this implementation is both robust and simple:
 			// We stop at the first directory that we have seen before, e.g.
 			// /content/blog will only be processed once.
-			return nil, realPath, errSkipCyclicDir
+			return errSkipCyclicDir
 		}
 
 		if c.contentChanges != nil {
 			// Keep track of symbolic links in watch mode.
 			var from, to string
-			if fileInfo.IsDir() {
+			if sfi.IsDir() {
 				from = realPath
 				to = path
 
@@ -667,12 +709,11 @@ func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) {
 					from = from + helpers.FilePathSeparator
 				}
 
-				baseDir := c.baseDir
-				if !strings.HasSuffix(baseDir, helpers.FilePathSeparator) {
-					baseDir = baseDir + helpers.FilePathSeparator
+				if !strings.HasSuffix(basePath, helpers.FilePathSeparator) {
+					basePath = basePath + helpers.FilePathSeparator
 				}
 
-				if strings.HasPrefix(from, baseDir) {
+				if strings.HasPrefix(from, basePath) {
 					// With symbolic links inside /content we need to keep
 					// a reference to both. This may be confusing with --navigateToChanged
 					// but the user has chosen this him or herself.
@@ -688,9 +729,13 @@ func (c *capturer) getRealFileInfo(path string) (os.FileInfo, string, error) {
 		}
 	}
 
-	return fileInfo, realPath, nil
+	return nil
 }
 
-func (c *capturer) lstatIfOs(path string) (os.FileInfo, error) {
-	return helpers.LstatIfOs(c.fs, path)
+func (c *capturer) lstatIfPossible(path string) (pathLangFileFi, error) {
+	fi, err := helpers.LstatIfPossible(c.fs, path)
+	if err != nil {
+		return nil, err
+	}
+	return fi.(pathLangFileFi), nil
 }

hugolib/page_bundler_capture_test.go

@@ -15,6 +15,7 @@ package hugolib
 
 import (
 	"fmt"
+	"os"
 	"path"
 	"path/filepath"
 	"sort"
@@ -62,13 +63,12 @@ func (s *storeFilenames) handleBundles(d *bundleDirs) {
 	s.dirKeys = append(s.dirKeys, keys...)
 }
 
-func (s *storeFilenames) handleCopyFiles(names ...string) {
+func (s *storeFilenames) handleCopyFiles(files ...pathLangFile) {
 	s.Lock()
 	defer s.Unlock()
-	for _, name := range names {
-		s.copyNames = append(s.copyNames, filepath.ToSlash(name))
+	for _, file := range files {
+		s.copyNames = append(s.copyNames, filepath.ToSlash(file.Filename()))
 	}
-
 }
 
 func (s *storeFilenames) sortedStr() string {
@@ -83,13 +83,12 @@ func (s *storeFilenames) sortedStr() string {
 
 func TestPageBundlerCaptureSymlinks(t *testing.T) {
 	assert := require.New(t)
-	cfg, fs, workDir := newTestBundleSymbolicSources(t)
-	contentDir := "base"
-	sourceSpec := source.NewSourceSpec(cfg, fs)
+	ps, workDir := newTestBundleSymbolicSources(t)
+	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
 
 	fileStore := &storeFilenames{}
 	logger := newErrorLogger()
-	c := newCapturer(logger, sourceSpec, fileStore, nil, filepath.Join(workDir, contentDir))
+	c := newCapturer(logger, sourceSpec, fileStore, nil)
 
 	assert.NoError(c.capture())
 
@@ -110,6 +109,7 @@ C:
 /base/symbolic3/s1.png
 /base/symbolic3/s2.png
 `
+
 	got := strings.Replace(fileStore.sortedStr(), filepath.ToSlash(workDir), "", -1)
 	got = strings.Replace(got, "//", "/", -1)
 
@@ -120,20 +120,26 @@ C:
 	}
 }
 
-func TestPageBundlerCapture(t *testing.T) {
+func TestPageBundlerCaptureBasic(t *testing.T) {
 	t.Parallel()
 
 	assert := require.New(t)
-	cfg, fs := newTestBundleSources(t)
+	fs, cfg := newTestBundleSources(t)
+	assert.NoError(loadDefaultSettingsFor(cfg))
+	assert.NoError(loadLanguageSettings(cfg, nil))
+	ps, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
 
-	sourceSpec := source.NewSourceSpec(cfg, fs)
+	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
 
 	fileStore := &storeFilenames{}
 
-	c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil, filepath.FromSlash("/work/base"))
+	c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil)
 
 	assert.NoError(c.capture())
 
+	printFs(fs.Source, "", os.Stdout)
+
 	expected := `
 F:
 /work/base/_1.md
@@ -165,10 +171,16 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) {
 	t.Parallel()
 
 	assert := require.New(t)
-	cfg, fs := newTestBundleSourcesMultilingual(t)
-	sourceSpec := source.NewSourceSpec(cfg, fs)
+	fs, cfg := newTestBundleSourcesMultilingual(t)
+	assert.NoError(loadDefaultSettingsFor(cfg))
+	assert.NoError(loadLanguageSettings(cfg, nil))
+
+	ps, err := helpers.NewPathSpec(fs, cfg)
+	assert.NoError(err)
+
+	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
 	fileStore := &storeFilenames{}
-	c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil, filepath.FromSlash("/work/base"))
+	c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil)
 
 	assert.NoError(c.capture())
 
@@ -204,7 +216,7 @@ C:
 	if expected != got {
 		diff := helpers.DiffStringSlices(strings.Fields(expected), strings.Fields(got))
 		t.Log(got)
-		t.Fatalf("Failed:\n%s", diff)
+		t.Fatalf("Failed:\n%s", strings.Join(diff, "\n"))
 	}
 
 }
@@ -213,14 +225,15 @@ type noOpFileStore int
 
 func (noOpFileStore) handleSingles(fis ...*fileInfo)        {}
 func (noOpFileStore) handleBundles(b *bundleDirs)           {}
-func (noOpFileStore) handleCopyFiles(names ...string) {}
+func (noOpFileStore) handleCopyFiles(files ...pathLangFile) {}
 
 func BenchmarkPageBundlerCapture(b *testing.B) {
 	capturers := make([]*capturer, b.N)
 
 	for i := 0; i < b.N; i++ {
 		cfg, fs := newTestCfg()
-		sourceSpec := source.NewSourceSpec(cfg, fs)
+		ps, _ := helpers.NewPathSpec(fs, cfg)
+		sourceSpec := source.NewSourceSpec(ps, fs.Source)
 
 		base := fmt.Sprintf("base%d", i)
 		for j := 1; j <= 5; j++ {

hugolib/page_bundler_handlers.go

@@ -101,9 +101,6 @@ type handlerContext struct {
 
 	bundle *bundleDir
 
-	// The source baseDir, e.g. "/myproject/content/"
-	baseDir string
-
 	source *fileInfo
 
 	// Relative path to the target.
@@ -130,7 +127,7 @@ func (c *handlerContext) targetPath() string {
 		return c.target
 	}
 
-	return strings.TrimPrefix(c.source.Filename(), c.baseDir)
+	return c.source.Filename()
 }
 
 func (c *handlerContext) file() *fileInfo {
@@ -326,7 +323,6 @@ func (c *contentHandlers) createResource() contentHandler {
 
 		resource, err := c.s.resourceSpec.NewResourceFromFilename(
 			ctx.parentPage.subResourceTargetPathFactory,
-			c.s.absPublishDir(),
 			ctx.source.Filename(), ctx.target)
 
 		return handlerResult{err: err, handled: true, resource: resource}
@@ -335,8 +331,9 @@ func (c *contentHandlers) createResource() contentHandler {
 
 func (c *contentHandlers) copyFile() contentHandler {
 	return func(ctx *handlerContext) handlerResult {
-		f, err := c.s.Fs.Source.Open(ctx.source.Filename())
+		f, err := c.s.BaseFs.ContentFs.Open(ctx.source.Filename())
 		if err != nil {
+			err := fmt.Errorf("failed to open file in copyFile: %s", err)
 			return handlerResult{err: err}
 		}
 

hugolib/page_bundler_test.go

@@ -20,6 +20,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
 	"io"
 
 	"github.com/spf13/afero"
@@ -38,7 +40,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestPageBundlerSite(t *testing.T) {
+func TestPageBundlerSiteRegular(t *testing.T) {
 	t.Parallel()
 
 	for _, ugly := range []bool{false, true} {
@@ -46,7 +48,9 @@ func TestPageBundlerSite(t *testing.T) {
 			func(t *testing.T) {
 
 				assert := require.New(t)
-				cfg, fs := newTestBundleSources(t)
+				fs, cfg := newTestBundleSources(t)
+				assert.NoError(loadDefaultSettingsFor(cfg))
+				assert.NoError(loadLanguageSettings(cfg, nil))
 
 				cfg.Set("permalinks", map[string]string{
 					"a": ":sections/:filename",
@@ -141,6 +145,8 @@ func TestPageBundlerSite(t *testing.T) {
 
 				assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename())
 				assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink())
+
+				printFs(th.Fs.Destination, "", os.Stdout)
 				th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content")
 				th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content")
 
@@ -195,8 +201,7 @@ func TestPageBundlerSiteMultilingual(t *testing.T) {
 			func(t *testing.T) {
 
 				assert := require.New(t)
-				cfg, fs := newTestBundleSourcesMultilingual(t)
-
+				fs, cfg := newTestBundleSourcesMultilingual(t)
 				cfg.Set("uglyURLs", ugly)
 
 				assert.NoError(loadDefaultSettingsFor(cfg))
@@ -260,7 +265,7 @@ func TestMultilingualDisableDefaultLanguage(t *testing.T) {
 	t.Parallel()
 
 	assert := require.New(t)
-	cfg, _ := newTestBundleSourcesMultilingual(t)
+	_, cfg := newTestBundleSourcesMultilingual(t)
 
 	cfg.Set("disableLanguages", []string{"en"})
 
@@ -275,10 +280,12 @@ func TestMultilingualDisableLanguage(t *testing.T) {
 	t.Parallel()
 
 	assert := require.New(t)
-	cfg, fs := newTestBundleSourcesMultilingual(t)
+	fs, cfg := newTestBundleSourcesMultilingual(t)
 	cfg.Set("disableLanguages", []string{"nn"})
 
 	assert.NoError(loadDefaultSettingsFor(cfg))
+	assert.NoError(loadLanguageSettings(cfg, nil))
+
 	sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})
 	assert.NoError(err)
 	assert.Equal(1, len(sites.Sites))
@@ -302,7 +309,9 @@ func TestMultilingualDisableLanguage(t *testing.T) {
 
 func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) {
 	assert := require.New(t)
-	cfg, fs, workDir := newTestBundleSymbolicSources(t)
+	ps, workDir := newTestBundleSymbolicSources(t)
+	cfg := ps.Cfg
+	fs := ps.Fs
 
 	s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: newErrorLogger()}, BuildCfg{})
 
@@ -401,7 +410,7 @@ HEADLESS {{< myShort >}}
 
 }
 
-func newTestBundleSources(t *testing.T) (*viper.Viper, *hugofs.Fs) {
+func newTestBundleSources(t *testing.T) (*hugofs.Fs, *viper.Viper) {
 	cfg, fs := newTestCfg()
 	assert := require.New(t)
 
@@ -543,10 +552,11 @@ Content for 은행.
 	src.Close()
 	assert.NoError(err)
 
-	return cfg, fs
+	return fs, cfg
+
 }
 
-func newTestBundleSourcesMultilingual(t *testing.T) (*viper.Viper, *hugofs.Fs) {
+func newTestBundleSourcesMultilingual(t *testing.T) (*hugofs.Fs, *viper.Viper) {
 	cfg, fs := newTestCfg()
 
 	workDir := "/work"
@@ -626,10 +636,10 @@ TheContent.
 	writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "index.nn.md"), pageContent)
 	writeSource(t, fs, filepath.Join(workDir, "base", "bf", "my-bf-bundle", "page.md"), pageContent)
 
-	return cfg, fs
+	return fs, cfg
 }
 
-func newTestBundleSymbolicSources(t *testing.T) (*viper.Viper, *hugofs.Fs, string) {
+func newTestBundleSymbolicSources(t *testing.T) (*helpers.PathSpec, string) {
 	assert := require.New(t)
 	// We need to use the OS fs for this.
 	cfg := viper.New()
@@ -650,6 +660,10 @@ func newTestBundleSymbolicSources(t *testing.T) (*viper.Viper, *hugofs.Fs, strin
 	cfg.Set("contentDir", contentDir)
 	cfg.Set("baseURL", "https://example.com")
 
+	if err := loadLanguageSettings(cfg, nil); err != nil {
+		t.Fatal(err)
+	}
+
 	layout := `{{ .Title }}|{{ .Content }}`
 	pageContent := `---
 slug: %s
@@ -709,5 +723,7 @@ TheContent.
 	os.Chdir(workDir)
 	assert.NoError(err)
 
-	return cfg, fs, workDir
+	ps, _ := helpers.NewPathSpec(fs, cfg)
+
+	return ps, workDir
 }

hugolib/page_collections.go

@@ -179,8 +179,8 @@ func (c *PageCollections) addPage(page *Page) {
 	c.rawAllPages = append(c.rawAllPages, page)
 }
 
-func (c *PageCollections) removePageByPath(path string) {
-	if i := c.rawAllPages.findPagePosByFilePath(path); i >= 0 {
+func (c *PageCollections) removePageFilename(filename string) {
+	if i := c.rawAllPages.findPagePosByFilename(filename); i >= 0 {
 		c.clearResourceCacheForPage(c.rawAllPages[i])
 		c.rawAllPages = append(c.rawAllPages[:i], c.rawAllPages[i+1:]...)
 	}
@@ -218,6 +218,7 @@ func (c *PageCollections) clearResourceCacheForPage(page *Page) {
 	if len(page.Resources) > 0 {
 		first := page.Resources[0]
 		dir := path.Dir(first.RelPermalink())
+		dir = strings.TrimPrefix(dir, page.LanguagePrefix())
 		// This is done to keep the memory usage in check when doing live reloads.
 		page.s.resourceSpec.DeleteCacheByPrefix(dir)
 	}

hugolib/page_test.go

@@ -1109,60 +1109,6 @@ func TestCreatePage(t *testing.T) {
 	}
 }
 
-func TestPageKind(t *testing.T) {
-	t.Parallel()
-	const sep = helpers.FilePathSeparator
-	var tests = []struct {
-		file string
-		kind string
-	}{
-		{"_index.md", KindHome},
-		{"about.md", KindPage},
-		{"sectionA" + sep + "_index.md", KindSection},
-		{"sectionA" + sep + "about.md", KindPage},
-		{"categories" + sep + "_index.md", KindTaxonomyTerm},
-		{"categories" + sep + "categoryA" + sep + "_index.md", KindTaxonomy},
-		{"tags" + sep + "_index.md", KindTaxonomyTerm},
-		{"tags" + sep + "tagA" + sep + "_index.md", KindTaxonomy},
-
-		// nn is configured as a language
-		{"_index.nn.md", KindHome},
-		{"about.nn.md", KindPage},
-		{"sectionA" + sep + "_index.nn.md", KindSection},
-		{"sectionA" + sep + "about.nn.md", KindPage},
-
-		// should NOT be categorized as KindHome
-		{"_indexNOT.md", KindPage},
-
-		// To be consistent with FileInfo.TranslationBaseName(),
-		// language codes not explicitly configured for the site
-		// are not treated as such. "fr" is not configured as
-		// a language in the test site, so ALL of the
-		// following should be KindPage
-		{"_index.fr.md", KindPage}, //not KindHome
-		{"about.fr.md", KindPage},
-		{"sectionA" + sep + "_index.fr.md", KindPage}, // KindSection
-		{"sectionA" + sep + "about.fr.md", KindPage},
-	}
-
-	for _, test := range tests {
-		s := newTestSite(t, "languages.nn.languageName", "Nynorsk")
-		taxonomies := make(map[string]string)
-		taxonomies["tag"] = "tags"
-		taxonomies["category"] = "categories"
-		s.Taxonomies = make(TaxonomyList)
-		for _, plural := range taxonomies {
-			s.Taxonomies[plural] = make(Taxonomy)
-		}
-
-		p, _ := s.NewPage(test.file)
-		p.setValuesForKind(s)
-		if p.Kind != test.kind {
-			t.Errorf("for %s expected p.Kind == %s, got %s", test.file, test.kind, p.Kind)
-		}
-	}
-}
-
 func TestDegenerateInvalidFrontMatterShortDelim(t *testing.T) {
 	t.Parallel()
 	var tests = []struct {

hugolib/prune_resources.go

@@ -25,7 +25,9 @@ import (
 // GC requires a build first.
 func (h *HugoSites) GC() (int, error) {
 	s := h.Sites[0]
-	imageCacheDir := s.resourceSpec.AbsGenImagePath
+	fs := h.PathSpec.BaseFs.ResourcesFs
+
+	imageCacheDir := s.resourceSpec.GenImagePath
 	if len(imageCacheDir) < 10 {
 		panic("invalid image cache")
 	}
@@ -43,7 +45,7 @@ func (h *HugoSites) GC() (int, error) {
 
 	counter := 0
 
-	err := afero.Walk(s.Fs.Source, imageCacheDir, func(path string, info os.FileInfo, err error) error {
+	err := afero.Walk(fs, imageCacheDir, func(path string, info os.FileInfo, err error) error {
 		if info == nil {
 			return nil
 		}
@@ -53,7 +55,7 @@ func (h *HugoSites) GC() (int, error) {
 		}
 
 		if info.IsDir() {
-			f, err := s.Fs.Source.Open(path)
+			f, err := fs.Open(path)
 			if err != nil {
 				return nil
 			}
@@ -69,7 +71,7 @@ func (h *HugoSites) GC() (int, error) {
 
 		inUse := isInUse(path)
 		if !inUse {
-			err := s.Fs.Source.Remove(path)
+			err := fs.Remove(path)
 			if err != nil && !os.IsNotExist(err) {
 				s.Log.ERROR.Printf("Failed to remove %q: %s", path, err)
 			} else {

hugolib/site.go

@@ -746,9 +746,7 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) {
 			}
 		}
 		if removed && isContentFile(ev.Name) {
-			path, _ := helpers.GetRelativePath(ev.Name, s.getContentDir(ev.Name))
-
-			h.removePageByPath(path)
+			h.removePageByFilename(ev.Name)
 		}
 
 		sourceReallyChanged = append(sourceReallyChanged, ev)
@@ -890,7 +888,7 @@ func (s *Site) handleDataFile(r source.ReadableFile) error {
 func (s *Site) readData(f source.ReadableFile) (interface{}, error) {
 	file, err := f.Open()
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("readData: failed to open data file: %s", err)
 	}
 	defer file.Close()
 	content := helpers.ReaderToBytes(file)
@@ -1295,9 +1293,9 @@ func (c *contentCaptureResultHandler) handleBundles(d *bundleDirs) {
 	}
 }
 
-func (c *contentCaptureResultHandler) handleCopyFiles(filenames ...string) {
+func (c *contentCaptureResultHandler) handleCopyFiles(files ...pathLangFile) {
 	for _, proc := range c.contentProcessors {
-		proc.processAssets(filenames)
+		proc.processAssets(files)
 	}
 }
 
@@ -1305,15 +1303,16 @@ func (s *Site) readAndProcessContent(filenames ...string) error {
 	ctx := context.Background()
 	g, ctx := errgroup.WithContext(ctx)
 
-	sourceSpec := source.NewSourceSpec(s.owner.Cfg, s.Fs)
-	baseDir := s.absContentDir()
 	defaultContentLanguage := s.SourceSpec.DefaultContentLanguage
 
 	contentProcessors := make(map[string]*siteContentProcessor)
 	var defaultContentProcessor *siteContentProcessor
 	sites := s.owner.langSite()
 	for k, v := range sites {
-		proc := newSiteContentProcessor(ctx, baseDir, len(filenames) > 0, v)
+		if v.Language.Disabled {
+			continue
+		}
+		proc := newSiteContentProcessor(ctx, len(filenames) > 0, v)
 		contentProcessors[k] = proc
 		if k == defaultContentLanguage {
 			defaultContentProcessor = proc
@@ -1330,6 +1329,8 @@ func (s *Site) readAndProcessContent(filenames ...string) error {
 
 	mainHandler := &contentCaptureResultHandler{contentProcessors: contentProcessors, defaultContentProcessor: defaultContentProcessor}
 
+	sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.ContentFs)
+
 	if s.running() {
 		// Need to track changes.
 		bundleMap = s.owner.ContentChanges
@@ -1339,7 +1340,7 @@ func (s *Site) readAndProcessContent(filenames ...string) error {
 		handler = mainHandler
 	}
 
-	c := newCapturer(s.Log, sourceSpec, handler, bundleMap, baseDir, filenames...)
+	c := newCapturer(s.Log, sourceSpec, handler, bundleMap, filenames...)
 
 	err1 := c.capture()
 

hugolib/site_url_test.go

@@ -119,10 +119,12 @@ Do not go gentle into that good night.
 
 	notUgly := s.getPage(KindPage, "sect1/p1.md")
 	assert.NotNil(notUgly)
+	assert.Equal("sect1", notUgly.Section())
 	assert.Equal("/sect1/p1/", notUgly.RelPermalink())
 
 	ugly := s.getPage(KindPage, "sect2/p2.md")
 	assert.NotNil(ugly)
+	assert.Equal("sect2", ugly.Section())
 	assert.Equal("/sect2/p2.html", ugly.RelPermalink())
 }
 

hugolib/taxonomy_test.go

@@ -115,8 +115,8 @@ permalinkeds:
 	writeSource(t, fs, "content/p3.md", fmt.Sprintf(pageTemplate, "t2/c12", "- tag2", "- cat2", "- o1", "- pl1"))
 	writeSource(t, fs, "content/p4.md", fmt.Sprintf(pageTemplate, "Hello World", "", "", "- \"Hello Hugo world\"", "- pl1"))
 
-	writeNewContentFile(t, fs, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
-	writeNewContentFile(t, fs, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10)
+	writeNewContentFile(t, fs.Source, "Category Terms", "2017-01-01", "content/categories/_index.md", 10)
+	writeNewContentFile(t, fs.Source, "Tag1 List", "2017-01-01", "content/tags/Tag1/_index.md", 10)
 
 	err := h.Build(BuildCfg{})
 

hugolib/testhelpers_test.go

@@ -51,6 +51,11 @@ type sitesBuilder struct {
 	// Default toml
 	configFormat string
 
+	// Default is empty.
+	// TODO(bep) revisit this and consider always setting it to something.
+	// Consider this in relation to using the BaseFs.PublishFs to all publishing.
+	workingDir string
+
 	// Base data/content
 	contentFilePairs  []string
 	templateFilePairs []string
@@ -83,6 +88,11 @@ func (s *sitesBuilder) Running() *sitesBuilder {
 	return s
 }
 
+func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {
+	s.workingDir = dir
+	return s
+}
+
 func (s *sitesBuilder) WithConfigTemplate(data interface{}, format, configTemplate string) *sitesBuilder {
 	if format == "" {
 		format = "toml"
@@ -233,7 +243,17 @@ func (s *sitesBuilder) writeFilePairs(folder string, filenameContent []string) *
 	}
 	for i := 0; i < len(filenameContent); i += 2 {
 		filename, content := filenameContent[i], filenameContent[i+1]
-		writeSource(s.T, s.Fs, filepath.Join(folder, filename), content)
+		target := folder
+		// TODO(bep) clean  up this magic.
+		if strings.HasPrefix(filename, folder) {
+			target = ""
+		}
+
+		if s.workingDir != "" {
+			target = filepath.Join(s.workingDir, target)
+		}
+
+		writeSource(s.T, s.Fs, filepath.Join(target, filename), content)
 	}
 	return s
 }
@@ -458,6 +478,7 @@ func newTestDefaultPathSpec() *helpers.PathSpec {
 	v := viper.New()
 	// Easier to reason about in tests.
 	v.Set("disablePathToLower", true)
+	v.Set("contentDir", "content")
 	fs := hugofs.NewDefault(v)
 	ps, _ := helpers.NewPathSpec(fs, v)
 	return ps

i18n/i18n_test.go

@@ -200,6 +200,7 @@ func TestI18nTranslate(t *testing.T) {
 	var actual, expected string
 	v := viper.New()
 	v.SetDefault("defaultContentLanguage", "en")
+	v.Set("contentDir", "content")
 
 	// Test without and with placeholders
 	for _, enablePlaceholders := range []bool{false, true} {

i18n/translationProvider.go

@@ -39,7 +39,7 @@ func NewTranslationProvider() *TranslationProvider {
 // Update updates the i18n func in the provided Deps.
 func (tp *TranslationProvider) Update(d *deps.Deps) error {
 	dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir"))
-	sp := source.NewSourceSpec(d.Cfg, d.Fs)
+	sp := source.NewSourceSpec(d.PathSpec, d.Fs.Source)
 	sources := []source.Input{sp.NewFilesystem(dir)}
 
 	themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath()

resource/image.go

@@ -419,7 +419,7 @@ func (i *Image) initConfig() error {
 			config image.Config
 		)
 
-		f, err = i.spec.Fs.Source.Open(i.AbsSourceFilename())
+		f, err = i.sourceFs().Open(i.AbsSourceFilename())
 		if err != nil {
 			return
 		}
@@ -432,13 +432,17 @@ func (i *Image) initConfig() error {
 		i.config = config
 	})
 
-	return err
+	if err != nil {
+		return fmt.Errorf("failed to load image config: %s", err)
+	}
+
+	return nil
 }
 
 func (i *Image) decodeSource() (image.Image, error) {
-	file, err := i.spec.Fs.Source.Open(i.AbsSourceFilename())
+	file, err := i.sourceFs().Open(i.AbsSourceFilename())
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to open image for decode: %s", err)
 	}
 	defer file.Close()
 	img, _, err := image.Decode(file)
@@ -448,32 +452,32 @@ func (i *Image) decodeSource() (image.Image, error) {
 func (i *Image) copyToDestination(src string) error {
 	var res error
 	i.copyToDestinationInit.Do(func() {
-		target := filepath.Join(i.absPublishDir, i.target())
+		target := i.target()
 
 		// Fast path:
 		// This is a processed version of the original.
 		// If it exists on destination with the same filename and file size, it is
 		// the same file, so no need to transfer it again.
-		if fi, err := i.spec.Fs.Destination.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() {
+		if fi, err := i.spec.BaseFs.PublishFs.Stat(target); err == nil && fi.Size() == i.osFileInfo.Size() {
 			return
 		}
 
-		in, err := i.spec.Fs.Source.Open(src)
+		in, err := i.sourceFs().Open(src)
 		if err != nil {
 			res = err
 			return
 		}
 		defer in.Close()
 
-		out, err := i.spec.Fs.Destination.Create(target)
+		out, err := i.spec.BaseFs.PublishFs.Create(target)
 		if err != nil && os.IsNotExist(err) {
 			// When called from shortcodes, the target directory may not exist yet.
 			// See https://github.com/gohugoio/hugo/issues/4202
-			if err = i.spec.Fs.Destination.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
+			if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
 				res = err
 				return
 			}
-			out, err = i.spec.Fs.Destination.Create(target)
+			out, err = i.spec.BaseFs.PublishFs.Create(target)
 			if err != nil {
 				res = err
 				return
@@ -491,20 +495,23 @@ func (i *Image) copyToDestination(src string) error {
 		}
 	})
 
-	return res
+	if res != nil {
+		return fmt.Errorf("failed to copy image to destination: %s", res)
+	}
+	return nil
 }
 
 func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resourceCacheFilename, filename string) error {
-	target := filepath.Join(i.absPublishDir, filename)
+	target := filepath.Clean(filename)
 
-	file1, err := i.spec.Fs.Destination.Create(target)
+	file1, err := i.spec.BaseFs.PublishFs.Create(target)
 	if err != nil && os.IsNotExist(err) {
 		// When called from shortcodes, the target directory may not exist yet.
 		// See https://github.com/gohugoio/hugo/issues/4202
-		if err = i.spec.Fs.Destination.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
+		if err = i.spec.BaseFs.PublishFs.MkdirAll(filepath.Dir(target), os.FileMode(0755)); err != nil {
 			return err
 		}
-		file1, err = i.spec.Fs.Destination.Create(target)
+		file1, err = i.spec.BaseFs.PublishFs.Create(target)
 		if err != nil {
 			return err
 		}
@@ -518,11 +525,11 @@ func (i *Image) encodeToDestinations(img image.Image, conf imageConfig, resource
 
 	if resourceCacheFilename != "" {
 		// Also save it to the image resource cache for later reuse.
-		if err = i.spec.Fs.Source.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil {
+		if err = i.spec.BaseFs.ResourcesFs.MkdirAll(filepath.Dir(resourceCacheFilename), os.FileMode(0755)); err != nil {
 			return err
 		}
 
-		file2, err := i.spec.Fs.Source.Create(resourceCacheFilename)
+		file2, err := i.spec.BaseFs.ResourcesFs.Create(resourceCacheFilename)
 		if err != nil {
 			return err
 		}

resource/image_cache.go

@@ -24,8 +24,7 @@ import (
 )
 
 type imageCache struct {
-	absPublishDir string
-	absCacheDir   string
+	cacheDir string
 	pathSpec *helpers.PathSpec
 	mu       sync.RWMutex
 
@@ -82,14 +81,14 @@ func (c *imageCache) getOrCreate(
 	parent.createMu.Lock()
 	defer parent.createMu.Unlock()
 
-	cacheFilename := filepath.Join(c.absCacheDir, key)
+	cacheFilename := filepath.Join(c.cacheDir, key)
 
 	// The definition of this counter is not that we have processed that amount
 	// (e.g. resized etc.), it can be fetched from file cache,
 	//  but the count of processed image variations for this site.
 	c.pathSpec.ProcessingStats.Incr(&c.pathSpec.ProcessingStats.ProcessedImages)
 
-	exists, err := helpers.Exists(cacheFilename, c.pathSpec.Fs.Source)
+	exists, err := helpers.Exists(cacheFilename, c.pathSpec.BaseFs.ResourcesFs)
 	if err != nil {
 		return nil, err
 	}
@@ -97,7 +96,9 @@ func (c *imageCache) getOrCreate(
 	if exists {
 		img = parent.clone()
 		img.relTargetPath.file = relTarget.file
-		img.absSourceFilename = cacheFilename
+		img.sourceFilename = cacheFilename
+		// We have to look resources file system for this.
+		img.overriddenSourceFs = img.spec.BaseFs.ResourcesFs
 	} else {
 		img, err = create(cacheFilename)
 		if err != nil {
@@ -124,8 +125,8 @@ func (c *imageCache) getOrCreate(
 
 }
 
-func newImageCache(ps *helpers.PathSpec, absCacheDir, absPublishDir string) *imageCache {
-	return &imageCache{pathSpec: ps, store: make(map[string]*Image), absCacheDir: absCacheDir, absPublishDir: absPublishDir}
+func newImageCache(ps *helpers.PathSpec, cacheDir string) *imageCache {
+	return &imageCache{pathSpec: ps, store: make(map[string]*Image), cacheDir: cacheDir}
 }
 
 func timeTrack(start time.Time, name string) {

resource/image_test.go

@@ -16,6 +16,7 @@ package resource
 import (
 	"fmt"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"strconv"
 	"testing"
@@ -57,12 +58,14 @@ func TestParseImageConfig(t *testing.T) {
 	}
 }
 
-func TestImageTransform(t *testing.T) {
+func TestImageTransformBasic(t *testing.T) {
 
 	assert := require.New(t)
 
 	image := fetchSunset(assert)
 
+	printFs(image.sourceFs(), "", os.Stdout)
+
 	assert.Equal("/a/sunset.jpg", image.RelPermalink())
 	assert.Equal("image", image.ResourceType())
 
@@ -75,19 +78,19 @@ func TestImageTransform(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal(320, resized0x.Width())
 	assert.Equal(200, resized0x.Height())
-	assertFileCache(assert, image.spec.Fs, resized0x.RelPermalink(), 320, 200)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized0x.RelPermalink(), 320, 200)
 
 	resizedx0, err := image.Resize("200x")
 	assert.NoError(err)
 	assert.Equal(200, resizedx0.Width())
 	assert.Equal(125, resizedx0.Height())
-	assertFileCache(assert, image.spec.Fs, resizedx0.RelPermalink(), 200, 125)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedx0.RelPermalink(), 200, 125)
 
 	resizedAndRotated, err := image.Resize("x200 r90")
 	assert.NoError(err)
 	assert.Equal(125, resizedAndRotated.Width())
 	assert.Equal(200, resizedAndRotated.Height())
-	assertFileCache(assert, image.spec.Fs, resizedAndRotated.RelPermalink(), 125, 200)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAndRotated.RelPermalink(), 125, 200)
 
 	assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_300x200_resize_q68_linear.jpg", resized.RelPermalink())
 	assert.Equal(300, resized.Width())
@@ -112,20 +115,20 @@ func TestImageTransform(t *testing.T) {
 	assert.Equal("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_bottomleft.jpg", filled.RelPermalink())
 	assert.Equal(200, filled.Width())
 	assert.Equal(100, filled.Height())
-	assertFileCache(assert, image.spec.Fs, filled.RelPermalink(), 200, 100)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filled.RelPermalink(), 200, 100)
 
 	smart, err := image.Fill("200x100 smart")
 	assert.NoError(err)
 	assert.Equal(fmt.Sprintf("/a/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x100_fill_q68_linear_smart%d.jpg", smartCropVersionNumber), smart.RelPermalink())
 	assert.Equal(200, smart.Width())
 	assert.Equal(100, smart.Height())
-	assertFileCache(assert, image.spec.Fs, smart.RelPermalink(), 200, 100)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, smart.RelPermalink(), 200, 100)
 
 	// Check cache
 	filledAgain, err := image.Fill("200x100 bottomLeft")
 	assert.NoError(err)
 	assert.True(filled == filledAgain)
-	assertFileCache(assert, image.spec.Fs, filledAgain.RelPermalink(), 200, 100)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, filledAgain.RelPermalink(), 200, 100)
 
 }
 
@@ -295,10 +298,10 @@ func TestImageResizeInSubPath(t *testing.T) {
 	assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resized.RelPermalink())
 	assert.Equal(101, resized.Width())
 
-	assertFileCache(assert, image.spec.Fs, resized.RelPermalink(), 101, 101)
-	publishedImageFilename := filepath.Join("/public", resized.RelPermalink())
-	assertImageFile(assert, image.spec.Fs, publishedImageFilename, 101, 101)
-	assert.NoError(image.spec.Fs.Destination.Remove(publishedImageFilename))
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resized.RelPermalink(), 101, 101)
+	publishedImageFilename := filepath.Clean(resized.RelPermalink())
+	assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
+	assert.NoError(image.spec.BaseFs.PublishFs.Remove(publishedImageFilename))
 
 	// Cleare mem cache to simulate reading from the file cache.
 	resized.spec.imageCache.clear()
@@ -307,8 +310,8 @@ func TestImageResizeInSubPath(t *testing.T) {
 	assert.NoError(err)
 	assert.Equal("/a/sub/gohugoio2_hu0e1b9e4a4be4d6f86c7b37b9ccce3fbc_73886_101x101_resize_linear_2.png", resizedAgain.RelPermalink())
 	assert.Equal(101, resizedAgain.Width())
-	assertFileCache(assert, image.spec.Fs, resizedAgain.RelPermalink(), 101, 101)
-	assertImageFile(assert, image.spec.Fs, publishedImageFilename, 101, 101)
+	assertFileCache(assert, image.spec.BaseFs.ResourcesFs, resizedAgain.RelPermalink(), 101, 101)
+	assertImageFile(assert, image.spec.BaseFs.PublishFs, publishedImageFilename, 101, 101)
 
 }
 

resource/resource.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/spf13/afero"
+
 	"github.com/spf13/cast"
 
 	"github.com/gobwas/glob"
@@ -214,6 +216,7 @@ func getGlob(pattern string) (glob.Glob, error) {
 
 type Spec struct {
 	*helpers.PathSpec
+
 	mimeTypes media.Types
 
 	// Holds default filter settings etc.
@@ -221,7 +224,7 @@ type Spec struct {
 
 	imageCache *imageCache
 
-	AbsGenImagePath string
+	GenImagePath string
 }
 
 func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) {
@@ -232,41 +235,44 @@ func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) {
 	}
 	s.GetLayoutDirPath()
 
-	genImagePath := s.AbsPathify(filepath.Join(s.Cfg.GetString("resourceDir"), "_gen", "images"))
+	genImagePath := filepath.FromSlash("_gen/images")
 
-	return &Spec{AbsGenImagePath: genImagePath, PathSpec: s, imaging: &imaging, mimeTypes: mimeTypes, imageCache: newImageCache(
+	return &Spec{PathSpec: s,
+		GenImagePath: genImagePath,
+		imaging:      &imaging, mimeTypes: mimeTypes, imageCache: newImageCache(
 			s,
 			// We're going to write a cache pruning routine later, so make it extremely
 			// unlikely that the user shoots him or herself in the foot
 			// and this is set to a value that represents data he/she
 			// cares about. This should be set in stone once released.
 			genImagePath,
-		s.AbsPathify(s.Cfg.GetString("publishDir")))}, nil
+		)}, nil
 }
 
 func (r *Spec) NewResourceFromFile(
 	targetPathBuilder func(base string) string,
-	absPublishDir string,
 	file source.File, relTargetFilename string) (Resource, error) {
 
-	return r.newResource(targetPathBuilder, absPublishDir, file.Filename(), file.FileInfo(), relTargetFilename)
+	return r.newResource(targetPathBuilder, file.Filename(), file.FileInfo(), relTargetFilename)
 }
 
 func (r *Spec) NewResourceFromFilename(
 	targetPathBuilder func(base string) string,
-	absPublishDir,
 	absSourceFilename, relTargetFilename string) (Resource, error) {
 
-	fi, err := r.Fs.Source.Stat(absSourceFilename)
+	fi, err := r.sourceFs().Stat(absSourceFilename)
 	if err != nil {
 		return nil, err
 	}
-	return r.newResource(targetPathBuilder, absPublishDir, absSourceFilename, fi, relTargetFilename)
+	return r.newResource(targetPathBuilder, absSourceFilename, fi, relTargetFilename)
+}
+
+func (r *Spec) sourceFs() afero.Fs {
+	return r.PathSpec.BaseFs.ContentFs
 }
 
 func (r *Spec) newResource(
 	targetPathBuilder func(base string) string,
-	absPublishDir,
 	absSourceFilename string, fi os.FileInfo, relTargetFilename string) (Resource, error) {
 
 	var mimeType string
@@ -283,7 +289,7 @@ func (r *Spec) newResource(
 		}
 	}
 
-	gr := r.newGenericResource(targetPathBuilder, fi, absPublishDir, absSourceFilename, relTargetFilename, mimeType)
+	gr := r.newGenericResource(targetPathBuilder, fi, absSourceFilename, relTargetFilename, mimeType)
 
 	if mimeType == "image" {
 		ext := strings.ToLower(helpers.Ext(absSourceFilename))
@@ -295,9 +301,9 @@ func (r *Spec) newResource(
 			return gr, nil
 		}
 
-		f, err := r.Fs.Source.Open(absSourceFilename)
+		f, err := gr.sourceFs().Open(absSourceFilename)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("failed to open image source file: %s", err)
 		}
 		defer f.Close()
 
@@ -369,15 +375,30 @@ type genericResource struct {
 	params map[string]interface{}
 
 	// Absolute filename to the source, including any content folder path.
-	absSourceFilename string
-	absPublishDir     string
+	// Note that this is absolute in relation to the filesystem it is stored in.
+	// It can be a base path filesystem, and then this filename will not match
+	// the path to the file on the real filesystem.
+	sourceFilename string
+
+	// This may be set to tell us to look in another filesystem for this resource.
+	// We, by default, use the sourceFs filesystem in the spec below.
+	overriddenSourceFs afero.Fs
+
+	spec *Spec
+
 	resourceType string
 	osFileInfo   os.FileInfo
 
-	spec              *Spec
 	targetPathBuilder func(rel string) string
 }
 
+func (l *genericResource) sourceFs() afero.Fs {
+	if l.overriddenSourceFs != nil {
+		return l.overriddenSourceFs
+	}
+	return l.spec.sourceFs()
+}
+
 func (l *genericResource) Permalink() string {
 	return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetPath.path(), false), l.spec.BaseURL.String())
 }
@@ -455,19 +476,16 @@ func (l *genericResource) ResourceType() string {
 }
 
 func (l *genericResource) AbsSourceFilename() string {
-	return l.absSourceFilename
+	return l.sourceFilename
 }
 
 func (l *genericResource) Publish() error {
-	f, err := l.spec.Fs.Source.Open(l.AbsSourceFilename())
+	f, err := l.sourceFs().Open(l.AbsSourceFilename())
 	if err != nil {
 		return err
 	}
 	defer f.Close()
-
-	target := filepath.Join(l.absPublishDir, l.target())
-
-	return helpers.WriteToDisk(target, f, l.spec.Fs.Destination)
+	return helpers.WriteToDisk(l.target(), f, l.spec.BaseFs.PublishFs)
 }
 
 const counterPlaceHolder = ":counter"
@@ -574,8 +592,7 @@ func (l *genericResource) target() string {
 func (r *Spec) newGenericResource(
 	targetPathBuilder func(base string) string,
 	osFileInfo os.FileInfo,
-	absPublishDir,
-	absSourceFilename,
+	sourceFilename,
 	baseFilename,
 	resourceType string) *genericResource {
 
@@ -587,8 +604,7 @@ func (r *Spec) newGenericResource(
 	return &genericResource{
 		targetPathBuilder: targetPathBuilder,
 		osFileInfo:        osFileInfo,
-		absPublishDir:     absPublishDir,
-		absSourceFilename: absSourceFilename,
+		sourceFilename:    sourceFilename,
 		relTargetPath:     dirFile{dir: fpath, file: fname},
 		resourceType:      resourceType,
 		spec:              r,

resource/resource_test.go

@@ -29,7 +29,7 @@ func TestGenericResource(t *testing.T) {
 	assert := require.New(t)
 	spec := newTestResourceSpec(assert)
 
-	r := spec.newGenericResource(nil, nil, "/public", "/a/foo.css", "foo.css", "css")
+	r := spec.newGenericResource(nil, nil, "/a/foo.css", "foo.css", "css")
 
 	assert.Equal("https://example.com/foo.css", r.Permalink())
 	assert.Equal("/foo.css", r.RelPermalink())
@@ -44,7 +44,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) {
 	factory := func(s string) string {
 		return path.Join("/foo", s)
 	}
-	r := spec.newGenericResource(factory, nil, "/public", "/a/foo.css", "foo.css", "css")
+	r := spec.newGenericResource(factory, nil, "/a/foo.css", "foo.css", "css")
 
 	assert.Equal("https://example.com/foo/foo.css", r.Permalink())
 	assert.Equal("/foo/foo.css", r.RelPermalink())
@@ -55,11 +55,11 @@ func TestNewResourceFromFilename(t *testing.T) {
 	assert := require.New(t)
 	spec := newTestResourceSpec(assert)
 
-	writeSource(t, spec.Fs, "/project/a/b/logo.png", "image")
-	writeSource(t, spec.Fs, "/root/a/b/data.json", "json")
+	writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
+	writeSource(t, spec.Fs, "content/a/b/data.json", "json")
 
-	r, err := spec.NewResourceFromFilename(nil, "/public",
-		filepath.FromSlash("/project/a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
+	r, err := spec.NewResourceFromFilename(nil,
+		filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
 
 	assert.NoError(err)
 	assert.NotNil(r)
@@ -67,7 +67,7 @@ func TestNewResourceFromFilename(t *testing.T) {
 	assert.Equal("/a/b/logo.png", r.RelPermalink())
 	assert.Equal("https://example.com/a/b/logo.png", r.Permalink())
 
-	r, err = spec.NewResourceFromFilename(nil, "/public", "/root/a/b/data.json", "a/b/data.json")
+	r, err = spec.NewResourceFromFilename(nil, "a/b/data.json", "a/b/data.json")
 
 	assert.NoError(err)
 	assert.NotNil(r)
@@ -82,10 +82,10 @@ func TestNewResourceFromFilenameSubPathInBaseURL(t *testing.T) {
 	assert := require.New(t)
 	spec := newTestResourceSpecForBaseURL(assert, "https://example.com/docs")
 
-	writeSource(t, spec.Fs, "/project/a/b/logo.png", "image")
+	writeSource(t, spec.Fs, "content/a/b/logo.png", "image")
 
-	r, err := spec.NewResourceFromFilename(nil, "/public",
-		filepath.FromSlash("/project/a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
+	r, err := spec.NewResourceFromFilename(nil,
+		filepath.FromSlash("a/b/logo.png"), filepath.FromSlash("a/b/logo.png"))
 
 	assert.NoError(err)
 	assert.NotNil(r)
@@ -101,10 +101,10 @@ func TestResourcesByType(t *testing.T) {
 	assert := require.New(t)
 	spec := newTestResourceSpec(assert)
 	resources := Resources{
-		spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/a/logo.png", "logo.css", "image"),
-		spec.newGenericResource(nil, nil, "/public", "/a/foo2.css", "foo2.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/a/foo3.css", "foo3.css", "css")}
+		spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
+		spec.newGenericResource(nil, nil, "/a/logo.png", "logo.css", "image"),
+		spec.newGenericResource(nil, nil, "/a/foo2.css", "foo2.css", "css"),
+		spec.newGenericResource(nil, nil, "/a/foo3.css", "foo3.css", "css")}
 
 	assert.Len(resources.ByType("css"), 3)
 	assert.Len(resources.ByType("image"), 1)
@@ -115,11 +115,11 @@ func TestResourcesGetByPrefix(t *testing.T) {
 	assert := require.New(t)
 	spec := newTestResourceSpec(assert)
 	resources := Resources{
-		spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image"),
-		spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image"),
-		spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css")}
+		spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
+		spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"),
+		spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"),
+		spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"),
+		spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")}
 
 	assert.Nil(resources.GetByPrefix("asdf"))
 	assert.Equal("/logo1.png", resources.GetByPrefix("logo").RelPermalink())
@@ -144,14 +144,14 @@ func TestResourcesGetMatch(t *testing.T) {
 	assert := require.New(t)
 	spec := newTestResourceSpec(assert)
 	resources := Resources{
-		spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image"),
-		spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image"),
-		spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/b/c/foo4.css", "c/foo4.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/b/c/foo5.css", "c/foo5.css", "css"),
-		spec.newGenericResource(nil, nil, "/public", "/b/c/d/foo6.css", "c/d/foo6.css", "css"),
+		spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css"),
+		spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image"),
+		spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image"),
+		spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css"),
+		spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css"),
+		spec.newGenericResource(nil, nil, "/b/c/foo4.css", "c/foo4.css", "css"),
+		spec.newGenericResource(nil, nil, "/b/c/foo5.css", "c/foo5.css", "css"),
+		spec.newGenericResource(nil, nil, "/b/c/d/foo6.css", "c/d/foo6.css", "css"),
 	}
 
 	assert.Equal("/logo1.png", resources.GetMatch("logo*").RelPermalink())
@@ -373,12 +373,12 @@ func TestAssignMetadata(t *testing.T) {
 		}},
 	} {
 
-		foo2 = spec.newGenericResource(nil, nil, "/public", "/b/foo2.css", "foo2.css", "css")
-		logo2 = spec.newGenericResource(nil, nil, "/public", "/b/Logo2.png", "Logo2.png", "image")
-		foo1 = spec.newGenericResource(nil, nil, "/public", "/a/foo1.css", "foo1.css", "css")
-		logo1 = spec.newGenericResource(nil, nil, "/public", "/a/logo1.png", "logo1.png", "image")
-		foo3 = spec.newGenericResource(nil, nil, "/public", "/b/foo3.css", "foo3.css", "css")
-		logo3 = spec.newGenericResource(nil, nil, "/public", "/b/logo3.png", "logo3.png", "image")
+		foo2 = spec.newGenericResource(nil, nil, "/b/foo2.css", "foo2.css", "css")
+		logo2 = spec.newGenericResource(nil, nil, "/b/Logo2.png", "Logo2.png", "image")
+		foo1 = spec.newGenericResource(nil, nil, "/a/foo1.css", "foo1.css", "css")
+		logo1 = spec.newGenericResource(nil, nil, "/a/logo1.png", "logo1.png", "image")
+		foo3 = spec.newGenericResource(nil, nil, "/b/foo3.css", "foo3.css", "css")
+		logo3 = spec.newGenericResource(nil, nil, "/b/logo3.png", "logo3.png", "image")
 
 		resources = Resources{
 			foo2,
@@ -428,7 +428,7 @@ func BenchmarkResourcesMatchA100(b *testing.B) {
 	a100 := strings.Repeat("a", 100)
 	pattern := "a*a*a*a*a*a*a*a*b"
 
-	resources := Resources{spec.newGenericResource(nil, nil, "/public", "/a/"+a100, a100, "css")}
+	resources := Resources{spec.newGenericResource(nil, nil, "/a/"+a100, a100, "css")}
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -444,17 +444,17 @@ func benchResources(b *testing.B) Resources {
 
 	for i := 0; i < 30; i++ {
 		name := fmt.Sprintf("abcde%d_%d.css", i%5, i)
-		resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css"))
+		resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
 	}
 
 	for i := 0; i < 30; i++ {
 		name := fmt.Sprintf("efghi%d_%d.css", i%5, i)
-		resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css"))
+		resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
 	}
 
 	for i := 0; i < 30; i++ {
 		name := fmt.Sprintf("jklmn%d_%d.css", i%5, i)
-		resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/b/sub/"+name, "sub/"+name, "css"))
+		resources = append(resources, spec.newGenericResource(nil, nil, "/b/sub/"+name, "sub/"+name, "css"))
 	}
 
 	return resources
@@ -482,7 +482,7 @@ func BenchmarkAssignMetadata(b *testing.B) {
 		}
 		for i := 0; i < 20; i++ {
 			name := fmt.Sprintf("foo%d_%d.css", i%5, i)
-			resources = append(resources, spec.newGenericResource(nil, nil, "/public", "/a/"+name, name, "css"))
+			resources = append(resources, spec.newGenericResource(nil, nil, "/a/"+name, name, "css"))
 		}
 		b.StartTimer()
 

resource/testhelpers_test.go

@@ -4,6 +4,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"fmt"
 	"image"
 	"io"
 	"io/ioutil"
@@ -27,7 +28,8 @@ func newTestResourceSpec(assert *require.Assertions) *Spec {
 func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *Spec {
 	cfg := viper.New()
 	cfg.Set("baseURL", baseURL)
-	cfg.Set("resourceDir", "/res")
+	cfg.Set("resourceDir", "resources")
+	cfg.Set("contentDir", "content")
 
 	imagingCfg := map[string]interface{}{
 		"resampleFilter": "linear",
@@ -60,9 +62,8 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec {
 		workDir = "/private" + workDir
 	}
 
-	contentDir := "base"
 	cfg.Set("workingDir", workDir)
-	cfg.Set("contentDir", contentDir)
+	cfg.Set("contentDir", filepath.Join(workDir, "content"))
 	cfg.Set("resourceDir", filepath.Join(workDir, "res"))
 
 	fs := hugofs.NewFrom(hugofs.Os, cfg)
@@ -97,10 +98,8 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R
 	src, err := os.Open(filepath.FromSlash("testdata/" + name))
 	assert.NoError(err)
 
-	workingDir := spec.Cfg.GetString("workingDir")
-	f := filepath.Join(workingDir, name)
-
-	out, err := spec.Fs.Source.Create(f)
+	assert.NoError(spec.BaseFs.ContentFs.MkdirAll(filepath.Dir(name), 0755))
+	out, err := spec.BaseFs.ContentFs.Create(name)
 	assert.NoError(err)
 	_, err = io.Copy(out, src)
 	out.Close()
@@ -111,14 +110,17 @@ func fetchResourceForSpec(spec *Spec, assert *require.Assertions, name string) R
 		return path.Join("/a", s)
 	}
 
-	r, err := spec.NewResourceFromFilename(factory, "/public", f, name)
+	r, err := spec.NewResourceFromFilename(factory, name, name)
 	assert.NoError(err)
 
 	return r
 }
 
-func assertImageFile(assert *require.Assertions, fs *hugofs.Fs, filename string, width, height int) {
-	f, err := fs.Source.Open(filename)
+func assertImageFile(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
+	f, err := fs.Open(filename)
+	if err != nil {
+		printFs(fs, "", os.Stdout)
+	}
 	assert.NoError(err)
 	defer f.Close()
 
@@ -129,8 +131,8 @@ func assertImageFile(assert *require.Assertions, fs *hugofs.Fs, filename string,
 	assert.Equal(height, config.Height)
 }
 
-func assertFileCache(assert *require.Assertions, fs *hugofs.Fs, filename string, width, height int) {
-	assertImageFile(assert, fs, filepath.Join("/res/_gen/images", filename), width, height)
+func assertFileCache(assert *require.Assertions, fs afero.Fs, filename string, width, height int) {
+	assertImageFile(assert, fs, filepath.Join("_gen/images", filename), width, height)
 }
 
 func writeSource(t testing.TB, fs *hugofs.Fs, filename, content string) {
@@ -142,3 +144,22 @@ func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
 		t.Fatalf("Failed to write file: %s", err)
 	}
 }
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+		if info != nil && !info.IsDir() {
+			s := path
+			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+				s = s + "\t" + lang.Lang()
+			}
+			if fp, ok := info.(hugofs.FilePather); ok {
+				s += "\tFilename: " + fp.Filename() + "\tBase: " + fp.BaseDir()
+			}
+			fmt.Fprintln(w, "    ", s)
+		}
+		return nil
+	})
+}

source/content_directory_test.go

@@ -17,11 +17,15 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
 )
 
 func TestIgnoreDotFilesAndDirectories(t *testing.T) {
+	assert := require.New(t)
 
 	tests := []struct {
 		path                string
@@ -35,7 +39,6 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 		{"foobar/.barfoo.md", true, nil},
 		{".barfoo.md", true, nil},
 		{".md", true, nil},
-		{"", true, nil},
 		{"foobar/barfoo.md~", true, nil},
 		{".foobar/barfoo.md~", true, nil},
 		{"foobar~/barfoo.md", false, nil},
@@ -51,9 +54,13 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 	for i, test := range tests {
 
 		v := viper.New()
+		v.Set("contentDir", "content")
 		v.Set("ignoreFiles", test.ignoreFilesRegexpes)
+		fs := hugofs.NewMem(v)
+		ps, err := helpers.NewPathSpec(fs, v)
+		assert.NoError(err)
 
-		s := NewSourceSpec(v, hugofs.NewMem(v))
+		s := NewSourceSpec(ps, fs.Source)
 
 		if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored {
 			t.Errorf("[%d] File not ignored", i)

source/dirs_test.go

@@ -101,6 +101,8 @@ func TestStaticDirs(t *testing.T) {
 	for i, test := range tests {
 		msg := fmt.Sprintf("Test %d", i)
 		v := viper.New()
+		v.Set("contentDir", "content")
+
 		fs := hugofs.NewMem(v)
 		cfg := test.setup(v, fs)
 		cfg.Set("workingDir", filepath.FromSlash("/work"))
@@ -134,6 +136,7 @@ func TestStaticDirsFs(t *testing.T) {
 	v.Set("workingDir", filepath.FromSlash("/work"))
 	v.Set("theme", "mytheme")
 	v.Set("themesDir", "themes")
+	v.Set("contentDir", "content")
 	v.Set("staticDir", []string{"s1", "s2"})
 	v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
 

source/fileInfo.go

@@ -14,12 +14,17 @@
 package source
 
 import (
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/gohugoio/hugo/helpers"
 )
 
@@ -86,6 +91,9 @@ type FileInfo struct {
 
 	// Absolute filename to the file on disk.
 	filename string
+
+	sp *SourceSpec
+
 	fi os.FileInfo
 
 	// Derived from filename
@@ -104,8 +112,6 @@ type FileInfo struct {
 
 	uniqueID string
 
-	sp *SourceSpec
-
 	lazyInit sync.Once
 }
 
@@ -146,7 +152,6 @@ func (fi *FileInfo) init() {
 	fi.lazyInit.Do(func() {
 		relDir := strings.Trim(fi.relDir, helpers.FilePathSeparator)
 		parts := strings.Split(relDir, helpers.FilePathSeparator)
-
 		var section string
 		if (!fi.isLeafBundle && len(parts) == 1) || len(parts) > 1 {
 			section = parts[0]
@@ -161,6 +166,19 @@ func (fi *FileInfo) init() {
 
 func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, fi os.FileInfo) *FileInfo {
 
+	var lang, translationBaseName, relPath string
+
+	if fp, ok := fi.(hugofs.FilePather); ok {
+		filename = fp.Filename()
+		baseDir = fp.BaseDir()
+		relPath = fp.Path()
+	}
+
+	if fl, ok := fi.(hugofs.LanguageAnnouncer); ok {
+		lang = fl.Lang()
+		translationBaseName = fl.TranslationBaseName()
+	}
+
 	dir, name := filepath.Split(filename)
 	if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
 		dir = dir + helpers.FilePathSeparator
@@ -175,19 +193,20 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
 
 	relDir = strings.TrimPrefix(relDir, helpers.FilePathSeparator)
 
-	relPath := filepath.Join(relDir, name)
+	if relPath == "" {
+		relPath = filepath.Join(relDir, name)
+	}
 
 	ext := strings.ToLower(strings.TrimPrefix(filepath.Ext(name), "."))
 	baseName := helpers.Filename(name)
 
-	lang := strings.TrimPrefix(filepath.Ext(baseName), ".")
-	var translationBaseName string
-
-	if _, ok := sp.Languages[lang]; lang == "" || !ok {
-		lang = sp.DefaultContentLanguage
-		translationBaseName = baseName
-	} else {
-		translationBaseName = helpers.Filename(baseName)
+	if translationBaseName == "" {
+		// This is usyally provided by the filesystem. But this FileInfo is also
+		// created in a standalone context when doing "hugo new". This is
+		// an approximate implementation, which is "good enough" in that case.
+		translationBaseName = strings.TrimSuffix(baseName, ext)
+		fileLangExt := filepath.Ext(translationBaseName)
+		translationBaseName = strings.TrimSuffix(translationBaseName, fileLangExt)
 	}
 
 	f := &FileInfo{
@@ -211,5 +230,27 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
 
 // Open implements ReadableFile.
 func (fi *FileInfo) Open() (io.ReadCloser, error) {
-	return fi.sp.Fs.Source.Open(fi.Filename())
+	f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename())
+	return f, err
+}
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+
+		if info != nil && !info.IsDir() {
+
+			s := path
+			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+				s = s + "\t" + lang.Lang()
+			}
+			if fp, ok := info.(hugofs.FilePather); ok {
+				s = s + "\t" + fp.Filename()
+			}
+			fmt.Fprintln(w, "    ", s)
+		}
+		return nil
+	})
 }

source/fileInfo_test.go

@@ -17,6 +17,12 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
+	"github.com/spf13/viper"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 )
 
@@ -35,6 +41,8 @@ func TestFileInfo(t *testing.T) {
 			assert.Equal(filepath.FromSlash("b/"), f.Dir())
 			assert.Equal(filepath.FromSlash("b/page.md"), f.Path())
 			assert.Equal("b", f.Section())
+			assert.Equal(filepath.FromSlash("page"), f.TranslationBaseName())
+			assert.Equal(filepath.FromSlash("page"), f.BaseFileName())
 
 		}},
 		{filepath.FromSlash("/a/"), filepath.FromSlash("/a/b/c/d/page.md"), func(f *FileInfo) {
@@ -47,3 +55,39 @@ func TestFileInfo(t *testing.T) {
 	}
 
 }
+
+func TestFileInfoLanguage(t *testing.T) {
+	assert := require.New(t)
+	langs := map[string]bool{
+		"sv": true,
+		"en": true,
+	}
+
+	m := afero.NewMemMapFs()
+	lfs := hugofs.NewLanguageFs("sv", langs, m)
+	v := viper.New()
+	v.Set("contentDir", "content")
+
+	fs := hugofs.NewFrom(m, v)
+
+	ps, err := helpers.NewPathSpec(fs, v)
+	assert.NoError(err)
+	s := SourceSpec{Fs: lfs, PathSpec: ps}
+	s.Languages = map[string]interface{}{
+		"en": true,
+	}
+
+	err = afero.WriteFile(lfs, "page.md", []byte("abc"), 0777)
+	assert.NoError(err)
+	err = afero.WriteFile(lfs, "page.en.md", []byte("abc"), 0777)
+	assert.NoError(err)
+
+	sv, _ := lfs.Stat("page.md")
+	en, _ := lfs.Stat("page.en.md")
+
+	fiSv := s.NewFileInfo("", "page.md", false, sv)
+	fiEn := s.NewFileInfo("", "page.en.md", false, en)
+
+	assert.Equal("sv", fiSv.Lang())
+	assert.Equal("en", fiEn.Lang())
+}

source/filesystem.go

@@ -82,11 +82,11 @@ func (f *Filesystem) captureFiles() {
 	if f.Fs == nil {
 		panic("Must have a fs")
 	}
-	err := helpers.SymbolicWalk(f.Fs.Source, f.Base, walker)
+	err := helpers.SymbolicWalk(f.Fs, f.Base, walker)
 
 	if err != nil {
 		jww.ERROR.Println(err)
-		if err == helpers.ErrWalkRootTooShort {
+		if err == helpers.ErrPathTooShort {
 			panic("The root path is too short. If this is a test, make sure to init the content paths.")
 		}
 	}
@@ -100,7 +100,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
 			jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
 			return false, nil
 		}
-		linkfi, err := f.Fs.Source.Stat(link)
+		linkfi, err := f.Fs.Stat(link)
 		if err != nil {
 			jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
 			return false, nil

source/filesystem_test.go

@@ -18,6 +18,8 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
+
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
@@ -69,5 +71,7 @@ func TestUnicodeNorm(t *testing.T) {
 
 func newTestSourceSpec() SourceSpec {
 	v := viper.New()
-	return SourceSpec{Fs: hugofs.NewMem(v), Cfg: v}
+	v.Set("contentDir", "content")
+	ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v)
+	return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps}
 }

source/sourceSpec.go

@@ -18,17 +18,18 @@ import (
 	"path/filepath"
 	"regexp"
 
-	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+
 	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/cast"
 )
 
 // SourceSpec abstracts language-specific file creation.
 // TODO(bep) rename to Spec
 type SourceSpec struct {
-	Cfg config.Provider
-	Fs  *hugofs.Fs
+	*helpers.PathSpec
+
+	Fs afero.Fs
 
 	// This is set if the ignoreFiles config is set.
 	ignoreFilesRe []*regexp.Regexp
@@ -38,8 +39,9 @@ type SourceSpec struct {
 	DisabledLanguages      map[string]bool
 }
 
-// NewSourceSpec initializes SourceSpec using languages from a given configuration.
-func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec {
+// NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec.
+func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
+	cfg := ps.Cfg
 	defaultLang := cfg.GetString("defaultContentLanguage")
 	languages := cfg.GetStringMap("languages")
 
@@ -69,10 +71,17 @@ func NewSourceSpec(cfg config.Provider, fs *hugofs.Fs) *SourceSpec {
 		}
 	}
 
-	return &SourceSpec{ignoreFilesRe: regexps, Cfg: cfg, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+	return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
 }
 
 func (s *SourceSpec) IgnoreFile(filename string) bool {
+	if filename == "" {
+		if _, ok := s.Fs.(*afero.OsFs); ok {
+			return true
+		}
+		return false
+	}
+
 	base := filepath.Base(filename)
 
 	if len(base) > 0 {
@@ -99,7 +108,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool {
 }
 
 func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
-	fi, err := helpers.LstatIfOs(s.Fs.Source, filename)
+	fi, err := helpers.LstatIfPossible(s.Fs, filename)
 	if err != nil {
 		return false, err
 	}
@@ -110,7 +119,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
 
 	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 		link, err := filepath.EvalSymlinks(filename)
-		fi, err = helpers.LstatIfOs(s.Fs.Source, link)
+		fi, err = helpers.LstatIfPossible(s.Fs, link)
 		if err != nil {
 			return false, err
 		}

tpl/collections/collections_test.go

@@ -369,7 +369,7 @@ func TestIntersect(t *testing.T) {
 func TestIsSet(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	ns := newTestNs()
 
 	for i, test := range []struct {
 		a      interface{}
@@ -787,3 +787,9 @@ func newDeps(cfg config.Provider) *deps.Deps {
 		Log:         jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime),
 	}
 }
+
+func newTestNs() *Namespace {
+	v := viper.New()
+	v.Set("contentDir", "content")
+	return New(newDeps(v))
+}

tpl/data/data_test.go

@@ -21,7 +21,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -29,7 +28,7 @@ import (
 func TestGetCSV(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	ns := newTestNs()
 
 	for i, test := range []struct {
 		sep     string
@@ -123,7 +122,7 @@ func TestGetCSV(t *testing.T) {
 func TestGetJSON(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	ns := newTestNs()
 
 	for i, test := range []struct {
 		url     string

tpl/data/resources_test.go

@@ -127,7 +127,7 @@ func TestScpGetRemote(t *testing.T) {
 func TestScpGetRemoteParallel(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	ns := newTestNs()
 
 	content := []byte(`T€st Content 123`)
 	srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
@@ -176,3 +176,9 @@ func newDeps(cfg config.Provider) *deps.Deps {
 		ContentSpec: cs,
 	}
 }
+
+func newTestNs() *Namespace {
+	v := viper.New()
+	v.Set("contentDir", "content")
+	return New(newDeps(v))
+}

tpl/os/os.go

@@ -25,13 +25,28 @@ import (
 
 // New returns a new instance of the os-namespaced template functions.
 func New(deps *deps.Deps) *Namespace {
+
+	// Since Hugo 0.38 we can have multiple content dirs. This can make it hard to
+	// reason about where the file is placed relative to the project root.
+	// To make the {{ readFile .Filename }} variant just work, we create a composite
+	// filesystem that first checks the work dir fs and then the content fs.
+	var rfs afero.Fs
+	if deps.Fs != nil {
+		rfs = deps.Fs.WorkingDir
+		if deps.PathSpec != nil && deps.PathSpec.BaseFs != nil {
+			rfs = afero.NewReadOnlyFs(afero.NewCopyOnWriteFs(deps.PathSpec.BaseFs.ContentFs, deps.Fs.WorkingDir))
+		}
+	}
+
 	return &Namespace{
+		readFileFs: rfs,
 		deps:       deps,
 	}
 }
 
 // Namespace provides template functions for the "os" namespace.
 type Namespace struct {
+	readFileFs afero.Fs
 	deps       *deps.Deps
 }
 
@@ -46,10 +61,10 @@ func (ns *Namespace) Getenv(key interface{}) (string, error) {
 	return _os.Getenv(skey), nil
 }
 
-// readFile reads the file named by filename relative to the given basepath
+// readFile reads the file named by filename in the given filesystem
 // and returns the contents as a string.
 // There is a upper size limit set at 1 megabytes.
-func readFile(fs *afero.BasePathFs, filename string) (string, error) {
+func readFile(fs afero.Fs, filename string) (string, error) {
 	if filename == "" {
 		return "", errors.New("readFile needs a filename")
 	}
@@ -79,7 +94,7 @@ func (ns *Namespace) ReadFile(i interface{}) (string, error) {
 		return "", err
 	}
 
-	return readFile(ns.deps.Fs.WorkingDir, s)
+	return readFile(ns.readFileFs, s)
 }
 
 // ReadDir lists the directory contents relative to the configured WorkingDir.

tpl/tplimpl/template_funcs_test.go

@@ -65,6 +65,7 @@ func TestTemplateFuncsExamples(t *testing.T) {
 
 	v.Set("workingDir", workingDir)
 	v.Set("multilingual", true)
+	v.Set("contentDir", "content")
 	v.Set("baseURL", "http://mysite.com/hugo/")
 	v.Set("CurrentContentLanguage", helpers.NewLanguage("en", v))
 
@@ -125,7 +126,10 @@ func TestPartialCached(t *testing.T) {
 	var data struct {
 	}
 
-	config := newDepsConfig(viper.New())
+	v := viper.New()
+	v.Set("contentDir", "content")
+
+	config := newDepsConfig(v)
 
 	config.WithTemplate = func(templ tpl.TemplateHandler) error {
 		err := templ.AddTemplate("partials/"+name, partial)

tpl/tplimpl/template_test.go

@@ -35,6 +35,7 @@ func TestHTMLEscape(t *testing.T) {
 		"other": "<h1>Hi!</h1>",
 	}
 	v := viper.New()
+	v.Set("contentDir", "content")
 	fs := hugofs.NewMem(v)
 
 	//afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755)

tpl/transform/remarshal_test.go

@@ -25,7 +25,9 @@ import (
 func TestRemarshal(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 	assert := require.New(t)
 
 	tomlExample := `title = "Test Metadata"
@@ -111,7 +113,10 @@ title: Test Metadata
 func TestRemarshalComments(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
+
 	assert := require.New(t)
 
 	input := `
@@ -153,7 +158,9 @@ Hugo = "Rules"
 func TestTestRemarshalError(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 	assert := require.New(t)
 
 	_, err := ns.Remarshal("asdf", "asdf")

tpl/transform/transform_test.go

@@ -32,7 +32,9 @@ type tstNoStringer struct{}
 func TestEmojify(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	for i, test := range []struct {
 		s      interface{}
@@ -60,7 +62,9 @@ func TestEmojify(t *testing.T) {
 func TestHighlight(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	for i, test := range []struct {
 		s      interface{}
@@ -90,7 +94,9 @@ func TestHighlight(t *testing.T) {
 func TestHTMLEscape(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	for i, test := range []struct {
 		s      interface{}
@@ -118,7 +124,9 @@ func TestHTMLEscape(t *testing.T) {
 func TestHTMLUnescape(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	for i, test := range []struct {
 		s      interface{}
@@ -146,7 +154,9 @@ func TestHTMLUnescape(t *testing.T) {
 func TestMarkdownify(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	for i, test := range []struct {
 		s      interface{}
@@ -176,7 +186,9 @@ func TestMarkdownifyBlocksOfText(t *testing.T) {
 
 	assert := require.New(t)
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	text := `
 #First 
@@ -201,7 +213,9 @@ And then some.
 func TestPlainify(t *testing.T) {
 	t.Parallel()
 
-	ns := New(newDeps(viper.New()))
+	v := viper.New()
+	v.Set("contentDir", "content")
+	ns := New(newDeps(v))
 
 	for i, test := range []struct {
 		s      interface{}