Add support for theme composition and inheritance

This commit adds support for theme composition and inheritance in Hugo.

With this, it helps thinking about a theme as a set of ordered components:

```toml
theme = ["my-shortcodes", "base-theme", "hyde"]
```

The theme definition example above in `config.toml` creates a theme with the 3 components with presedence from left to right.

So, Hugo will, for any given file, data entry etc., look first in the project, and then in `my-shortcode`, `base-theme` and lastly `hyde`.

Hugo uses two different algorithms to merge the filesystems, depending on the file type:

* For `i18n` and `data` files, Hugo merges deeply using the translation id and data key inside the files.
* For `static`, `layouts` (templates) and `archetypes` files, these are merged on file level. So the left-most file will be chosen.

The name used in the `theme` definition above must match a folder in `/your-site/themes`, e.g. `/your-site/themes/my-shortcodes`. There are  plans to improve on this and get a URL scheme so this can be resolved automatically.

Also note that a component that is part of a theme can have its own configuration file, e.g. `config.toml`. There are currently some restrictions to what a theme component can configure:

* `params` (global and per language)
* `menu` (global and per language)
* `outputformats` and `mediatypes`

The same rules apply here: The left-most param/menu etc. with the same ID will win. There are some hidden and experimental namespace support in the above, which we will work to improve in the future, but theme authors are encouraged to create their own namespaces to avoid naming conflicts.

A final note: Themes/components can also have a `theme` definition in their `config.toml` and similar, which is the "inheritance" part of this commit's title. This is currently not supported by the Hugo theme site. We will have to wait for some "auto dependency" feature to be implemented for that to happen, but this can be a powerful feature if you want to create your own theme-variant based on others.

Fixes #4460
Fixes #4450

Gopkg.lock

@@ -298,8 +298,8 @@
     ".",
     "mem"
   ]
-  revision = "63644898a8da0bc22138abf860edaf5277b6102e"
+  revision = "787d034dfe70e44075ccc060d346146ef53270ad"
-  version = "v1.1.0"
+  version = "v1.1.1"
 
 [[projects]]
   name = "github.com/spf13/cast"

commands/benchmark.go

@@ -56,7 +56,7 @@ func (c *benchmarkCmd) benchmark(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	comm, err := initializeConfig(false, &c.hugoBuilderCommon, c, cfgInit)
+	comm, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, cfgInit)
 	if err != nil {
 		return err
 	}

commands/commandeer.go

@@ -34,7 +34,7 @@ import (
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
-	src "github.com/gohugoio/hugo/source"
+	"github.com/gohugoio/hugo/langs"
 )
 
 type commandeer struct {
@@ -45,11 +45,8 @@ type commandeer struct {
 	h    *hugoBuilderCommon
 	ftch flagsToConfigHandler
 
-	pathSpec    *helpers.PathSpec
 	visitedURLs *types.EvictingStringQueue
 
-	staticDirsConfig []*src.Dirs
-
 	// We watch these for changes.
 	configFiles []string
 
@@ -63,7 +60,7 @@ type commandeer struct {
 
 	serverPorts         []int
 	languagesConfigured bool
-	languages           helpers.Languages
+	languages           langs.Languages
 
 	configured bool
 }
@@ -75,31 +72,13 @@ func (c *commandeer) Set(key string, value interface{}) {
 	c.Cfg.Set(key, value)
 }
 
-// PathSpec lazily creates a new PathSpec, as all the paths must
-// be configured before it is created.
-func (c *commandeer) PathSpec() *helpers.PathSpec {
-	c.configured = true
-	return c.pathSpec
-}
-
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
 	c.DepsCfg.Fs = fs
-	ps, err := helpers.NewPathSpec(fs, c.Cfg)
-	if err != nil {
-		return err
-	}
-	c.pathSpec = ps
-
-	dirsConfig, err := c.createStaticDirsConfig()
-	if err != nil {
-		return err
-	}
-	c.staticDirsConfig = dirsConfig
 
 	return nil
 }
 
-func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
+func newCommandeer(mustHaveConfigFile, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, doWithCommandeer func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
 
 	var rebuildDebouncer func(f func())
 	if running {
@@ -117,10 +96,10 @@ func newCommandeer(running bool, h *hugoBuilderCommon, f flagsToConfigHandler, d
 		debounce:         rebuildDebouncer,
 	}
 
-	return c, c.loadConfig(running)
+	return c, c.loadConfig(mustHaveConfigFile, running)
 }
 
-func (c *commandeer) loadConfig(running bool) error {
+func (c *commandeer) loadConfig(mustHaveConfigFile, running bool) error {
 
 	if c.DepsCfg == nil {
 		c.DepsCfg = &deps.DepsCfg{}
@@ -168,12 +147,18 @@ func (c *commandeer) loadConfig(running bool) error {
 		doWithConfig)
 
 	if err != nil {
-		return err
+		if mustHaveConfigFile {
+			return err
+		}
+		if err != hugolib.ErrNoConfigFile {
+			return err
+		}
+
 	}
 
 	c.configFiles = configFiles
 
-	if l, ok := c.Cfg.Get("languagesSorted").(helpers.Languages); ok {
+	if l, ok := c.Cfg.Get("languagesSorted").(langs.Languages); ok {
 		c.languagesConfigured = true
 		c.languages = l
 	}
@@ -209,6 +194,15 @@ func (c *commandeer) loadConfig(running bool) error {
 		}
 
 		err = c.initFs(fs)
+		if err != nil {
+			return
+		}
+
+		var h *hugolib.HugoSites
+
+		h, err = hugolib.NewHugoSites(*c.DepsCfg)
+		c.hugo = h
+
 	})
 
 	if err != nil {
@@ -232,7 +226,7 @@ func (c *commandeer) loadConfig(running bool) error {
 
 	cfg.Logger.INFO.Println("Using config file:", config.ConfigFileUsed())
 
-	themeDir := c.PathSpec().GetThemeDir()
+	themeDir := c.hugo.PathSpec.GetFirstThemeDir()
 	if themeDir != "" {
 		if _, err := sourceFs.Stat(themeDir); os.IsNotExist(err) {
 			return newSystemError("Unable to find theme Directory:", themeDir)

commands/commands.go

@@ -148,7 +148,7 @@ Complete documentation is available at http://gohugo.io/.`,
 				return nil
 			}
 
-			c, err := initializeConfig(cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit)
+			c, err := initializeConfig(true, cc.buildWatch, &cc.hugoBuilderCommon, cc, cfgInit)
 			if err != nil {
 				return err
 			}

commands/commands_test.go

@@ -235,6 +235,11 @@ Single: {{ .Title }}
 
 List: {{ .Title }}
 
+`)
+
+	writeFile(t, filepath.Join(d, "static", "my.txt"), `
+MyMy
+
 `)
 
 	return d, nil

commands/config.go

@@ -44,7 +44,7 @@ func newConfigCmd() *configCmd {
 }
 
 func (c *configCmd) printConfig(cmd *cobra.Command, args []string) error {
-	cfg, err := initializeConfig(false, &c.hugoBuilderCommon, c, nil)
+	cfg, err := initializeConfig(true, false, &c.hugoBuilderCommon, c, nil)
 
 	if err != nil {
 		return err

commands/convert.go

@@ -96,7 +96,7 @@ func (cc *convertCmd) convertContents(mark rune) error {
 		return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
 	}
 
-	c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, nil)
+	c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, nil)
 	if err != nil {
 		return err
 	}

commands/hugo.go

@@ -23,6 +23,8 @@ import (
 	"sync/atomic"
 	"syscall"
 
+	"github.com/gohugoio/hugo/hugolib/filesystems"
+
 	"golang.org/x/sync/errgroup"
 
 	"log"
@@ -32,8 +34,6 @@ import (
 	"strings"
 	"time"
 
-	src "github.com/gohugoio/hugo/source"
-
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/parser"
@@ -103,12 +103,12 @@ func Execute(args []string) Response {
 }
 
 // InitializeConfig initializes a config file with sensible default configuration flags.
-func initializeConfig(running bool,
+func initializeConfig(mustHaveConfigFile, running bool,
 	h *hugoBuilderCommon,
 	f flagsToConfigHandler,
 	doWithCommandeer func(c *commandeer) error) (*commandeer, error) {
 
-	c, err := newCommandeer(running, h, f, doWithCommandeer)
+	c, err := newCommandeer(mustHaveConfigFile, running, h, f, doWithCommandeer)
 	if err != nil {
 		return nil, err
 	}
@@ -280,6 +280,7 @@ func (c *commandeer) fullBuild() error {
 			return fmt.Errorf("Error copying static files: %s", err)
 		}
 		langCount = cnt
+		langCount = cnt
 		return nil
 	}
 	buildSitesFunc := func() error {
@@ -344,7 +345,7 @@ func (c *commandeer) build() error {
 		if err != nil {
 			return err
 		}
-		c.Logger.FEEDBACK.Println("Watching for changes in", c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")))
+		c.Logger.FEEDBACK.Println("Watching for changes in", c.hugo.PathSpec.AbsPathify(c.Cfg.GetString("contentDir")))
 		c.Logger.FEEDBACK.Println("Press Ctrl+C to stop")
 		watcher, err := c.newWatcher(watchDirs...)
 		utils.CheckErr(c.Logger, err)
@@ -380,49 +381,30 @@ func (c *commandeer) copyStatic() (map[string]uint64, error) {
 	return c.doWithPublishDirs(c.copyStaticTo)
 }
 
-func (c *commandeer) createStaticDirsConfig() ([]*src.Dirs, error) {
+func (c *commandeer) doWithPublishDirs(f func(sourceFs *filesystems.SourceFilesystem) (uint64, error)) (map[string]uint64, error) {
-	var dirsConfig []*src.Dirs
-
-	if !c.languages.IsMultihost() {
-		dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
-		if err != nil {
-			return nil, err
-		}
-		dirsConfig = append(dirsConfig, dirs)
-	} else {
-		for _, l := range c.languages {
-			dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger)
-			if err != nil {
-				return nil, err
-			}
-			dirsConfig = append(dirsConfig, dirs)
-		}
-	}
-
-	return dirsConfig, nil
-
-}
-
-func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) (uint64, error)) (map[string]uint64, error) {
 
 	langCount := make(map[string]uint64)
 
-	for _, dirs := range c.staticDirsConfig {
+	staticFilesystems := c.hugo.BaseFs.SourceFilesystems.Static
 
-		cnt, err := f(dirs, c.pathSpec.PublishDir)
+	if len(staticFilesystems) == 0 {
+		c.Logger.WARN.Println("No static directories found to sync")
+		return langCount, nil
+	}
+
+	for lang, fs := range staticFilesystems {
+		cnt, err := f(fs)
 		if err != nil {
 			return langCount, err
 		}
-
+		if lang == "" {
-		if dirs.Language == nil {
 			// Not multihost
 			for _, l := range c.languages {
 				langCount[l.Lang] = cnt
 			}
 		} else {
-			langCount[dirs.Language.Lang] = cnt
+			langCount[lang] = cnt
 		}
-
 	}
 
 	return langCount, nil
@@ -443,29 +425,18 @@ func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) {
 	return f, err
 }
 
-func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, error) {
+func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
-
+	publishDir := c.hugo.PathSpec.PublishDir
 	// If root, remove the second '/'
 	if publishDir == "//" {
 		publishDir = helpers.FilePathSeparator
 	}
 
-	if dirs.Language != nil {
+	if sourceFs.PublishFolder != "" {
-		// Multihost setup.
+		publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
-		publishDir = filepath.Join(publishDir, dirs.Language.Lang)
 	}
 
-	staticSourceFs, err := dirs.CreateStaticFs()
+	fs := &countingStatFs{Fs: sourceFs.Fs}
-	if err != nil {
-		return 0, err
-	}
-
-	if staticSourceFs == nil {
-		c.Logger.WARN.Println("No static directories found to sync")
-		return 0, nil
-	}
-
-	fs := &countingStatFs{Fs: staticSourceFs}
 
 	syncer := fsync.NewSyncer()
 	syncer.NoTimes = c.Cfg.GetBool("noTimes")
@@ -485,6 +456,8 @@ func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) (uint64, er
 	}
 	c.Logger.INFO.Println("syncing static files to", publishDir)
 
+	var err error
+
 	// because we are using a baseFs (to get the union right).
 	// set sync src to root
 	err = syncer.Sync(publishDir, helpers.FilePathSeparator)
@@ -514,41 +487,10 @@ func (c *commandeer) getDirList() ([]string, error) {
 	var seen = make(map[string]bool)
 	var nested []string
 
-	dataDir := c.PathSpec().AbsPathify(c.Cfg.GetString("dataDir"))
-	i18nDir := c.PathSpec().AbsPathify(c.Cfg.GetString("i18nDir"))
-	staticSyncer, err := newStaticSyncer(c)
-	if err != nil {
-		return nil, err
-	}
-
-	layoutDir := c.PathSpec().GetLayoutDirPath()
-	staticDirs := staticSyncer.d.AbsStaticDirs
-
 	newWalker := func(allowSymbolicDirs bool) func(path string, fi os.FileInfo, err error) error {
 		return func(path string, fi os.FileInfo, err error) error {
 			if err != nil {
-				if path == dataDir && os.IsNotExist(err) {
-					c.Logger.WARN.Println("Skip dataDir:", err)
-					return nil
-				}
-
-				if path == i18nDir && os.IsNotExist(err) {
-					c.Logger.WARN.Println("Skip i18nDir:", err)
-					return nil
-				}
-
-				if path == layoutDir && os.IsNotExist(err) {
-					c.Logger.WARN.Println("Skip layoutDir:", err)
-					return nil
-				}
-
 				if os.IsNotExist(err) {
-					for _, staticDir := range staticDirs {
-						if path == staticDir && os.IsNotExist(err) {
-							c.Logger.WARN.Println("Skip staticDir:", err)
-						}
-					}
-					// Ignore.
 					return nil
 				}
 
@@ -605,23 +547,28 @@ func (c *commandeer) getDirList() ([]string, error) {
 	regularWalker := newWalker(false)
 
 	// SymbolicWalk will log anny ERRORs
-	_ = helpers.SymbolicWalk(c.Fs.Source, dataDir, regularWalker)
+	// Also note that the Dirnames fetched below will contain any relevant theme
-	_ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, regularWalker)
+	// directories.
-	_ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, regularWalker)
+	for _, contentDir := range c.hugo.PathSpec.BaseFs.AbsContentDirs {
-
-	for _, contentDir := range c.PathSpec().ContentDirs() {
 		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir.Value, symLinkWalker)
 	}
 
-	for _, staticDir := range staticDirs {
+	for _, staticDir := range c.hugo.PathSpec.BaseFs.Data.Dirnames {
 		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
 	}
 
-	if c.PathSpec().ThemeSet() {
+	for _, staticDir := range c.hugo.PathSpec.BaseFs.I18n.Dirnames {
-		themesDir := c.PathSpec().GetThemeDir()
+		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
-		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "layouts"), regularWalker)
+	}
-		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "i18n"), regularWalker)
+
-		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "data"), regularWalker)
+	for _, staticDir := range c.hugo.PathSpec.BaseFs.Layouts.Dirnames {
+		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
+	}
+
+	for _, staticFilesystem := range c.hugo.PathSpec.BaseFs.Static {
+		for _, staticDir := range staticFilesystem.Dirnames {
+			_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, regularWalker)
+		}
 	}
 
 	if len(nested) > 0 {
@@ -648,9 +595,6 @@ func (c *commandeer) getDirList() ([]string, error) {
 
 func (c *commandeer) recreateAndBuildSites(watching bool) (err error) {
 	defer c.timeTrack(time.Now(), "Total")
-	if err := c.initSites(); err != nil {
-		return err
-	}
 	if !c.h.quiet {
 		c.Logger.FEEDBACK.Println("Started building sites ...")
 	}
@@ -658,56 +602,30 @@ func (c *commandeer) recreateAndBuildSites(watching bool) (err error) {
 }
 
 func (c *commandeer) resetAndBuildSites() (err error) {
-	if err = c.initSites(); err != nil {
-		return
-	}
 	if !c.h.quiet {
 		c.Logger.FEEDBACK.Println("Started building sites ...")
 	}
 	return c.hugo.Build(hugolib.BuildCfg{ResetState: true})
 }
 
-func (c *commandeer) initSites() error {
-	if c.hugo != nil {
-		c.hugo.Cfg = c.Cfg
-		return nil
-	}
-
-	h, err := hugolib.NewHugoSites(*c.DepsCfg)
-
-	if err != nil {
-		return err
-	}
-
-	c.hugo = h
-
-	return nil
-}
-
 func (c *commandeer) buildSites() (err error) {
-	if err := c.initSites(); err != nil {
-		return err
-	}
 	return c.hugo.Build(hugolib.BuildCfg{})
 }
 
 func (c *commandeer) rebuildSites(events []fsnotify.Event) error {
 	defer c.timeTrack(time.Now(), "Total")
 
-	if err := c.initSites(); err != nil {
-		return err
-	}
 	visited := c.visitedURLs.PeekAllSet()
 	doLiveReload := !c.h.buildWatch && !c.Cfg.GetBool("disableLiveReload")
 	if doLiveReload && !c.Cfg.GetBool("disableFastRender") {
 
 		// Make sure we always render the home pages
 		for _, l := range c.languages {
-			langPath := c.PathSpec().GetLangSubDir(l.Lang)
+			langPath := c.hugo.PathSpec.GetLangSubDir(l.Lang)
 			if langPath != "" {
 				langPath = langPath + "/"
 			}
-			home := c.pathSpec.PrependBasePath("/" + langPath)
+			home := c.hugo.PathSpec.PrependBasePath("/" + langPath)
 			visited[home] = true
 		}
 
@@ -716,7 +634,7 @@ func (c *commandeer) rebuildSites(events []fsnotify.Event) error {
 }
 
 func (c *commandeer) fullRebuild() {
-	if err := c.loadConfig(true); err != nil {
+	if err := c.loadConfig(true, true); err != nil {
 		jww.ERROR.Println("Failed to reload config:", err)
 	} else if err := c.recreateAndBuildSites(true); err != nil {
 		jww.ERROR.Println(err)
@@ -906,7 +824,8 @@ func (c *commandeer) newWatcher(dirList ...string) (*watcher.Batcher, error) {
 						// force refresh when more than one file
 						if len(staticEvents) > 0 {
 							for _, ev := range staticEvents {
-								path := staticSyncer.d.MakeStaticPathRelative(ev.Name)
+
+								path := c.hugo.BaseFs.SourceFilesystems.MakeStaticPathRelative(ev.Name)
 								livereload.RefreshPath(path)
 							}
 
@@ -975,32 +894,36 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
 }
 
 // isThemeVsHugoVersionMismatch returns whether the current Hugo version is
-// less than the theme's min_version.
+// less than any of the themes' min_version.
 func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (mismatch bool, requiredMinVersion string) {
-	if !c.PathSpec().ThemeSet() {
+	if !c.hugo.PathSpec.ThemeSet() {
 		return
 	}
 
-	themeDir := c.PathSpec().GetThemeDir()
+	for _, absThemeDir := range c.hugo.BaseFs.AbsThemeDirs {
 
-	path := filepath.Join(themeDir, "theme.toml")
+		path := filepath.Join(absThemeDir, "theme.toml")
 
-	exists, err := helpers.Exists(path, fs)
+		exists, err := helpers.Exists(path, fs)
 
-	if err != nil || !exists {
+		if err != nil || !exists {
-		return
+			continue
-	}
+		}
 
-	b, err := afero.ReadFile(fs, path)
+		b, err := afero.ReadFile(fs, path)
 
-	tomlMeta, err := parser.HandleTOMLMetaData(b)
+		tomlMeta, err := parser.HandleTOMLMetaData(b)
 
-	if err != nil {
+		if err != nil {
-		return
+			continue
-	}
+		}
+
+		if minVersion, ok := tomlMeta["min_version"]; ok {
+			if helpers.CompareVersion(minVersion) > 0 {
+				return true, fmt.Sprint(minVersion)
+			}
+		}
 
-	if minVersion, ok := tomlMeta["min_version"]; ok {
-		return helpers.CompareVersion(minVersion) > 0, fmt.Sprint(minVersion)
 	}
 
 	return

commands/list.go

@@ -50,7 +50,7 @@ List requires a subcommand, e.g. ` + "`hugo list drafts`.",
 					c.Set("buildDrafts", true)
 					return nil
 				}
-				c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit)
+				c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
 				if err != nil {
 					return err
 				}
@@ -86,7 +86,7 @@ posted in the future.`,
 					c.Set("buildFuture", true)
 					return nil
 				}
-				c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit)
+				c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
 				if err != nil {
 					return err
 				}
@@ -122,7 +122,7 @@ expired.`,
 					c.Set("buildExpired", true)
 					return nil
 				}
-				c, err := initializeConfig(false, &cc.hugoBuilderCommon, cc, cfgInit)
+				c, err := initializeConfig(true, false, &cc.hugoBuilderCommon, cc, cfgInit)
 				if err != nil {
 					return err
 				}

commands/new.go

@@ -71,7 +71,7 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error {
 		return nil
 	}
 
-	c, err := initializeConfig(false, &n.hugoBuilderCommon, n, cfgInit)
+	c, err := initializeConfig(true, false, &n.hugoBuilderCommon, n, cfgInit)
 
 	if err != nil {
 		return err
@@ -104,9 +104,6 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error {
 			return hugolib.NewSite(*cfg)
 		}
 		var s *hugolib.Site
-		if err := c.initSites(); err != nil {
-			return nil, err
-		}
 
 		if err := c.hugo.Build(hugolib.BuildCfg{SkipRender: true}); err != nil {
 			return nil, err

commands/new_theme.go

@@ -54,7 +54,7 @@ as you see fit.`,
 }
 
 func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error {
-	c, err := initializeConfig(false, &n.hugoBuilderCommon, n, nil)
+	c, err := initializeConfig(false, false, &n.hugoBuilderCommon, n, nil)
 
 	if err != nil {
 		return err
@@ -64,7 +64,7 @@ func (n *newThemeCmd) newTheme(cmd *cobra.Command, args []string) error {
 		return newUserError("theme name needs to be provided")
 	}
 
-	createpath := c.PathSpec().AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0]))
+	createpath := c.hugo.PathSpec.AbsPathify(filepath.Join(c.Cfg.GetString("themesDir"), args[0]))
 	jww.INFO.Println("creating theme at", createpath)
 
 	cfg := c.DepsCfg
@@ -140,7 +140,7 @@ description = ""
 homepage = "http://example.com/"
 tags = []
 features = []
-min_version = "0.38"
+min_version = "0.41"
 
 [author]
   name = ""

commands/server.go

@@ -226,7 +226,7 @@ func (s *serverCmd) server(cmd *cobra.Command, args []string) error {
 		jww.ERROR.Println("memstats error:", err)
 	}
 
-	c, err := initializeConfig(true, &s.hugoBuilderCommon, s, cfgInit)
+	c, err := initializeConfig(true, true, &s.hugoBuilderCommon, s, cfgInit)
 	if err != nil {
 		return err
 	}
@@ -288,7 +288,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
 		publishDir = filepath.Join(publishDir, root)
 	}
 
-	absPublishDir := f.c.PathSpec().AbsPathify(publishDir)
+	absPublishDir := f.c.hugo.PathSpec.AbsPathify(publishDir)
 
 	if i == 0 {
 		if f.s.renderToDisk {

commands/static_syncer.go

@@ -17,53 +17,43 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/gohugoio/hugo/hugolib/filesystems"
+
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/helpers"
-	src "github.com/gohugoio/hugo/source"
 	"github.com/spf13/fsync"
 )
 
 type staticSyncer struct {
 	c *commandeer
-	d *src.Dirs
 }
 
 func newStaticSyncer(c *commandeer) (*staticSyncer, error) {
-	dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
+	return &staticSyncer{c: c}, nil
-	if err != nil {
-		return nil, err
-	}
-
-	return &staticSyncer{c: c, d: dirs}, nil
 }
 
-func (s *staticSyncer) isStatic(path string) bool {
+func (s *staticSyncer) isStatic(filename string) bool {
-	return s.d.IsStatic(path)
+	return s.c.hugo.BaseFs.SourceFilesystems.IsStatic(filename)
 }
 
 func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 	c := s.c
 
-	syncFn := func(dirs *src.Dirs, publishDir string) (uint64, error) {
+	syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
-		staticSourceFs, err := dirs.CreateStaticFs()
+		publishDir := c.hugo.PathSpec.PublishDir
-		if err != nil {
+		// If root, remove the second '/'
-			return 0, err
+		if publishDir == "//" {
+			publishDir = helpers.FilePathSeparator
 		}
 
-		if dirs.Language != nil {
+		if sourceFs.PublishFolder != "" {
-			// Multihost setup
+			publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
-			publishDir = filepath.Join(publishDir, dirs.Language.Lang)
-		}
-
-		if staticSourceFs == nil {
-			c.Logger.WARN.Println("No static directories found to sync")
-			return 0, nil
 		}
 
 		syncer := fsync.NewSyncer()
 		syncer.NoTimes = c.Cfg.GetBool("noTimes")
 		syncer.NoChmod = c.Cfg.GetBool("noChmod")
-		syncer.SrcFs = staticSourceFs
+		syncer.SrcFs = sourceFs.Fs
 		syncer.DestFs = c.Fs.Destination
 
 		// prevent spamming the log on changes
@@ -88,8 +78,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 
 			fromPath := ev.Name
 
-			// If we are here we already know the event took place in a static dir
+			relPath := sourceFs.MakePathRelative(fromPath)
-			relPath := dirs.MakeStaticPathRelative(fromPath)
 			if relPath == "" {
 				// Not member of this virtual host.
 				continue
@@ -105,7 +94,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// the source of that static file. In this case Hugo will incorrectly remove that file
 			// from the published directory.
 			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
-				if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {
+				if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) {
 					// If file doesn't exist in any static dir, remove it
 					toRemove := filepath.Join(publishDir, relPath)
 

common/loggers/loggers.go

@@ -0,0 +1,37 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loggers
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+// NewDebugLogger is a convenience function to create a debug logger.
+func NewDebugLogger() *jww.Notepad {
+	return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+}
+
+// NewWarningLogger is a convenience function to create a warning logger.
+func NewWarningLogger() *jww.Notepad {
+	return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+}
+
+// NewErrorLogger is a convenience function to create an error logger.
+func NewErrorLogger() *jww.Notepad {
+	return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+}

common/maps/maps.go

@@ -0,0 +1,44 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"strings"
+
+	"github.com/spf13/cast"
+)
+
+// ToLower makes all the keys in the given map lower cased and will do so
+// recursively.
+// Notes:
+// * This will modify the map given.
+// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}.
+func ToLower(m map[string]interface{}) {
+	for k, v := range m {
+		switch v.(type) {
+		case map[interface{}]interface{}:
+			v = cast.ToStringMap(v)
+			ToLower(v.(map[string]interface{}))
+		case map[string]interface{}:
+			ToLower(v.(map[string]interface{}))
+		}
+
+		lKey := strings.ToLower(k)
+		if k != lKey {
+			delete(m, k)
+			m[lKey] = v
+		}
+
+	}
+}

common/maps/maps_test.go

@@ -0,0 +1,72 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestToLower(t *testing.T) {
+
+	tests := []struct {
+		input    map[string]interface{}
+		expected map[string]interface{}
+	}{
+		{
+			map[string]interface{}{
+				"abC": 32,
+			},
+			map[string]interface{}{
+				"abc": 32,
+			},
+		},
+		{
+			map[string]interface{}{
+				"abC": 32,
+				"deF": map[interface{}]interface{}{
+					23: "A value",
+					24: map[string]interface{}{
+						"AbCDe": "A value",
+						"eFgHi": "Another value",
+					},
+				},
+				"gHi": map[string]interface{}{
+					"J": 25,
+				},
+			},
+			map[string]interface{}{
+				"abc": 32,
+				"def": map[string]interface{}{
+					"23": "A value",
+					"24": map[string]interface{}{
+						"abcde": "A value",
+						"efghi": "Another value",
+					},
+				},
+				"ghi": map[string]interface{}{
+					"j": 25,
+				},
+			},
+		},
+	}
+
+	for i, test := range tests {
+		// ToLower modifies input.
+		ToLower(test.input)
+		if !reflect.DeepEqual(test.expected, test.input) {
+			t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input)
+		}
+	}
+}

config/configProvider.go

@@ -16,6 +16,8 @@ package config
 import (
 	"strings"
 
+	"github.com/spf13/cast"
+
 	"github.com/spf13/viper"
 )
 
@@ -40,5 +42,16 @@ func FromConfigString(config, configType string) (Provider, error) {
 		return nil, err
 	}
 	return v, nil
-
+}
+
+// GetStringSlicePreserveString returns a string slice from the given config and key.
+// It differs from the GetStringSlice method in that if the config value is a string,
+// we do not attempt to split it into fields.
+func GetStringSlicePreserveString(cfg Provider, key string) []string {
+	sd := cfg.Get(key)
+	if sds, ok := sd.(string); ok {
+		return []string{sds}
+	} else {
+		return cast.ToStringSlice(sd)
+	}
 }

config/configProvider_test.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetStringSlicePreserveString(t *testing.T) {
+	assert := require.New(t)
+	cfg := viper.New()
+
+	s := "This is a string"
+	sSlice := []string{"This", "is", "a", "slice"}
+
+	cfg.Set("s1", s)
+	cfg.Set("s2", sSlice)
+
+	assert.Equal([]string{s}, GetStringSlicePreserveString(cfg, "s1"))
+	assert.Equal(sSlice, GetStringSlicePreserveString(cfg, "s2"))
+	assert.Nil(GetStringSlicePreserveString(cfg, "s3"))
+}

create/content.go

@@ -16,6 +16,7 @@ package create
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -31,6 +32,7 @@ func NewContent(
 	ps *helpers.PathSpec,
 	siteFactory func(filename string, siteUsed bool) (*hugolib.Site, error), kind, targetPath string) error {
 	ext := helpers.Ext(targetPath)
+	fs := ps.BaseFs.SourceFilesystems.Archetypes.Fs
 
 	jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext)
 
@@ -40,9 +42,9 @@ func NewContent(
 	siteUsed := false
 
 	if archetypeFilename != "" {
-		f, err := ps.Fs.Source.Open(archetypeFilename)
+		f, err := fs.Open(archetypeFilename)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to open archetype file: %s", err)
 		}
 		defer f.Close()
 
@@ -71,7 +73,7 @@ func NewContent(
 	targetDir := filepath.Dir(targetPath)
 
 	if targetDir != "" && targetDir != "." {
-		exists, _ = helpers.Exists(targetDir, ps.Fs.Source)
+		exists, _ = helpers.Exists(targetDir, fs)
 	}
 
 	if exists {
@@ -101,42 +103,27 @@ func NewContent(
 	return nil
 }
 
-// FindArchetype takes a given kind/archetype of content and returns an output
+// FindArchetype takes a given kind/archetype of content and returns the path
-// path for that archetype.  If no archetype is found, an empty string is
+// to the archetype in the archetype filesystem, blank if none found.
-// returned.
 func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string) {
-	search := []string{ps.AbsPathify(ps.Cfg.GetString("archetypeDir"))}
+	fs := ps.BaseFs.Archetypes.Fs
 
-	if ps.Cfg.GetString("theme") != "" {
+	// If the new content isn't in a subdirectory, kind == "".
-		themeDir := filepath.Join(ps.AbsPathify(ps.Cfg.GetString("themesDir")+"/"+ps.Cfg.GetString("theme")), "/archetypes/")
+	// Therefore it should be excluded otherwise `is a directory`
-		if _, err := ps.Fs.Source.Stat(themeDir); os.IsNotExist(err) {
+	// error will occur. github.com/gohugoio/hugo/issues/411
-			jww.ERROR.Printf("Unable to find archetypes directory for theme %q at %q", ps.Cfg.GetString("theme"), themeDir)
+	var pathsToCheck = []string{"default"}
+
+	if ext != "" {
+		if kind != "" {
+			pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...)
 		} else {
-			search = append(search, themeDir)
+			pathsToCheck = append([]string{"default" + ext}, pathsToCheck...)
 		}
 	}
 
-	for _, x := range search {
+	for _, p := range pathsToCheck {
-		// If the new content isn't in a subdirectory, kind == "".
+		if exists, _ := helpers.Exists(p, fs); exists {
-		// Therefore it should be excluded otherwise `is a directory`
+			return p
-		// error will occur. github.com/gohugoio/hugo/issues/411
-		var pathsToCheck = []string{"default"}
-
-		if ext != "" {
-			if kind != "" {
-				pathsToCheck = append([]string{kind + ext, "default" + ext}, pathsToCheck...)
-			} else {
-				pathsToCheck = append([]string{"default" + ext}, pathsToCheck...)
-			}
-		}
-
-		for _, p := range pathsToCheck {
-			curpath := filepath.Join(x, p)
-			jww.DEBUG.Println("checking", curpath, "for archetypes")
-			if exists, _ := helpers.Exists(curpath, ps.Fs.Source); exists {
-				jww.INFO.Println("curpath: " + curpath)
-				return curpath
-			}
 		}
 	}
 

create/content_template_handler.go

@@ -89,10 +89,11 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 	)
 
 	ps, err := helpers.NewPathSpec(s.Deps.Fs, s.Deps.Cfg)
-	sp := source.NewSourceSpec(ps, ps.Fs.Source)
 	if err != nil {
 		return nil, err
 	}
+	sp := source.NewSourceSpec(ps, ps.Fs.Source)
+
 	f := sp.NewFileInfo("", targetPath, false, nil)
 
 	name := f.TranslationBaseName()
@@ -115,9 +116,9 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 		// TODO(bep) archetype revive the issue about wrong tpl funcs arg order
 		archetypeTemplate = []byte(ArchetypeTemplateTemplate)
 	} else {
-		archetypeTemplate, err = afero.ReadFile(s.Fs.Source, archetypeFilename)
+		archetypeTemplate, err = afero.ReadFile(s.BaseFs.Archetypes.Fs, archetypeFilename)
 		if err != nil {
-			return nil, fmt.Errorf("Failed to read archetype file %q: %s", archetypeFilename, err)
+			return nil, fmt.Errorf("failed to read archetype file %s", err)
 		}
 
 	}

create/content_test.go

@@ -58,17 +58,15 @@ func TestNewContent(t *testing.T) {
 
 	for _, c := range cases {
 		cfg, fs := newTestCfg()
-		ps, err := helpers.NewPathSpec(fs, cfg)
+		require.NoError(t, initFs(fs))
-		require.NoError(t, err)
 		h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs})
 		require.NoError(t, err)
-		require.NoError(t, initFs(fs))
 
 		siteFactory := func(filename string, siteUsed bool) (*hugolib.Site, error) {
 			return h.Sites[0], nil
 		}
 
-		require.NoError(t, create.NewContent(ps, siteFactory, c.kind, c.path))
+		require.NoError(t, create.NewContent(h.PathSpec, siteFactory, c.kind, c.path))
 
 		fname := filepath.Join("content", filepath.FromSlash(c.path))
 		content := readFileFromFs(t, fs.Source, fname)
@@ -89,6 +87,7 @@ func initViper(v *viper.Viper) {
 	v.Set("layoutDir", "layouts")
 	v.Set("i18nDir", "i18n")
 	v.Set("theme", "sample")
+	v.Set("archetypeDir", "archetypes")
 }
 
 func initFs(fs *hugofs.Fs) error {
@@ -187,6 +186,12 @@ func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string {
 func newTestCfg() (*viper.Viper, *hugofs.Fs) {
 
 	v := viper.New()
+	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
+
 	fs := hugofs.NewMem(v)
 
 	v.SetFs(fs.Source)

deps/deps.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/metrics"
 	"github.com/gohugoio/hugo/output"
 	"github.com/gohugoio/hugo/source"
@@ -47,7 +48,7 @@ type Deps struct {
 	// The translation func to use
 	Translate func(translationID string, args ...interface{}) string `json:"-"`
 
-	Language *helpers.Language
+	Language *langs.Language
 
 	// All the output formats available for the current site.
 	OutputFormatsConfig output.Formats
@@ -166,10 +167,10 @@ func New(cfg DepsCfg) (*Deps, error) {
 
 // ForLanguage creates a copy of the Deps with the language dependent
 // parts switched out.
-func (d Deps) ForLanguage(l *helpers.Language) (*Deps, error) {
+func (d Deps) ForLanguage(l *langs.Language) (*Deps, error) {
 	var err error
 
-	d.PathSpec, err = helpers.NewPathSpec(d.Fs, l)
+	d.PathSpec, err = helpers.NewPathSpecWithBaseBaseFsProvided(d.Fs, l, d.BaseFs)
 	if err != nil {
 		return nil, err
 	}
@@ -206,7 +207,7 @@ type DepsCfg struct {
 	Fs *hugofs.Fs
 
 	// The language to use.
-	Language *helpers.Language
+	Language *langs.Language
 
 	// The configuration to use.
 	Cfg config.Provider

helpers/content.go

@@ -25,6 +25,8 @@ import (
 	"unicode"
 	"unicode/utf8"
 
+	"github.com/gohugoio/hugo/common/maps"
+
 	"github.com/chaseadamsio/goorgeous"
 	bp "github.com/gohugoio/hugo/bufferpool"
 	"github.com/gohugoio/hugo/config"
@@ -134,7 +136,7 @@ func newBlackfriday(config map[string]interface{}) *BlackFriday {
 		"taskLists":             true,
 	}
 
-	ToLowerMap(defaultParam)
+	maps.ToLower(defaultParam)
 
 	siteConfig := make(map[string]interface{})
 

helpers/general.go

@@ -20,18 +20,20 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 	"unicode"
 	"unicode/utf8"
 
+	"github.com/gohugoio/hugo/hugofs"
+
 	"github.com/spf13/afero"
 
 	"github.com/jdkato/prose/transform"
 
 	bp "github.com/gohugoio/hugo/bufferpool"
-	"github.com/spf13/cast"
 	jww "github.com/spf13/jwalterweatherman"
 	"github.com/spf13/pflag"
 )
@@ -129,30 +131,6 @@ func ReaderToBytes(lines io.Reader) []byte {
 	return bc
 }
 
-// ToLowerMap makes all the keys in the given map lower cased and will do so
-// recursively.
-// Notes:
-// * This will modify the map given.
-// * Any nested map[interface{}]interface{} will be converted to map[string]interface{}.
-func ToLowerMap(m map[string]interface{}) {
-	for k, v := range m {
-		switch v.(type) {
-		case map[interface{}]interface{}:
-			v = cast.ToStringMap(v)
-			ToLowerMap(v.(map[string]interface{}))
-		case map[string]interface{}:
-			ToLowerMap(v.(map[string]interface{}))
-		}
-
-		lKey := strings.ToLower(k)
-		if k != lKey {
-			delete(m, k)
-			m[lKey] = v
-		}
-
-	}
-}
-
 // ReaderToString is the same as ReaderToBytes, but returns a string.
 func ReaderToString(lines io.Reader) string {
 	if lines == nil {
@@ -255,11 +233,6 @@ func compareStringSlices(a, b []string) bool {
 	return true
 }
 
-// ThemeSet checks whether a theme is in use or not.
-func (p *PathSpec) ThemeSet() bool {
-	return p.theme != ""
-}
-
 // LogPrinter is the common interface of the JWWs loggers.
 type LogPrinter interface {
 	// Println is the only common method that works in all of JWWs loggers.
@@ -477,3 +450,24 @@ func DiffStringSlices(slice1 []string, slice2 []string) []string {
 func DiffStrings(s1, s2 string) []string {
 	return DiffStringSlices(strings.Fields(s1), strings.Fields(s2))
 }
+
+// PrintFs prints the given filesystem to the given writer starting from the given path.
+// This is useful for debugging.
+func PrintFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+		if info != nil && !info.IsDir() {
+			s := path
+			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+				s = s + "\tLANG: " + lang.Lang()
+			}
+			if fp, ok := info.(hugofs.FilePather); ok {
+				s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir()
+			}
+			fmt.Fprintln(w, "    ", s)
+		}
+		return nil
+	})
+}

helpers/general_test.go

@@ -1,4 +1,4 @@
-// Copyright 2015 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -220,59 +220,6 @@ func TestFindAvailablePort(t *testing.T) {
 	assert.True(t, addr.Port > 0)
 }
 
-func TestToLowerMap(t *testing.T) {
-
-	tests := []struct {
-		input    map[string]interface{}
-		expected map[string]interface{}
-	}{
-		{
-			map[string]interface{}{
-				"abC": 32,
-			},
-			map[string]interface{}{
-				"abc": 32,
-			},
-		},
-		{
-			map[string]interface{}{
-				"abC": 32,
-				"deF": map[interface{}]interface{}{
-					23: "A value",
-					24: map[string]interface{}{
-						"AbCDe": "A value",
-						"eFgHi": "Another value",
-					},
-				},
-				"gHi": map[string]interface{}{
-					"J": 25,
-				},
-			},
-			map[string]interface{}{
-				"abc": 32,
-				"def": map[string]interface{}{
-					"23": "A value",
-					"24": map[string]interface{}{
-						"abcde": "A value",
-						"efghi": "Another value",
-					},
-				},
-				"ghi": map[string]interface{}{
-					"j": 25,
-				},
-			},
-		},
-	}
-
-	for i, test := range tests {
-		// ToLowerMap modifies input.
-		ToLowerMap(test.input)
-		if !reflect.DeepEqual(test.expected, test.input) {
-			t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input)
-		}
-	}
-}
-
 func TestFastMD5FromFile(t *testing.T) {
 	fs := afero.NewMemMapFs()
 

helpers/path.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 	"unicode"
 
@@ -31,9 +32,6 @@ import (
 var (
 	// ErrThemeUndefined is returned when a theme has not be defined by the user.
 	ErrThemeUndefined = errors.New("no theme set")
-
-	// ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters.
-	ErrPathTooShort = errors.New("file path is too short")
 )
 
 // filepathPathBridge is a bridge for common functionality in filepath vs path
@@ -86,7 +84,7 @@ func (p *PathSpec) MakePath(s string) string {
 
 // MakePathSanitized creates a Unicode-sanitized string, with the spaces replaced
 func (p *PathSpec) MakePathSanitized(s string) string {
-	if p.disablePathToLower {
+	if p.DisablePathToLower {
 		return p.MakePath(s)
 	}
 	return strings.ToLower(p.MakePath(s))
@@ -129,7 +127,7 @@ func (p *PathSpec) UnicodeSanitize(s string) string {
 
 	var result string
 
-	if p.removePathAccents {
+	if p.RemovePathAccents {
 		// remove accents - see https://blog.golang.org/normalization
 		t := transform.Chain(norm.NFD, transform.RemoveFunc(isMn), norm.NFC)
 		result, _, _ = transform.String(t, string(target))
@@ -151,32 +149,19 @@ func ReplaceExtension(path string, newExt string) string {
 	return f + "." + newExt
 }
 
-// AbsPathify creates an absolute path if given a relative path. If already
+// GetFirstThemeDir gets the root directory of the first theme, if there is one.
-// absolute, the path is just cleaned.
-func (p *PathSpec) AbsPathify(inPath string) string {
-	return AbsPathify(p.workingDir, inPath)
-}
-
-// AbsPathify creates an absolute path if given a working dir and arelative path.
-// If already absolute, the path is just cleaned.
-func AbsPathify(workingDir, inPath string) string {
-	if filepath.IsAbs(inPath) {
-		return filepath.Clean(inPath)
-	}
-	return filepath.Join(workingDir, inPath)
-}
-
-// GetLayoutDirPath returns the absolute path to the layout file dir
-// for the current Hugo project.
-func (p *PathSpec) GetLayoutDirPath() string {
-	return p.AbsPathify(p.layoutDir)
-}
-
-// GetThemeDir gets the root directory of the current theme, if there is one.
 // If there is no theme, returns the empty string.
-func (p *PathSpec) GetThemeDir() string {
+func (p *PathSpec) GetFirstThemeDir() string {
 	if p.ThemeSet() {
-		return p.AbsPathify(filepath.Join(p.themesDir, p.theme))
+		return p.AbsPathify(filepath.Join(p.ThemesDir, p.Themes()[0]))
+	}
+	return ""
+}
+
+// GetThemesDir gets the absolute root theme dir path.
+func (p *PathSpec) GetThemesDir() string {
+	if p.ThemeSet() {
+		return p.AbsPathify(p.ThemesDir)
 	}
 	return ""
 }
@@ -185,50 +170,11 @@ func (p *PathSpec) GetThemeDir() string {
 // If there is no theme, returns the empty string.
 func (p *PathSpec) GetRelativeThemeDir() string {
 	if p.ThemeSet() {
-		return strings.TrimPrefix(filepath.Join(p.themesDir, p.theme), FilePathSeparator)
+		return strings.TrimPrefix(filepath.Join(p.ThemesDir, p.Themes()[0]), FilePathSeparator)
 	}
 	return ""
 }
 
-// GetThemeStaticDirPath returns the theme's static dir path if theme is set.
-// If theme is set and the static dir doesn't exist, an error is returned.
-func (p *PathSpec) GetThemeStaticDirPath() (string, error) {
-	return p.getThemeDirPath("static")
-}
-
-// GetThemeDataDirPath returns the theme's data dir path if theme is set.
-// If theme is set and the data dir doesn't exist, an error is returned.
-func (p *PathSpec) GetThemeDataDirPath() (string, error) {
-	return p.getThemeDirPath("data")
-}
-
-// GetThemeI18nDirPath returns the theme's i18n dir path if theme is set.
-// If theme is set and the i18n dir doesn't exist, an error is returned.
-func (p *PathSpec) GetThemeI18nDirPath() (string, error) {
-	return p.getThemeDirPath("i18n")
-}
-
-func (p *PathSpec) getThemeDirPath(path string) (string, error) {
-	if !p.ThemeSet() {
-		return "", ErrThemeUndefined
-	}
-
-	themeDir := filepath.Join(p.GetThemeDir(), path)
-	if _, err := p.Fs.Source.Stat(themeDir); os.IsNotExist(err) {
-		return "", fmt.Errorf("Unable to find %s directory for theme %s in %s", path, p.theme, themeDir)
-	}
-
-	return themeDir, nil
-}
-
-// GetThemesDirPath gets the static files directory of the current theme, if there is one.
-// Ignores underlying errors.
-// TODO(bep) Candidate for deprecation?
-func (p *PathSpec) GetThemesDirPath() string {
-	dir, _ := p.getThemeDirPath("static")
-	return dir
-}
-
 func makePathRelative(inPath string, possibleDirectories ...string) (string, error) {
 
 	for _, currentPath := range possibleDirectories {
@@ -445,8 +391,8 @@ func FindCWD() (string, error) {
 func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 
 	// Sanity check
-	if len(root) < 4 {
+	if root != "" && len(root) < 4 {
-		return ErrPathTooShort
+		return errors.New("Path is too short")
 	}
 
 	// Handle the root first
@@ -464,7 +410,10 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 		return err
 	}
 
-	rootContent, err := afero.ReadDir(fs, root)
+	// Some of Hugo's filesystems represents an ordered root folder, i.e. project first, then theme folders.
+	// Make sure that order is preserved. afero.Walk will sort the directories down in the file tree,
+	// but we don't care about that.
+	rootContent, err := readDir(fs, root, false)
 
 	if err != nil {
 		return walker(root, nil, err)
@@ -480,6 +429,22 @@ func SymbolicWalk(fs afero.Fs, root string, walker filepath.WalkFunc) error {
 
 }
 
+func readDir(fs afero.Fs, dirname string, doSort bool) ([]os.FileInfo, error) {
+	f, err := fs.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	list, err := f.Readdir(-1)
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+	if doSort {
+		sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() })
+	}
+	return list, nil
+}
+
 func getRealFileInfo(fs afero.Fs, path string) (os.FileInfo, string, error) {
 	fileInfo, err := LstatIfPossible(fs, path)
 	realPath := path

helpers/path_test.go

@@ -25,6 +25,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/stretchr/testify/assert"
@@ -56,11 +58,10 @@ func TestMakePath(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		v := viper.New()
+		v := newTestCfg()
-		v.Set("contentDir", "content")
 		v.Set("removePathAccents", test.removeAccents)
 
-		l := NewDefaultLanguage(v)
+		l := langs.NewDefaultLanguage(v)
 		p, err := NewPathSpec(hugofs.NewMem(v), l)
 		require.NoError(t, err)
 
@@ -74,8 +75,12 @@ func TestMakePath(t *testing.T) {
 func TestMakePathSanitized(t *testing.T) {
 	v := viper.New()
 	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
 
-	l := NewDefaultLanguage(v)
+	l := langs.NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
 	tests := []struct {
@@ -99,12 +104,11 @@ func TestMakePathSanitized(t *testing.T) {
 }
 
 func TestMakePathSanitizedDisablePathToLower(t *testing.T) {
-	v := viper.New()
+	v := newTestCfg()
 
 	v.Set("disablePathToLower", true)
-	v.Set("contentDir", "content")
 
-	l := NewDefaultLanguage(v)
+	l := langs.NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
 	tests := []struct {

helpers/pathspec.go

@@ -14,354 +14,71 @@
 package helpers
 
 import (
-	"fmt"
 	"strings"
 
-	"github.com/spf13/afero"
-
-	"github.com/gohugoio/hugo/common/types"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/cast"
+	"github.com/gohugoio/hugo/hugolib/filesystems"
+	"github.com/gohugoio/hugo/hugolib/paths"
 )
 
 // PathSpec holds methods that decides how paths in URLs and files in Hugo should look like.
 type PathSpec struct {
-	BaseURL
+	*paths.Paths
-
+	*filesystems.BaseFs
-	// If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath.
-	// This will not be set if canonifyURLs is enabled.
-	BasePath string
-
-	disablePathToLower bool
-	removePathAccents  bool
-	uglyURLs           bool
-	canonifyURLs       bool
-
-	Language  *Language
-	Languages Languages
-
-	// pagination path handling
-	paginatePath string
-
-	theme string
-
-	// Directories
-	contentDir     string
-	themesDir      string
-	layoutDir      string
-	workingDir     string
-	staticDirs     []string
-	absContentDirs []types.KeyValueStr
-
-	PublishDir string
-
-	// The PathSpec looks up its config settings in both the current language
-	// and then in the global Viper config.
-	// Some settings, the settings listed below, does not make sense to be set
-	// on per-language-basis. We have no good way of protecting against this
-	// other than a "white-list". See language.go.
-	defaultContentLanguageInSubdir bool
-	defaultContentLanguage         string
-	multilingual                   bool
 
 	ProcessingStats *ProcessingStats
 
 	// The file systems to use
 	Fs *hugofs.Fs
 
-	// The fine grained filesystems in play (resources, content etc.).
-	BaseFs *hugofs.BaseFs
-
 	// The config provider to use
 	Cfg config.Provider
 }
 
-func (p PathSpec) String() string {
+// NewPathSpec creats a new PathSpec from the given filesystems and language.
-	return fmt.Sprintf("PathSpec, language %q, prefix %q, multilingual: %T", p.Language.Lang, p.getLanguagePrefix(), p.multilingual)
+func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
+	return NewPathSpecWithBaseBaseFsProvided(fs, cfg, nil)
 }
 
-// NewPathSpec creats a new PathSpec from the given filesystems and Language.
+// NewPathSpecWithBaseBaseFsProvided creats a new PathSpec from the given filesystems and language.
-func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
+// If an existing BaseFs is provided, parts of that is reused.
+func NewPathSpecWithBaseBaseFsProvided(fs *hugofs.Fs, cfg config.Provider, baseBaseFs *filesystems.BaseFs) (*PathSpec, error) {
 
-	baseURLstr := cfg.GetString("baseURL")
+	p, err := paths.New(fs, cfg)
-	baseURL, err := newBaseURLFromString(baseURLstr)
-
-	if err != nil {
-		return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err)
-	}
-
-	var staticDirs []string
-
-	for i := -1; i <= 10; i++ {
-		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
-	}
-
-	var (
-		lang      string
-		language  *Language
-		languages Languages
-	)
-
-	if l, ok := cfg.(*Language); ok {
-		language = l
-		lang = l.Lang
-
-	}
-
-	if l, ok := cfg.Get("languagesSorted").(Languages); ok {
-		languages = l
-	}
-
-	defaultContentLanguage := cfg.GetString("defaultContentLanguage")
-
-	// We will eventually pull out this badly placed path logic.
-	contentDir := cfg.GetString("contentDir")
-	workingDir := cfg.GetString("workingDir")
-	resourceDir := cfg.GetString("resourceDir")
-	publishDir := cfg.GetString("publishDir")
-
-	if len(languages) == 0 {
-		// We have some old tests that does not test the entire chain, hence
-		// they have no languages. So create one so we get the proper filesystem.
-		languages = Languages{&Language{Lang: "en", ContentDir: contentDir}}
-	}
-
-	absPuslishDir := AbsPathify(workingDir, publishDir)
-	if !strings.HasSuffix(absPuslishDir, FilePathSeparator) {
-		absPuslishDir += FilePathSeparator
-	}
-	// If root, remove the second '/'
-	if absPuslishDir == "//" {
-		absPuslishDir = FilePathSeparator
-	}
-	absResourcesDir := AbsPathify(workingDir, resourceDir)
-	if !strings.HasSuffix(absResourcesDir, FilePathSeparator) {
-		absResourcesDir += FilePathSeparator
-	}
-	if absResourcesDir == "//" {
-		absResourcesDir = FilePathSeparator
-	}
-
-	contentFs, absContentDirs, err := createContentFs(fs.Source, workingDir, defaultContentLanguage, languages)
 	if err != nil {
 		return nil, err
 	}
 
-	// Make sure we don't have any overlapping content dirs. That will never work.
+	var options []func(*filesystems.BaseFs) error
-	for i, d1 := range absContentDirs {
+	if baseBaseFs != nil {
-		for j, d2 := range absContentDirs {
+		options = []func(*filesystems.BaseFs) error{
-			if i == j {
+			filesystems.WithBaseFs(baseBaseFs),
-				continue
-			}
-			if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
-				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
-			}
 		}
 	}
-
+	bfs, err := filesystems.NewBase(p, options...)
-	resourcesFs := afero.NewBasePathFs(fs.Source, absResourcesDir)
+	if err != nil {
-	publishFs := afero.NewBasePathFs(fs.Destination, absPuslishDir)
+		return nil, err
-
-	baseFs := &hugofs.BaseFs{
-		ContentFs:   contentFs,
-		ResourcesFs: resourcesFs,
-		PublishFs:   publishFs,
 	}
 
 	ps := &PathSpec{
-		Fs:                             fs,
+		Paths:           p,
-		BaseFs:                         baseFs,
+		BaseFs:          bfs,
-		Cfg:                            cfg,
+		Fs:              fs,
-		disablePathToLower:             cfg.GetBool("disablePathToLower"),
+		Cfg:             cfg,
-		removePathAccents:              cfg.GetBool("removePathAccents"),
+		ProcessingStats: NewProcessingStats(p.Lang()),
-		uglyURLs:                       cfg.GetBool("uglyURLs"),
-		canonifyURLs:                   cfg.GetBool("canonifyURLs"),
-		multilingual:                   cfg.GetBool("multilingual"),
-		Language:                       language,
-		Languages:                      languages,
-		defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),
-		defaultContentLanguage:         defaultContentLanguage,
-		paginatePath:                   cfg.GetString("paginatePath"),
-		BaseURL:                        baseURL,
-		contentDir:                     contentDir,
-		themesDir:                      cfg.GetString("themesDir"),
-		layoutDir:                      cfg.GetString("layoutDir"),
-		workingDir:                     workingDir,
-		staticDirs:                     staticDirs,
-		absContentDirs:                 absContentDirs,
-		theme:                          cfg.GetString("theme"),
-		ProcessingStats:                NewProcessingStats(lang),
 	}
 
-	if !ps.canonifyURLs {
+	if !ps.CanonifyURLs {
-		basePath := ps.BaseURL.url.Path
+		basePath := ps.BaseURL.Path()
 		if basePath != "" && basePath != "/" {
 			ps.BasePath = basePath
 		}
 	}
 
-	// TODO(bep) remove this, eventually
-	ps.PublishDir = absPuslishDir
-
 	return ps, nil
 }
 
-func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
-
-	if id >= 0 {
-		key = fmt.Sprintf("%s%d", key, id)
-	}
-
-	var out []string
-
-	sd := cfg.Get(key)
-
-	if sds, ok := sd.(string); ok {
-		out = []string{sds}
-	} else if sd != nil {
-		out = cast.ToStringSlice(sd)
-	}
-
-	return out
-}
-
-func createContentFs(fs afero.Fs,
-	workingDir,
-	defaultContentLanguage string,
-	languages Languages) (afero.Fs, []types.KeyValueStr, error) {
-
-	var contentLanguages Languages
-	var contentDirSeen = make(map[string]bool)
-	languageSet := make(map[string]bool)
-
-	// The default content language needs to be first.
-	for _, language := range languages {
-		if language.Lang == defaultContentLanguage {
-			contentLanguages = append(contentLanguages, language)
-			contentDirSeen[language.ContentDir] = true
-		}
-		languageSet[language.Lang] = true
-	}
-
-	for _, language := range languages {
-		if contentDirSeen[language.ContentDir] {
-			continue
-		}
-		if language.ContentDir == "" {
-			language.ContentDir = defaultContentLanguage
-		}
-		contentDirSeen[language.ContentDir] = true
-		contentLanguages = append(contentLanguages, language)
-
-	}
-
-	var absContentDirs []types.KeyValueStr
-
-	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
-	return fs, absContentDirs, err
-
-}
-
-func createContentOverlayFs(source afero.Fs,
-	workingDir string,
-	languages Languages,
-	languageSet map[string]bool,
-	absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
-	if len(languages) == 0 {
-		return source, nil
-	}
-
-	language := languages[0]
-
-	contentDir := language.ContentDir
-	if contentDir == "" {
-		panic("missing contentDir")
-	}
-
-	absContentDir := AbsPathify(workingDir, language.ContentDir)
-	if !strings.HasSuffix(absContentDir, FilePathSeparator) {
-		absContentDir += FilePathSeparator
-	}
-
-	// If root, remove the second '/'
-	if absContentDir == "//" {
-		absContentDir = FilePathSeparator
-	}
-
-	if len(absContentDir) < 6 {
-		return nil, fmt.Errorf("invalid content dir %q: %s", absContentDir, ErrPathTooShort)
-	}
-
-	*absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
-
-	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
-	if len(languages) == 1 {
-		return overlay, nil
-	}
-
-	base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
-	if err != nil {
-		return nil, err
-	}
-
-	return hugofs.NewLanguageCompositeFs(base, overlay), nil
-
-}
-
-// RelContentDir tries to create a path relative to the content root from
-// the given filename. The return value is the path and language code.
-func (p *PathSpec) RelContentDir(filename string) (string, string) {
-	for _, dir := range p.absContentDirs {
-		if strings.HasPrefix(filename, dir.Value) {
-			rel := strings.TrimPrefix(filename, dir.Value)
-			return strings.TrimPrefix(rel, FilePathSeparator), dir.Key
-		}
-	}
-	// Either not a content dir or already relative.
-	return filename, ""
-}
-
-// ContentDirs returns all the content dirs (absolute paths).
-func (p *PathSpec) ContentDirs() []types.KeyValueStr {
-	return p.absContentDirs
-}
-
-// PaginatePath returns the configured root path used for paginator pages.
-func (p *PathSpec) PaginatePath() string {
-	return p.paginatePath
-}
-
-// ContentDir returns the configured workingDir.
-func (p *PathSpec) ContentDir() string {
-	return p.contentDir
-}
-
-// WorkingDir returns the configured workingDir.
-func (p *PathSpec) WorkingDir() string {
-	return p.workingDir
-}
-
-// StaticDirs returns the relative static dirs for the current configuration.
-func (p *PathSpec) StaticDirs() []string {
-	return p.staticDirs
-}
-
-// LayoutDir returns the relative layout dir in the current configuration.
-func (p *PathSpec) LayoutDir() string {
-	return p.layoutDir
-}
-
-// Theme returns the theme name if set.
-func (p *PathSpec) Theme() string {
-	return p.theme
-}
-
-// Theme returns the theme relative theme dir.
-func (p *PathSpec) ThemesDir() string {
-	return p.themesDir
-}
-
 // PermalinkForBaseURL creates a permalink from the given link and baseURL.
 func (p *PathSpec) PermalinkForBaseURL(link, baseURL string) string {
 	link = strings.TrimPrefix(link, "/")

helpers/pathspec_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,20 +18,16 @@ import (
 
 	"github.com/gohugoio/hugo/hugofs"
 
-	"github.com/spf13/viper"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/stretchr/testify/require"
 )
 
 func TestNewPathSpecFromConfig(t *testing.T) {
-	v := viper.New()
+	v := newTestCfg()
-	v.Set("contentDir", "content")
+	l := langs.NewLanguage("no", v)
-	l := NewLanguage("no", v)
 	v.Set("disablePathToLower", true)
 	v.Set("removePathAccents", true)
 	v.Set("uglyURLs", true)
-	v.Set("multilingual", true)
-	v.Set("defaultContentLanguageInSubdir", true)
-	v.Set("defaultContentLanguage", "no")
 	v.Set("canonifyURLs", true)
 	v.Set("paginatePath", "side")
 	v.Set("baseURL", "http://base.com")
@@ -44,19 +40,15 @@ func TestNewPathSpecFromConfig(t *testing.T) {
 	p, err := NewPathSpec(hugofs.NewMem(v), l)
 
 	require.NoError(t, err)
-	require.True(t, p.canonifyURLs)
+	require.True(t, p.CanonifyURLs)
-	require.True(t, p.defaultContentLanguageInSubdir)
+	require.True(t, p.DisablePathToLower)
-	require.True(t, p.disablePathToLower)
+	require.True(t, p.RemovePathAccents)
-	require.True(t, p.multilingual)
+	require.True(t, p.UglyURLs)
-	require.True(t, p.removePathAccents)
-	require.True(t, p.uglyURLs)
-	require.Equal(t, "no", p.defaultContentLanguage)
 	require.Equal(t, "no", p.Language.Lang)
-	require.Equal(t, "side", p.paginatePath)
+	require.Equal(t, "side", p.PaginatePath)
 
 	require.Equal(t, "http://base.com", p.BaseURL.String())
-	require.Equal(t, "thethemes", p.themesDir)
+	require.Equal(t, "thethemes", p.ThemesDir)
-	require.Equal(t, "thelayouts", p.layoutDir)
+	require.Equal(t, "thework", p.WorkingDir)
-	require.Equal(t, "thework", p.workingDir)
+	require.Equal(t, []string{"thetheme"}, p.Themes())
-	require.Equal(t, "thetheme", p.theme)
 }

helpers/testhelpers_test.go

@@ -4,10 +4,11 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/langs"
 )
 
 func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec {
-	l := NewDefaultLanguage(v)
+	l := langs.NewDefaultLanguage(v)
 	ps, _ := NewPathSpec(fs, l)
 	return ps
 }
@@ -15,7 +16,7 @@ func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *PathSpec {
 func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec {
 	v := viper.New()
 	fs := hugofs.NewMem(v)
-	cfg := newTestCfg(fs)
+	cfg := newTestCfgFor(fs)
 
 	for i := 0; i < len(configKeyValues); i += 2 {
 		cfg.Set(configKeyValues[i].(string), configKeyValues[i+1])
@@ -23,16 +24,24 @@ func newTestDefaultPathSpec(configKeyValues ...interface{}) *PathSpec {
 	return newTestPathSpec(fs, cfg)
 }
 
-func newTestCfg(fs *hugofs.Fs) *viper.Viper {
+func newTestCfgFor(fs *hugofs.Fs) *viper.Viper {
-	v := viper.New()
+	v := newTestCfg()
-	v.Set("contentDir", "content")
-
 	v.SetFs(fs.Source)
 
 	return v
 
 }
 
+func newTestCfg() *viper.Viper {
+	v := viper.New()
+	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
+	return v
+}
+
 func newTestContentSpec() *ContentSpec {
 	v := viper.New()
 	spec, err := NewContentSpec(v)

helpers/url.go

@@ -177,7 +177,7 @@ func (p *PathSpec) AbsURL(in string, addLanguage bool) string {
 	}
 
 	if addLanguage {
-		prefix := p.getLanguagePrefix()
+		prefix := p.GetLanguagePrefix()
 		if prefix != "" {
 			hasPrefix := false
 			// avoid adding language prefix if already present
@@ -200,38 +200,6 @@ func (p *PathSpec) AbsURL(in string, addLanguage bool) string {
 	return MakePermalink(baseURL, in).String()
 }
 
-func (p *PathSpec) getLanguagePrefix() string {
-	if !p.multilingual {
-		return ""
-	}
-
-	defaultLang := p.defaultContentLanguage
-	defaultInSubDir := p.defaultContentLanguageInSubdir
-
-	currentLang := p.Language.Lang
-	if currentLang == "" || (currentLang == defaultLang && !defaultInSubDir) {
-		return ""
-	}
-	return currentLang
-}
-
-// GetLangSubDir returns the given language's subdir if needed.
-func (p *PathSpec) GetLangSubDir(lang string) string {
-	if !p.multilingual {
-		return ""
-	}
-
-	if p.Languages.IsMultihost() {
-		return ""
-	}
-
-	if lang == "" || (lang == p.defaultContentLanguage && !p.defaultContentLanguageInSubdir) {
-		return ""
-	}
-
-	return lang
-}
-
 // IsAbsURL determines whether the given path points to an absolute URL.
 func IsAbsURL(path string) bool {
 	url, err := url.Parse(path)
@@ -246,7 +214,7 @@ func IsAbsURL(path string) bool {
 // Note: The result URL will not include the context root if canonifyURLs is enabled.
 func (p *PathSpec) RelURL(in string, addLanguage bool) string {
 	baseURL := p.BaseURL.String()
-	canonifyURLs := p.canonifyURLs
+	canonifyURLs := p.CanonifyURLs
 	if (!strings.HasPrefix(in, baseURL) && strings.HasPrefix(in, "http")) || strings.HasPrefix(in, "//") {
 		return in
 	}
@@ -258,7 +226,7 @@ func (p *PathSpec) RelURL(in string, addLanguage bool) string {
 	}
 
 	if addLanguage {
-		prefix := p.getLanguagePrefix()
+		prefix := p.GetLanguagePrefix()
 		if prefix != "" {
 			hasPrefix := false
 			// avoid adding language prefix if already present
@@ -339,7 +307,7 @@ func (p *PathSpec) URLizeAndPrep(in string) string {
 
 // URLPrep applies misc sanitation to the given URL.
 func (p *PathSpec) URLPrep(in string) string {
-	if p.uglyURLs {
+	if p.UglyURLs {
 		return Uglify(SanitizeURL(in))
 	}
 	pretty := PrettifyURL(SanitizeURL(in))

helpers/url_test.go

@@ -19,16 +19,15 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/viper"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestURLize(t *testing.T) {
 
-	v := viper.New()
+	v := newTestCfg()
-	v.Set("contentDir", "content")
+	l := langs.NewDefaultLanguage(v)
-	l := NewDefaultLanguage(v)
 	p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
 	tests := []struct {
@@ -64,7 +63,7 @@ func TestAbsURL(t *testing.T) {
 }
 
 func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, lang string) {
-	v := viper.New()
+	v := newTestCfg()
 	v.Set("multilingual", multilingual)
 	v.Set("defaultContentLanguage", "en")
 	v.Set("defaultContentLanguageInSubdir", defaultInSubDir)
@@ -90,7 +89,7 @@ func doTestAbsURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool,
 	for _, test := range tests {
 		v.Set("baseURL", test.baseURL)
 		v.Set("contentDir", "content")
-		l := NewLanguage(lang, v)
+		l := langs.NewLanguage(lang, v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
 		output := p.AbsURL(test.input, addLanguage)
@@ -140,7 +139,7 @@ func TestRelURL(t *testing.T) {
 }
 
 func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool, lang string) {
-	v := viper.New()
+	v := newTestCfg()
 	v.Set("multilingual", multilingual)
 	v.Set("defaultContentLanguage", "en")
 	v.Set("defaultContentLanguageInSubdir", defaultInSubDir)
@@ -168,8 +167,7 @@ func doTestRelURL(t *testing.T, defaultInSubDir, addLanguage, multilingual bool,
 	for i, test := range tests {
 		v.Set("baseURL", test.baseURL)
 		v.Set("canonifyURLs", test.canonify)
-		v.Set("contentDir", "content")
+		l := langs.NewLanguage(lang, v)
-		l := NewLanguage(lang, v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
 		output := p.RelURL(test.input, addLanguage)
@@ -255,10 +253,9 @@ func TestURLPrep(t *testing.T) {
 	}
 
 	for i, d := range data {
-		v := viper.New()
+		v := newTestCfg()
 		v.Set("uglyURLs", d.ugly)
-		v.Set("contentDir", "content")
+		l := langs.NewDefaultLanguage(v)
-		l := NewDefaultLanguage(v)
 		p, _ := NewPathSpec(hugofs.NewMem(v), l)
 
 		output := p.URLPrep(d.input)

hugofs/base_fs.go

@@ -1,35 +0,0 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package hugofs
-
-import (
-	"github.com/spf13/afero"
-)
-
-// BaseFs contains the core base filesystems used by Hugo. The name "base" is used
-// to underline that even if they can be composites, they all have a base path set to a specific
-// resource folder, e.g "/my-project/content". So, no absolute filenames needed.
-type BaseFs struct {
-	// The filesystem used to capture content. This can be a composite and
-	// language aware file system.
-	ContentFs afero.Fs
-
-	// The filesystem used to store resources (processed images etc.).
-	// This usually maps to /my-project/resources.
-	ResourcesFs afero.Fs
-
-	// The filesystem used to publish the rendered site.
-	// This usually maps to /my-project/public.
-	PublishFs afero.Fs
-}

hugofs/noop_fs.go

@@ -0,0 +1,79 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"errors"
+	"os"
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+var (
+	noOpErr          = errors.New("this is a filesystem that does nothing and this operation is not supported")
+	_       afero.Fs = (*noOpFs)(nil)
+	NoOpFs           = &noOpFs{}
+)
+
+type noOpFs struct {
+}
+
+func (fs noOpFs) Create(name string) (afero.File, error) {
+	return nil, noOpErr
+}
+
+func (fs noOpFs) Mkdir(name string, perm os.FileMode) error {
+	return noOpErr
+}
+
+func (fs noOpFs) MkdirAll(path string, perm os.FileMode) error {
+	return noOpErr
+}
+
+func (fs noOpFs) Open(name string) (afero.File, error) {
+	return nil, os.ErrNotExist
+}
+
+func (fs noOpFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	return nil, os.ErrNotExist
+}
+
+func (fs noOpFs) Remove(name string) error {
+	return noOpErr
+}
+
+func (fs noOpFs) RemoveAll(path string) error {
+	return noOpErr
+}
+
+func (fs noOpFs) Rename(oldname string, newname string) error {
+	return noOpErr
+}
+
+func (fs noOpFs) Stat(name string) (os.FileInfo, error) {
+	return nil, os.ErrNotExist
+}
+
+func (fs noOpFs) Name() string {
+	return "noOpFs"
+}
+
+func (fs noOpFs) Chmod(name string, mode os.FileMode) error {
+	return noOpErr
+}
+
+func (fs noOpFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return noOpErr
+}

hugofs/rootmapping_fs.go

@@ -0,0 +1,180 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	radix "github.com/hashicorp/go-immutable-radix"
+	"github.com/spf13/afero"
+)
+
+var filepathSeparator = string(filepath.Separator)
+
+// A RootMappingFs maps several roots into one. Note that the root of this filesystem
+// is directories only, and they will be returned in Readdir and Readdirnames
+// in the order given.
+type RootMappingFs struct {
+	afero.Fs
+	rootMapToReal *radix.Node
+	virtualRoots  []string
+}
+
+type rootMappingFile struct {
+	afero.File
+	fs   *RootMappingFs
+	name string
+}
+
+type rootMappingFileInfo struct {
+	name string
+}
+
+func (fi *rootMappingFileInfo) Name() string {
+	return fi.name
+}
+
+func (fi *rootMappingFileInfo) Size() int64 {
+	panic("not implemented")
+}
+
+func (fi *rootMappingFileInfo) Mode() os.FileMode {
+	return os.ModeDir
+}
+
+func (fi *rootMappingFileInfo) ModTime() time.Time {
+	panic("not implemented")
+}
+
+func (fi *rootMappingFileInfo) IsDir() bool {
+	return true
+}
+
+func (fi *rootMappingFileInfo) Sys() interface{} {
+	return nil
+}
+
+func newRootMappingDirFileInfo(name string) *rootMappingFileInfo {
+	return &rootMappingFileInfo{name: name}
+}
+
+// NewRootMappingFs creates a new RootMappingFs on top of the provided with
+// a list of from, to string pairs of root mappings.
+// Note that 'from' represents a virtual root that maps to the actual filename in 'to'.
+func NewRootMappingFs(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {
+	rootMapToReal := radix.New().Txn()
+	var virtualRoots []string
+
+	for i := 0; i < len(fromTo); i += 2 {
+		vr := filepath.Clean(fromTo[i])
+		rr := filepath.Clean(fromTo[i+1])
+
+		// We need to preserve the original order for Readdir
+		virtualRoots = append(virtualRoots, vr)
+
+		rootMapToReal.Insert([]byte(vr), rr)
+	}
+
+	return &RootMappingFs{Fs: fs,
+		virtualRoots:  virtualRoots,
+		rootMapToReal: rootMapToReal.Commit().Root()}, nil
+}
+
+func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {
+	if fs.isRoot(name) {
+		return newRootMappingDirFileInfo(name), nil
+	}
+	realName := fs.realName(name)
+	return fs.Fs.Stat(realName)
+}
+
+func (fs *RootMappingFs) isRoot(name string) bool {
+	return name == "" || name == filepathSeparator
+
+}
+
+func (fs *RootMappingFs) Open(name string) (afero.File, error) {
+	if fs.isRoot(name) {
+		return &rootMappingFile{name: name, fs: fs}, nil
+	}
+	realName := fs.realName(name)
+	f, err := fs.Fs.Open(realName)
+	if err != nil {
+		return nil, err
+	}
+	return &rootMappingFile{File: f, name: name, fs: fs}, nil
+}
+
+func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	if fs.isRoot(name) {
+		return newRootMappingDirFileInfo(name), false, nil
+	}
+	name = fs.realName(name)
+	if ls, ok := fs.Fs.(afero.Lstater); ok {
+		return ls.LstatIfPossible(name)
+	}
+	fi, err := fs.Stat(name)
+	return fi, false, err
+}
+
+func (fs *RootMappingFs) realName(name string) string {
+	key, val, found := fs.rootMapToReal.LongestPrefix([]byte(filepath.Clean(name)))
+	if !found {
+		return name
+	}
+	keystr := string(key)
+
+	return filepath.Join(val.(string), strings.TrimPrefix(name, keystr))
+}
+
+func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
+	if f.File == nil {
+		dirsn := make([]os.FileInfo, 0)
+		for i := 0; i < len(f.fs.virtualRoots); i++ {
+			if count != -1 && i >= count {
+				break
+			}
+			dirsn = append(dirsn, newRootMappingDirFileInfo(f.fs.virtualRoots[i]))
+		}
+		return dirsn, nil
+	}
+	return f.File.Readdir(count)
+
+}
+
+func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {
+	dirs, err := f.Readdir(count)
+	if err != nil {
+		return nil, err
+	}
+	dirss := make([]string, len(dirs))
+	for i, d := range dirs {
+		dirss[i] = d.Name()
+	}
+	return dirss, nil
+}
+
+func (f *rootMappingFile) Name() string {
+	return f.name
+}
+
+func (f *rootMappingFile) Close() error {
+	if f.File == nil {
+		return nil
+	}
+	return f.File.Close()
+}

hugofs/rootmapping_fs_test.go

@@ -0,0 +1,93 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRootMappingFsRealName(t *testing.T) {
+	assert := require.New(t)
+	fs := afero.NewMemMapFs()
+
+	rfs, err := NewRootMappingFs(fs, "f1", "f1t", "f2", "f2t")
+	assert.NoError(err)
+
+	assert.Equal(filepath.FromSlash("f1t/foo/file.txt"), rfs.realName(filepath.Join("f1", "foo", "file.txt")))
+
+}
+
+func TestRootMappingFsDirnames(t *testing.T) {
+	assert := require.New(t)
+	fs := afero.NewMemMapFs()
+
+	testfile := "myfile.txt"
+	assert.NoError(fs.Mkdir("f1t", 0755))
+	assert.NoError(fs.Mkdir("f2t", 0755))
+	assert.NoError(fs.Mkdir("f3t", 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755))
+
+	rfs, err := NewRootMappingFs(fs, "bf1", "f1t", "cf2", "f2t", "af3", "f3t")
+	assert.NoError(err)
+
+	fif, err := rfs.Stat(filepath.Join("cf2", testfile))
+	assert.NoError(err)
+	assert.Equal("myfile.txt", fif.Name())
+
+	root, err := rfs.Open(filepathSeparator)
+	assert.NoError(err)
+
+	dirnames, err := root.Readdirnames(-1)
+	assert.NoError(err)
+	assert.Equal([]string{"bf1", "cf2", "af3"}, dirnames)
+
+}
+
+func TestRootMappingFsOs(t *testing.T) {
+	assert := require.New(t)
+	fs := afero.NewOsFs()
+
+	d, err := ioutil.TempDir("", "hugo-root-mapping")
+	assert.NoError(err)
+	defer func() {
+		os.RemoveAll(d)
+	}()
+
+	testfile := "myfile.txt"
+	assert.NoError(fs.Mkdir(filepath.Join(d, "f1t"), 0755))
+	assert.NoError(fs.Mkdir(filepath.Join(d, "f2t"), 0755))
+	assert.NoError(fs.Mkdir(filepath.Join(d, "f3t"), 0755))
+	assert.NoError(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755))
+
+	rfs, err := NewRootMappingFs(fs, "bf1", filepath.Join(d, "f1t"), "cf2", filepath.Join(d, "f2t"), "af3", filepath.Join(d, "f3t"))
+	assert.NoError(err)
+
+	fif, err := rfs.Stat(filepath.Join("cf2", testfile))
+	assert.NoError(err)
+	assert.Equal("myfile.txt", fif.Name())
+
+	root, err := rfs.Open(filepathSeparator)
+	assert.NoError(err)
+
+	dirnames, err := root.Readdirnames(-1)
+	assert.NoError(err)
+	assert.Equal([]string{"bf1", "cf2", "af3"}, dirnames)
+
+}

hugolib/alias_test.go

@@ -18,6 +18,8 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/gohugoio/hugo/common/loggers"
+
 	"github.com/stretchr/testify/require"
 )
 
@@ -97,7 +99,7 @@ func TestAliasTemplate(t *testing.T) {
 }
 
 func TestTargetPathHTMLRedirectAlias(t *testing.T) {
-	h := newAliasHandler(nil, newErrorLogger(), false)
+	h := newAliasHandler(nil, loggers.NewErrorLogger(), false)
 
 	errIsNilForThisOS := runtime.GOOS != "windows"
 

hugolib/case_insensitive_test.go

@@ -19,8 +19,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/hugofs"
+
+	"github.com/gohugoio/hugo/deps"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
 )

hugolib/config.go

@@ -16,11 +16,14 @@ package hugolib
 import (
 	"errors"
 	"fmt"
-	"path/filepath"
+
+	"github.com/gohugoio/hugo/hugolib/paths"
 
 	"io"
 	"strings"
 
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/config/privacy"
 	"github.com/gohugoio/hugo/config/services"
@@ -81,6 +84,8 @@ func LoadConfigDefault(fs afero.Fs) (*viper.Viper, error) {
 	return v, err
 }
 
+var ErrNoConfigFile = errors.New("Unable to locate Config file. Perhaps you need to create a new site.\n       Run `hugo help new` for details.\n")
+
 // LoadConfig loads Hugo configuration into a new Viper and then adds
 // a set of defaults.
 func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provider) error) (*viper.Viper, []string, error) {
@@ -100,41 +105,50 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid
 	v.SetConfigFile(configFilenames[0])
 	v.AddConfigPath(d.Path)
 
+	var configFileErr error
+
 	err := v.ReadInConfig()
 	if err != nil {
 		if _, ok := err.(viper.ConfigParseError); ok {
 			return nil, configFiles, err
 		}
-		return nil, configFiles, fmt.Errorf("Unable to locate Config file. Perhaps you need to create a new site.\n       Run `hugo help new` for details. (%s)\n", err)
+		configFileErr = ErrNoConfigFile
 	}
 
-	if cf := v.ConfigFileUsed(); cf != "" {
+	if configFileErr == nil {
-		configFiles = append(configFiles, cf)
-	}
 
-	for _, configFile := range configFilenames[1:] {
+		if cf := v.ConfigFileUsed(); cf != "" {
-		var r io.Reader
+			configFiles = append(configFiles, cf)
-		var err error
-		if r, err = fs.Open(configFile); err != nil {
-			return nil, configFiles, fmt.Errorf("Unable to open Config file.\n (%s)\n", err)
 		}
-		if err = v.MergeConfig(r); err != nil {
+
-			return nil, configFiles, fmt.Errorf("Unable to parse/merge Config file (%s).\n (%s)\n", configFile, err)
+		for _, configFile := range configFilenames[1:] {
+			var r io.Reader
+			var err error
+			if r, err = fs.Open(configFile); err != nil {
+				return nil, configFiles, fmt.Errorf("Unable to open Config file.\n (%s)\n", err)
+			}
+			if err = v.MergeConfig(r); err != nil {
+				return nil, configFiles, fmt.Errorf("Unable to parse/merge Config file (%s).\n (%s)\n", configFile, err)
+			}
+			configFiles = append(configFiles, configFile)
 		}
-		configFiles = append(configFiles, configFile)
+
 	}
 
 	if err := loadDefaultSettingsFor(v); err != nil {
 		return v, configFiles, err
 	}
 
-	themeConfigFile, err := loadThemeConfig(d, v)
+	if configFileErr == nil {
-	if err != nil {
-		return v, configFiles, err
-	}
 
-	if themeConfigFile != "" {
+		themeConfigFiles, err := loadThemeConfig(d, v)
-		configFiles = append(configFiles, themeConfigFile)
+		if err != nil {
+			return v, configFiles, err
+		}
+
+		if len(themeConfigFiles) > 0 {
+			configFiles = append(configFiles, themeConfigFiles...)
+		}
 	}
 
 	// We create languages based on the settings, so we need to make sure that
@@ -149,11 +163,11 @@ func LoadConfig(d ConfigSourceDescriptor, doWithConfig ...func(cfg config.Provid
 		return v, configFiles, err
 	}
 
-	return v, configFiles, nil
+	return v, configFiles, configFileErr
 
 }
 
-func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error {
+func loadLanguageSettings(cfg config.Provider, oldLangs langs.Languages) error {
 
 	defaultLang := cfg.GetString("defaultContentLanguage")
 
@@ -182,14 +196,14 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 	}
 
 	var (
-		langs helpers.Languages
+		languages2 langs.Languages
-		err   error
+		err        error
 	)
 
 	if len(languages) == 0 {
-		langs = append(langs, helpers.NewDefaultLanguage(cfg))
+		languages2 = append(languages2, langs.NewDefaultLanguage(cfg))
 	} else {
-		langs, err = toSortedLanguages(cfg, languages)
+		languages2, err = toSortedLanguages(cfg, languages)
 		if err != nil {
 			return fmt.Errorf("Failed to parse multilingual config: %s", err)
 		}
@@ -201,10 +215,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 		// The validation below isn't complete, but should cover the most
 		// important cases.
 		var invalid bool
-		if langs.IsMultihost() != oldLangs.IsMultihost() {
+		if languages2.IsMultihost() != oldLangs.IsMultihost() {
 			invalid = true
 		} else {
-			if langs.IsMultihost() && len(langs) != len(oldLangs) {
+			if languages2.IsMultihost() && len(languages2) != len(oldLangs) {
 				invalid = true
 			}
 		}
@@ -213,10 +227,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 			return errors.New("language change needing a server restart detected")
 		}
 
-		if langs.IsMultihost() {
+		if languages2.IsMultihost() {
 			// We need to transfer any server baseURL to the new language
 			for i, ol := range oldLangs {
-				nl := langs[i]
+				nl := languages2[i]
 				nl.Set("baseURL", ol.GetString("baseURL"))
 			}
 		}
@@ -225,7 +239,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 	// The defaultContentLanguage is something the user has to decide, but it needs
 	// to match a language in the language definition list.
 	langExists := false
-	for _, lang := range langs {
+	for _, lang := range languages2 {
 		if lang.Lang == defaultLang {
 			langExists = true
 			break
@@ -236,10 +250,10 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 		return fmt.Errorf("site config value %q for defaultContentLanguage does not match any language definition", defaultLang)
 	}
 
-	cfg.Set("languagesSorted", langs)
+	cfg.Set("languagesSorted", languages2)
-	cfg.Set("multilingual", len(langs) > 1)
+	cfg.Set("multilingual", len(languages2) > 1)
 
-	multihost := langs.IsMultihost()
+	multihost := languages2.IsMultihost()
 
 	if multihost {
 		cfg.Set("defaultContentLanguageInSubdir", true)
@@ -250,7 +264,7 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 		// The baseURL may be provided at the language level. If that is true,
 		// then every language must have a baseURL. In this case we always render
 		// to a language sub folder, which is then stripped from all the Permalink URLs etc.
-		for _, l := range langs {
+		for _, l := range languages2 {
 			burl := l.GetLocal("baseURL")
 			if burl == nil {
 				return errors.New("baseURL must be set on all or none of the languages")
@@ -262,49 +276,32 @@ func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error
 	return nil
 }
 
-func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error) {
+func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) ([]string, error) {
+	themesDir := paths.AbsPathify(d.WorkingDir, v1.GetString("themesDir"))
+	themes := config.GetStringSlicePreserveString(v1, "theme")
 
-	theme := v1.GetString("theme")
+	//  CollectThemes(fs afero.Fs, themesDir string, themes []strin
-	if theme == "" {
+	themeConfigs, err := paths.CollectThemes(d.Fs, themesDir, themes)
-		return "", nil
-	}
-
-	themesDir := helpers.AbsPathify(d.WorkingDir, v1.GetString("themesDir"))
-	configDir := filepath.Join(themesDir, theme)
-
-	var (
-		configPath string
-		exists     bool
-		err        error
-	)
-
-	// Viper supports more, but this is the sub-set supported by Hugo.
-	for _, configFormats := range []string{"toml", "yaml", "yml", "json"} {
-		configPath = filepath.Join(configDir, "config."+configFormats)
-		exists, err = helpers.Exists(configPath, d.Fs)
-		if err != nil {
-			return "", err
-		}
-		if exists {
-			break
-		}
-	}
-
-	if !exists {
-		// No theme config set.
-		return "", nil
-	}
-
-	v2 := viper.New()
-	v2.SetFs(d.Fs)
-	v2.AutomaticEnv()
-	v2.SetEnvPrefix("hugo")
-	v2.SetConfigFile(configPath)
-
-	err = v2.ReadInConfig()
 	if err != nil {
-		return "", err
+		return nil, err
 	}
+	v1.Set("allThemes", themeConfigs)
+
+	var configFilenames []string
+	for _, tc := range themeConfigs {
+		if tc.ConfigFilename != "" {
+			configFilenames = append(configFilenames, tc.ConfigFilename)
+			if err := applyThemeConfig(v1, tc); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return configFilenames, nil
+
+}
+
+func applyThemeConfig(v1 *viper.Viper, theme paths.ThemeConfig) error {
 
 	const (
 		paramsKey    = "params"
@@ -312,11 +309,13 @@ func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error)
 		menuKey      = "menu"
 	)
 
+	v2 := theme.Cfg
+
 	for _, key := range []string{paramsKey, "outputformats", "mediatypes"} {
 		mergeStringMapKeepLeft("", key, v1, v2)
 	}
 
-	themeLower := strings.ToLower(theme)
+	themeLower := strings.ToLower(theme.Name)
 	themeParamsNamespace := paramsKey + "." + themeLower
 
 	// Set namespaced params
@@ -371,11 +370,11 @@ func loadThemeConfig(d ConfigSourceDescriptor, v1 *viper.Viper) (string, error)
 		}
 	}
 
-	return v2.ConfigFileUsed(), nil
+	return nil
 
 }
 
-func mergeStringMapKeepLeft(rootKey, key string, v1, v2 *viper.Viper) {
+func mergeStringMapKeepLeft(rootKey, key string, v1, v2 config.Provider) {
 	if !v2.IsSet(key) {
 		return
 	}

hugolib/datafiles_test.go

@@ -19,6 +19,8 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/gohugoio/hugo/common/loggers"
+
 	"github.com/gohugoio/hugo/deps"
 
 	"fmt"
@@ -322,7 +324,7 @@ func doTestDataDirImpl(t *testing.T, dd dataDir, expected interface{}, configKey
 	}
 
 	var (
-		logger  = newErrorLogger()
+		logger  = loggers.NewErrorLogger()
 		depsCfg = deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: logger}
 	)
 

hugolib/filesystems/basefs.go

@@ -0,0 +1,644 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package filesystems provides the fine grained file systems used by Hugo. These
+// are typically virtual filesystems that are composites of project and theme content.
+package filesystems
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/config"
+
+	"github.com/gohugoio/hugo/hugofs"
+
+	"fmt"
+
+	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/hugolib/paths"
+	"github.com/gohugoio/hugo/langs"
+	"github.com/spf13/afero"
+)
+
+// When we create a virtual filesystem with data and i18n bundles for the project and the themes,
+// this is the name of the project's virtual root. It got it's funky name to make sure
+// (or very unlikely) that it collides with a theme name.
+const projectVirtualFolder = "__h__project"
+
+var filePathSeparator = string(filepath.Separator)
+
+// BaseFs contains the core base filesystems used by Hugo. The name "base" is used
+// to underline that even if they can be composites, they all have a base path set to a specific
+// resource folder, e.g "/my-project/content". So, no absolute filenames needed.
+type BaseFs struct {
+	// TODO(bep) make this go away
+	AbsContentDirs []types.KeyValueStr
+
+	// The filesystem used to capture content. This can be a composite and
+	// language aware file system.
+	ContentFs afero.Fs
+
+	// SourceFilesystems contains the different source file systems.
+	*SourceFilesystems
+
+	// The filesystem used to store resources (processed images etc.).
+	// This usually maps to /my-project/resources.
+	ResourcesFs afero.Fs
+
+	// The filesystem used to publish the rendered site.
+	// This usually maps to /my-project/public.
+	PublishFs afero.Fs
+
+	themeFs afero.Fs
+
+	// TODO(bep) improve the "theme interaction"
+	AbsThemeDirs []string
+}
+
+// RelContentDir tries to create a path relative to the content root from
+// the given filename. The return value is the path and language code.
+func (b *BaseFs) RelContentDir(filename string) (string, string) {
+	for _, dir := range b.AbsContentDirs {
+		if strings.HasPrefix(filename, dir.Value) {
+			rel := strings.TrimPrefix(filename, dir.Value)
+			return strings.TrimPrefix(rel, filePathSeparator), dir.Key
+		}
+	}
+	// Either not a content dir or already relative.
+	return filename, ""
+}
+
+// IsContent returns whether the given filename is in the content filesystem.
+func (b *BaseFs) IsContent(filename string) bool {
+	for _, dir := range b.AbsContentDirs {
+		if strings.HasPrefix(filename, dir.Value) {
+			return true
+		}
+	}
+	return false
+}
+
+// SourceFilesystems contains the different source file systems. These can be
+// composite file systems (theme and project etc.), and they have all root
+// set to the source type the provides: data, i18n, static, layouts.
+type SourceFilesystems struct {
+	Data       *SourceFilesystem
+	I18n       *SourceFilesystem
+	Layouts    *SourceFilesystem
+	Archetypes *SourceFilesystem
+
+	// When in multihost we have one static filesystem per language. The sync
+	// static files is currently done outside of the Hugo build (where there is
+	// a concept of a site per language).
+	// When in non-multihost mode there will be one entry in this map with a blank key.
+	Static map[string]*SourceFilesystem
+}
+
+// A SourceFilesystem holds the filesystem for a given source type in Hugo (data,
+// i18n, layouts, static) and additional metadata to be able to use that filesystem
+// in server mode.
+type SourceFilesystem struct {
+	Fs afero.Fs
+
+	Dirnames []string
+
+	// When syncing a source folder to the target (e.g. /public), this may
+	// be set to publish into a subfolder. This is used for static syncing
+	// in multihost mode.
+	PublishFolder string
+}
+
+// IsStatic returns true if the given filename is a member of one of the static
+// filesystems.
+func (s SourceFilesystems) IsStatic(filename string) bool {
+	for _, staticFs := range s.Static {
+		if staticFs.Contains(filename) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsLayout returns true if the given filename is a member of the layouts filesystem.
+func (s SourceFilesystems) IsLayout(filename string) bool {
+	return s.Layouts.Contains(filename)
+}
+
+// IsData returns true if the given filename is a member of the data filesystem.
+func (s SourceFilesystems) IsData(filename string) bool {
+	return s.Data.Contains(filename)
+}
+
+// IsI18n returns true if the given filename is a member of the i18n filesystem.
+func (s SourceFilesystems) IsI18n(filename string) bool {
+	return s.I18n.Contains(filename)
+}
+
+// MakeStaticPathRelative makes an absolute static filename into a relative one.
+// It will return an empty string if the filename is not a member of a static filesystem.
+func (s SourceFilesystems) MakeStaticPathRelative(filename string) string {
+	for _, staticFs := range s.Static {
+		rel := staticFs.MakePathRelative(filename)
+		if rel != "" {
+			return rel
+		}
+	}
+	return ""
+}
+
+// MakePathRelative creates a relative path from the given filename.
+// It will return an empty string if the filename is not a member of this filesystem.
+func (d *SourceFilesystem) MakePathRelative(filename string) string {
+	for _, currentPath := range d.Dirnames {
+		if strings.HasPrefix(filename, currentPath) {
+			return strings.TrimPrefix(filename, currentPath)
+		}
+	}
+	return ""
+}
+
+// Contains returns whether the given filename is a member of the current filesystem.
+func (d *SourceFilesystem) Contains(filename string) bool {
+	for _, dir := range d.Dirnames {
+		if strings.HasPrefix(filename, dir) {
+			return true
+		}
+	}
+	return false
+}
+
+// WithBaseFs allows reuse of some potentially expensive to create parts that remain
+// the same across sites/languages.
+func WithBaseFs(b *BaseFs) func(*BaseFs) error {
+	return func(bb *BaseFs) error {
+		bb.themeFs = b.themeFs
+		bb.AbsThemeDirs = b.AbsThemeDirs
+		return nil
+	}
+}
+
+// NewBase builds the filesystems used by Hugo given the paths and options provided.NewBase
+func NewBase(p *paths.Paths, options ...func(*BaseFs) error) (*BaseFs, error) {
+	fs := p.Fs
+
+	resourcesFs := afero.NewBasePathFs(fs.Source, p.AbsResourcesDir)
+	publishFs := afero.NewBasePathFs(fs.Destination, p.AbsPublishDir)
+
+	contentFs, absContentDirs, err := createContentFs(fs.Source, p.WorkingDir, p.DefaultContentLanguage, p.Languages)
+	if err != nil {
+		return nil, err
+	}
+
+	// Make sure we don't have any overlapping content dirs. That will never work.
+	for i, d1 := range absContentDirs {
+		for j, d2 := range absContentDirs {
+			if i == j {
+				continue
+			}
+			if strings.HasPrefix(d1.Value, d2.Value) || strings.HasPrefix(d2.Value, d1.Value) {
+				return nil, fmt.Errorf("found overlapping content dirs (%q and %q)", d1, d2)
+			}
+		}
+	}
+
+	b := &BaseFs{
+		AbsContentDirs: absContentDirs,
+		ContentFs:      contentFs,
+		ResourcesFs:    resourcesFs,
+		PublishFs:      publishFs,
+	}
+
+	for _, opt := range options {
+		if err := opt(b); err != nil {
+			return nil, err
+		}
+	}
+
+	builder := newSourceFilesystemsBuilder(p, b)
+	sourceFilesystems, err := builder.Build()
+	if err != nil {
+		return nil, err
+	}
+
+	b.SourceFilesystems = sourceFilesystems
+	b.themeFs = builder.themeFs
+	b.AbsThemeDirs = builder.absThemeDirs
+
+	return b, nil
+}
+
+type sourceFilesystemsBuilder struct {
+	p            *paths.Paths
+	result       *SourceFilesystems
+	themeFs      afero.Fs
+	hasTheme     bool
+	absThemeDirs []string
+}
+
+func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder {
+	return &sourceFilesystemsBuilder{p: p, themeFs: b.themeFs, absThemeDirs: b.AbsThemeDirs, result: &SourceFilesystems{}}
+}
+
+func (b *sourceFilesystemsBuilder) Build() (*SourceFilesystems, error) {
+	if b.themeFs == nil && b.p.ThemeSet() {
+		themeFs, absThemeDirs, err := createThemesOverlayFs(b.p)
+		if err != nil {
+			return nil, err
+		}
+		if themeFs == nil {
+			panic("createThemesFs returned nil")
+		}
+		b.themeFs = themeFs
+		b.absThemeDirs = absThemeDirs
+
+	}
+
+	b.hasTheme = len(b.absThemeDirs) > 0
+
+	sfs, err := b.createRootMappingFs("dataDir", "data")
+	if err != nil {
+		return nil, err
+	}
+	b.result.Data = sfs
+
+	sfs, err = b.createRootMappingFs("i18nDir", "i18n")
+	if err != nil {
+		return nil, err
+	}
+	b.result.I18n = sfs
+
+	sfs, err = b.createFs("layoutDir", "layouts")
+	if err != nil {
+		return nil, err
+	}
+	b.result.Layouts = sfs
+
+	sfs, err = b.createFs("archetypeDir", "archetypes")
+	if err != nil {
+		return nil, err
+	}
+	b.result.Archetypes = sfs
+
+	err = b.createStaticFs()
+	if err != nil {
+		return nil, err
+	}
+
+	return b.result, nil
+}
+
+func (b *sourceFilesystemsBuilder) createFs(dirKey, themeFolder string) (*SourceFilesystem, error) {
+	s := &SourceFilesystem{}
+	dir := b.p.Cfg.GetString(dirKey)
+	if dir == "" {
+		return s, fmt.Errorf("config %q not set", dirKey)
+	}
+
+	var fs afero.Fs
+
+	absDir := b.p.AbsPathify(dir)
+	if b.existsInSource(absDir) {
+		fs = afero.NewBasePathFs(b.p.Fs.Source, absDir)
+		s.Dirnames = []string{absDir}
+	}
+
+	if b.hasTheme {
+		themeFolderFs := afero.NewBasePathFs(b.themeFs, themeFolder)
+		if fs == nil {
+			fs = themeFolderFs
+		} else {
+			fs = afero.NewCopyOnWriteFs(themeFolderFs, fs)
+		}
+
+		for _, absThemeDir := range b.absThemeDirs {
+			absThemeFolderDir := filepath.Join(absThemeDir, themeFolder)
+			if b.existsInSource(absThemeFolderDir) {
+				s.Dirnames = append(s.Dirnames, absThemeFolderDir)
+			}
+		}
+	}
+
+	if fs == nil {
+		s.Fs = hugofs.NoOpFs
+	} else {
+		s.Fs = afero.NewReadOnlyFs(fs)
+	}
+
+	return s, nil
+}
+
+// Used for data, i18n -- we cannot use overlay filsesystems for those, but we need
+// to keep a strict order.
+func (b *sourceFilesystemsBuilder) createRootMappingFs(dirKey, themeFolder string) (*SourceFilesystem, error) {
+	s := &SourceFilesystem{}
+
+	projectDir := b.p.Cfg.GetString(dirKey)
+	if projectDir == "" {
+		return nil, fmt.Errorf("config %q not set", dirKey)
+	}
+
+	var fromTo []string
+	to := b.p.AbsPathify(projectDir)
+
+	if b.existsInSource(to) {
+		s.Dirnames = []string{to}
+		fromTo = []string{projectVirtualFolder, to}
+	}
+
+	for _, theme := range b.p.AllThemes {
+		to := b.p.AbsPathify(filepath.Join(b.p.ThemesDir, theme.Name, themeFolder))
+		if b.existsInSource(to) {
+			s.Dirnames = append(s.Dirnames, to)
+			from := theme
+			fromTo = append(fromTo, from.Name, to)
+		}
+	}
+
+	if len(fromTo) == 0 {
+		s.Fs = hugofs.NoOpFs
+		return s, nil
+	}
+
+	fs, err := hugofs.NewRootMappingFs(b.p.Fs.Source, fromTo...)
+	if err != nil {
+		return nil, err
+	}
+
+	s.Fs = afero.NewReadOnlyFs(fs)
+
+	return s, nil
+
+}
+
+func (b *sourceFilesystemsBuilder) existsInSource(abspath string) bool {
+	exists, _ := afero.Exists(b.p.Fs.Source, abspath)
+	return exists
+}
+
+func (b *sourceFilesystemsBuilder) createStaticFs() error {
+	isMultihost := b.p.Cfg.GetBool("multihost")
+	ms := make(map[string]*SourceFilesystem)
+	b.result.Static = ms
+
+	if isMultihost {
+		for _, l := range b.p.Languages {
+			s := &SourceFilesystem{PublishFolder: l.Lang}
+			staticDirs := removeDuplicatesKeepRight(getStaticDirs(l))
+			if len(staticDirs) == 0 {
+				continue
+			}
+
+			for _, dir := range staticDirs {
+				absDir := b.p.AbsPathify(dir)
+				if !b.existsInSource(absDir) {
+					continue
+				}
+
+				s.Dirnames = append(s.Dirnames, absDir)
+			}
+
+			fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames)
+			if err != nil {
+				return err
+			}
+
+			s.Fs = fs
+			ms[l.Lang] = s
+
+		}
+
+		return nil
+	}
+
+	s := &SourceFilesystem{}
+	var staticDirs []string
+
+	for _, l := range b.p.Languages {
+		staticDirs = append(staticDirs, getStaticDirs(l)...)
+	}
+
+	staticDirs = removeDuplicatesKeepRight(staticDirs)
+	if len(staticDirs) == 0 {
+		return nil
+	}
+
+	for _, dir := range staticDirs {
+		absDir := b.p.AbsPathify(dir)
+		if !b.existsInSource(absDir) {
+			continue
+		}
+		s.Dirnames = append(s.Dirnames, absDir)
+	}
+
+	fs, err := createOverlayFs(b.p.Fs.Source, s.Dirnames)
+	if err != nil {
+		return err
+	}
+
+	if b.hasTheme {
+		themeFolder := "static"
+		fs = afero.NewCopyOnWriteFs(afero.NewBasePathFs(b.themeFs, themeFolder), fs)
+		for _, absThemeDir := range b.absThemeDirs {
+			s.Dirnames = append(s.Dirnames, filepath.Join(absThemeDir, themeFolder))
+		}
+	}
+
+	s.Fs = fs
+	ms[""] = s
+
+	return nil
+}
+
+func getStaticDirs(cfg config.Provider) []string {
+	var staticDirs []string
+	for i := -1; i <= 10; i++ {
+		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
+	}
+	return staticDirs
+}
+
+func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
+
+	if id >= 0 {
+		key = fmt.Sprintf("%s%d", key, id)
+	}
+
+	return config.GetStringSlicePreserveString(cfg, key)
+
+}
+
+func createContentFs(fs afero.Fs,
+	workingDir,
+	defaultContentLanguage string,
+	languages langs.Languages) (afero.Fs, []types.KeyValueStr, error) {
+
+	var contentLanguages langs.Languages
+	var contentDirSeen = make(map[string]bool)
+	languageSet := make(map[string]bool)
+
+	// The default content language needs to be first.
+	for _, language := range languages {
+		if language.Lang == defaultContentLanguage {
+			contentLanguages = append(contentLanguages, language)
+			contentDirSeen[language.ContentDir] = true
+		}
+		languageSet[language.Lang] = true
+	}
+
+	for _, language := range languages {
+		if contentDirSeen[language.ContentDir] {
+			continue
+		}
+		if language.ContentDir == "" {
+			language.ContentDir = defaultContentLanguage
+		}
+		contentDirSeen[language.ContentDir] = true
+		contentLanguages = append(contentLanguages, language)
+
+	}
+
+	var absContentDirs []types.KeyValueStr
+
+	fs, err := createContentOverlayFs(fs, workingDir, contentLanguages, languageSet, &absContentDirs)
+	return fs, absContentDirs, err
+
+}
+
+func createContentOverlayFs(source afero.Fs,
+	workingDir string,
+	languages langs.Languages,
+	languageSet map[string]bool,
+	absContentDirs *[]types.KeyValueStr) (afero.Fs, error) {
+	if len(languages) == 0 {
+		return source, nil
+	}
+
+	language := languages[0]
+
+	contentDir := language.ContentDir
+	if contentDir == "" {
+		panic("missing contentDir")
+	}
+
+	absContentDir := paths.AbsPathify(workingDir, language.ContentDir)
+	if !strings.HasSuffix(absContentDir, paths.FilePathSeparator) {
+		absContentDir += paths.FilePathSeparator
+	}
+
+	// If root, remove the second '/'
+	if absContentDir == "//" {
+		absContentDir = paths.FilePathSeparator
+	}
+
+	if len(absContentDir) < 6 {
+		return nil, fmt.Errorf("invalid content dir %q: Path is too short", absContentDir)
+	}
+
+	*absContentDirs = append(*absContentDirs, types.KeyValueStr{Key: language.Lang, Value: absContentDir})
+
+	overlay := hugofs.NewLanguageFs(language.Lang, languageSet, afero.NewBasePathFs(source, absContentDir))
+	if len(languages) == 1 {
+		return overlay, nil
+	}
+
+	base, err := createContentOverlayFs(source, workingDir, languages[1:], languageSet, absContentDirs)
+	if err != nil {
+		return nil, err
+	}
+
+	return hugofs.NewLanguageCompositeFs(base, overlay), nil
+
+}
+
+func createThemesOverlayFs(p *paths.Paths) (afero.Fs, []string, error) {
+
+	themes := p.AllThemes
+
+	if len(themes) == 0 {
+		panic("AllThemes not set")
+	}
+
+	themesDir := p.AbsPathify(p.ThemesDir)
+	if themesDir == "" {
+		return nil, nil, errors.New("no themes dir set")
+	}
+
+	absPaths := make([]string, len(themes))
+
+	// The themes are ordered from left to right. We need to revert it to get the
+	// overlay logic below working as expected.
+	for i := 0; i < len(themes); i++ {
+		absPaths[i] = filepath.Join(themesDir, themes[len(themes)-1-i].Name)
+	}
+
+	fs, err := createOverlayFs(p.Fs.Source, absPaths)
+
+	return fs, absPaths, err
+
+}
+
+func createOverlayFs(source afero.Fs, absPaths []string) (afero.Fs, error) {
+	if len(absPaths) == 0 {
+		return hugofs.NoOpFs, nil
+	}
+
+	if len(absPaths) == 1 {
+		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])), nil
+	}
+
+	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
+	overlay, err := createOverlayFs(source, absPaths[1:])
+	if err != nil {
+		return nil, err
+	}
+
+	return afero.NewCopyOnWriteFs(base, overlay), nil
+}
+
+func removeDuplicatesKeepRight(in []string) []string {
+	seen := make(map[string]bool)
+	var out []string
+	for i := len(in) - 1; i >= 0; i-- {
+		v := in[i]
+		if seen[v] {
+			continue
+		}
+		out = append([]string{v}, out...)
+		seen[v] = true
+	}
+
+	return out
+}
+
+func printFs(fs afero.Fs, path string, w io.Writer) {
+	if fs == nil {
+		return
+	}
+	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
+		if info != nil && !info.IsDir() {
+			s := path
+			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
+				s = s + "\tLANG: " + lang.Lang()
+			}
+			if fp, ok := info.(hugofs.FilePather); ok {
+				s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir()
+			}
+			fmt.Fprintln(w, "    ", s)
+		}
+		return nil
+	})
+}

hugolib/filesystems/basefs_test.go

@@ -0,0 +1,170 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filesystems
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/hugolib/paths"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewBaseFs(t *testing.T) {
+	assert := require.New(t)
+	v := viper.New()
+
+	fs := hugofs.NewMem(v)
+
+	themes := []string{"btheme", "atheme"}
+
+	workingDir := filepath.FromSlash("/my/work")
+	v.Set("workingDir", workingDir)
+	v.Set("themesDir", "themes")
+	v.Set("theme", themes[:1])
+
+	// Write some data to the themes
+	for _, theme := range themes {
+		for _, dir := range []string{"i18n", "data"} {
+			base := filepath.Join(workingDir, "themes", theme, dir)
+			fs.Source.Mkdir(base, 0755)
+			afero.WriteFile(fs.Source, filepath.Join(base, fmt.Sprintf("theme-file-%s-%s.txt", theme, dir)), []byte(fmt.Sprintf("content:%s:%s", theme, dir)), 0755)
+		}
+	}
+
+	afero.WriteFile(fs.Source, filepath.Join(workingDir, "themes", "btheme", "config.toml"), []byte(`
+theme = ["atheme"]
+`), 0755)
+
+	setConfigAndWriteSomeFilesTo(fs.Source, v, "contentDir", "mycontent", 3)
+	setConfigAndWriteSomeFilesTo(fs.Source, v, "i18nDir", "myi18n", 4)
+	setConfigAndWriteSomeFilesTo(fs.Source, v, "layoutDir", "mylayouts", 5)
+	setConfigAndWriteSomeFilesTo(fs.Source, v, "staticDir", "mystatic", 6)
+	setConfigAndWriteSomeFilesTo(fs.Source, v, "dataDir", "mydata", 7)
+	setConfigAndWriteSomeFilesTo(fs.Source, v, "archetypeDir", "myarchetypes", 8)
+
+	p, err := paths.New(fs, v)
+	assert.NoError(err)
+
+	bfs, err := NewBase(p)
+	assert.NoError(err)
+	assert.NotNil(bfs)
+
+	root, err := bfs.I18n.Fs.Open("")
+	assert.NoError(err)
+	dirnames, err := root.Readdirnames(-1)
+	assert.NoError(err)
+	assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames)
+	ff, err := bfs.I18n.Fs.Open("myi18n")
+	assert.NoError(err)
+	_, err = ff.Readdirnames(-1)
+	assert.NoError(err)
+
+	root, err = bfs.Data.Fs.Open("")
+	assert.NoError(err)
+	dirnames, err = root.Readdirnames(-1)
+	assert.NoError(err)
+	assert.Equal([]string{projectVirtualFolder, "btheme", "atheme"}, dirnames)
+	ff, err = bfs.I18n.Fs.Open("mydata")
+	assert.NoError(err)
+	_, err = ff.Readdirnames(-1)
+	assert.NoError(err)
+
+	checkFileCount(bfs.ContentFs, "", assert, 3)
+	checkFileCount(bfs.I18n.Fs, "", assert, 6) // 4 + 2 themes
+	checkFileCount(bfs.Layouts.Fs, "", assert, 5)
+	checkFileCount(bfs.Static[""].Fs, "", assert, 6)
+	checkFileCount(bfs.Data.Fs, "", assert, 9) // 7 + 2 themes
+	checkFileCount(bfs.Archetypes.Fs, "", assert, 8)
+
+	assert.Equal([]string{filepath.FromSlash("/my/work/mydata"), filepath.FromSlash("/my/work/themes/btheme/data"), filepath.FromSlash("/my/work/themes/atheme/data")}, bfs.Data.Dirnames)
+
+	assert.True(bfs.IsData(filepath.Join(workingDir, "mydata", "file1.txt")))
+	assert.True(bfs.IsI18n(filepath.Join(workingDir, "myi18n", "file1.txt")))
+	assert.True(bfs.IsLayout(filepath.Join(workingDir, "mylayouts", "file1.txt")))
+	assert.True(bfs.IsStatic(filepath.Join(workingDir, "mystatic", "file1.txt")))
+	contentFilename := filepath.Join(workingDir, "mycontent", "file1.txt")
+	assert.True(bfs.IsContent(contentFilename))
+	rel, _ := bfs.RelContentDir(contentFilename)
+	assert.Equal("file1.txt", rel)
+
+}
+
+func TestNewBaseFsEmpty(t *testing.T) {
+	assert := require.New(t)
+	v := viper.New()
+	v.Set("contentDir", "mycontent")
+	v.Set("i18nDir", "myi18n")
+	v.Set("staticDir", "mystatic")
+	v.Set("dataDir", "mydata")
+	v.Set("layoutDir", "mylayouts")
+	v.Set("archetypeDir", "myarchetypes")
+
+	fs := hugofs.NewMem(v)
+	p, err := paths.New(fs, v)
+	bfs, err := NewBase(p)
+	assert.NoError(err)
+	assert.NotNil(bfs)
+	assert.Equal(hugofs.NoOpFs, bfs.Archetypes.Fs)
+	assert.Equal(hugofs.NoOpFs, bfs.Layouts.Fs)
+	assert.Equal(hugofs.NoOpFs, bfs.Data.Fs)
+	assert.Equal(hugofs.NoOpFs, bfs.I18n.Fs)
+	assert.NotNil(hugofs.NoOpFs, bfs.ContentFs)
+	assert.NotNil(hugofs.NoOpFs, bfs.Static)
+}
+
+func checkFileCount(fs afero.Fs, dirname string, assert *require.Assertions, expected int) {
+	count, _, err := countFileaAndGetDirs(fs, dirname)
+	assert.NoError(err)
+	assert.Equal(expected, count)
+}
+
+func countFileaAndGetDirs(fs afero.Fs, dirname string) (int, []string, error) {
+	if fs == nil {
+		return 0, nil, errors.New("no fs")
+	}
+
+	counter := 0
+	var dirs []string
+
+	afero.Walk(fs, dirname, func(path string, info os.FileInfo, err error) error {
+		if info != nil {
+			if !info.IsDir() {
+				counter++
+			} else if info.Name() != "." {
+				dirs = append(dirs, filepath.Join(path, info.Name()))
+			}
+		}
+
+		return nil
+	})
+
+	return counter, dirs, nil
+}
+
+func setConfigAndWriteSomeFilesTo(fs afero.Fs, v *viper.Viper, key, val string, num int) {
+	workingDir := v.GetString("workingDir")
+	v.Set(key, val)
+	fs.Mkdir(val, 0755)
+	for i := 0; i < num; i++ {
+		afero.WriteFile(fs, filepath.Join(workingDir, val, fmt.Sprintf("file%d.txt", i+1)), []byte(fmt.Sprintf("content:%s:%d", key, i+1)), 0755)
+	}
+}

hugolib/hugo_sites.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/langs"
 
 	"github.com/gohugoio/hugo/i18n"
 	"github.com/gohugoio/hugo/tpl"
@@ -228,10 +229,7 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 
 func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error {
 	return func(templ tpl.TemplateHandler) error {
-		templ.LoadTemplates(s.PathSpec.GetLayoutDirPath(), "")
+		templ.LoadTemplates("")
-		if s.PathSpec.ThemeSet() {
-			templ.LoadTemplates(s.PathSpec.GetThemeDir()+"/layouts", "theme")
-		}
 
 		for _, wt := range withTemplates {
 			if wt == nil {
@@ -289,7 +287,7 @@ func (h *HugoSites) resetLogs() {
 }
 
 func (h *HugoSites) createSitesFromConfig() error {
-	oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages)
+	oldLangs, _ := h.Cfg.Get("languagesSorted").(langs.Languages)
 
 	if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil {
 		return err

hugolib/hugo_sites_build_test.go

@@ -3,7 +3,6 @@ package hugolib
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"strings"
 	"testing"
 
@@ -12,6 +11,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/fortytw2/leaktest"
 	"github.com/fsnotify/fsnotify"
 	"github.com/gohugoio/hugo/helpers"
@@ -660,7 +661,7 @@ title = "Svenska"
 	sites := b.H
 
 	// Watching does not work with in-memory fs, so we trigger a reload manually
-	assert.NoError(sites.Cfg.(*helpers.Language).Cfg.(*viper.Viper).ReadInConfig())
+	assert.NoError(sites.Cfg.(*langs.Language).Cfg.(*viper.Viper).ReadInConfig())
 	err := b.H.Build(BuildCfg{CreateSitesFromConfig: true})
 
 	if err != nil {
@@ -723,7 +724,7 @@ func TestChangeDefaultLanguage(t *testing.T) {
 
 	// Watching does not work with in-memory fs, so we trigger a reload manually
 	// This does not look pretty, so we should think of something else.
-	assert.NoError(b.H.Cfg.(*helpers.Language).Cfg.(*viper.Viper).ReadInConfig())
+	assert.NoError(b.H.Cfg.(*langs.Language).Cfg.(*viper.Viper).ReadInConfig())
 	err := b.H.Build(BuildCfg{CreateSitesFromConfig: true})
 	if err != nil {
 		t.Fatalf("Failed to rebuild sites: %s", err)
@@ -1177,31 +1178,12 @@ func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
 	if err != nil {
 		// Print some debug info
 		root := strings.Split(filename, helpers.FilePathSeparator)[0]
-		printFs(fs, root, os.Stdout)
+		helpers.PrintFs(fs, root, os.Stdout)
 		Fatalf(t, "Failed to read file: %s", err)
 	}
 	return string(b)
 }
 
-func printFs(fs afero.Fs, path string, w io.Writer) {
-	if fs == nil {
-		return
-	}
-	afero.Walk(fs, path, func(path string, info os.FileInfo, err error) error {
-		if info != nil && !info.IsDir() {
-			s := path
-			if lang, ok := info.(hugofs.LanguageAnnouncer); ok {
-				s = s + "\tLANG: " + lang.Lang()
-			}
-			if fp, ok := info.(hugofs.FilePather); ok {
-				s = s + "\tRF: " + fp.Filename() + "\tBP: " + fp.BaseDir()
-			}
-			fmt.Fprintln(w, "    ", s)
-		}
-		return nil
-	})
-}
-
 const testPageTemplate = `---
 title: "%s"
 publishdate: "%s"

hugolib/hugo_sites_multihost_test.go

@@ -55,8 +55,6 @@ languageName = "Nynorsk"
 
 	s1 := b.H.Sites[0]
 
-	assert.Equal([]string{"s1", "s2", "ens1", "ens2"}, s1.StaticDirs())
-
 	s1h := s1.getPage(KindHome)
 	assert.True(s1h.IsTranslated())
 	assert.Len(s1h.Translations(), 2)
@@ -79,7 +77,6 @@ languageName = "Nynorsk"
 	b.AssertFileContent("public/en/al/alias2/index.html", `content="0; url=https://example.com/docs/superbob/"`)
 
 	s2 := b.H.Sites[1]
-	assert.Equal([]string{"s1", "s2", "frs1", "frs2"}, s2.StaticDirs())
 
 	s2h := s2.getPage(KindHome)
 	assert.Equal("https://example.fr/", s2h.Permalink())

hugolib/hugo_themes_test.go

@@ -0,0 +1,268 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/gohugoio/hugo/common/loggers"
+)
+
+func TestThemesGraph(t *testing.T) {
+	t.Parallel()
+
+	const (
+		themeStandalone = `
+title = "Theme Standalone"
+[params]
+v1 = "v1s"
+v2 = "v2s"
+`
+		themeCyclic = `
+title = "Theme Cyclic"
+theme = "theme3"
+[params]
+v1 = "v1c"
+v2 = "v2c"
+`
+		theme1 = `
+title = "Theme #1"
+theme = "themeStandalone"
+[params]
+v2 = "v21"
+`
+
+		theme2 = `
+title = "Theme #2"
+theme = "theme1"
+[params]
+v1 = "v12"
+`
+
+		theme3 = `
+title = "Theme #3"
+theme = ["theme2", "themeStandalone", "themeCyclic"]
+[params]
+v1 = "v13"
+v2 = "v24"
+`
+
+		theme4 = `
+title = "Theme #4"
+theme = "theme3"
+[params]
+v1 = "v14"
+v2 = "v24"
+`
+
+		site1 = `
+			theme = "theme4"
+			
+			[params]
+			v1 = "site"
+`
+		site2 = `
+			theme = ["theme2", "themeStandalone"]
+`
+	)
+
+	var (
+		testConfigs = []struct {
+			siteConfig string
+
+			// The name of theme somewhere in the middle to write custom key/files.
+			offset string
+
+			check func(b *sitesBuilder)
+		}{
+			{site1, "theme3", func(b *sitesBuilder) {
+
+				// site1: theme4 theme3 theme2 theme1 themeStandalone themeCyclic
+
+				// Check data
+				// theme3 should win the offset competition
+				b.AssertFileContent("public/index.html", "theme1o::[offset][v]theme3", "theme4o::[offset][v]theme3", "themeStandaloneo::[offset][v]theme3")
+				b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme4|[theme1][other]theme1")
+				b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme4|[theme][other]theme4|[theme1][other]theme1|[theme2][other]theme2|[theme3][other]theme3")
+				b.AssertFileContent("public/index.html", "theme1::[inner][other]project|[project][other]project|[theme][other]theme1|[theme1][other]theme1|")
+				b.AssertFileContent("public/index.html", "theme4::[inner][other]project|[project][other]project|[theme][other]theme4|[theme4][other]theme4|")
+
+				// Check layouts
+				b.AssertFileContent("public/index.html", "partial ntheme: theme4", "partial theme2o: theme3")
+
+				// Check i18n
+				b.AssertFileContent("public/index.html", "i18n: project theme4")
+
+				// Check static files
+				// TODO(bep) static files not currently part of the build b.AssertFileContent("public/nproject.txt", "TODO")
+
+				// Check site params
+				b.AssertFileContent("public/index.html", "v1::site", "v2::v24")
+			}},
+			{site2, "", func(b *sitesBuilder) {
+
+				// site2: theme2 theme1 themeStandalone
+				b.AssertFileContent("public/index.html", "nproject::[inner][other]project|[project][other]project|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|")
+				b.AssertFileContent("public/index.html", "ntheme::[inner][other]theme2|[theme][other]theme2|[theme1][other]theme1|[theme2][other]theme2|[themeStandalone][other]themeStandalone|")
+				b.AssertFileContent("public/index.html", "i18n: project theme2")
+				b.AssertFileContent("public/index.html", "partial ntheme: theme2")
+
+				// Params only set in themes
+				b.AssertFileContent("public/index.html", "v1::v12", "v2::v21")
+
+			}},
+		}
+
+		themeConfigs = []struct {
+			name   string
+			config string
+		}{
+			{"themeStandalone", themeStandalone},
+			{"themeCyclic", themeCyclic},
+			{"theme1", theme1},
+			{"theme2", theme2},
+			{"theme3", theme3},
+			{"theme4", theme4},
+		}
+	)
+
+	for i, testConfig := range testConfigs {
+		t.Log(fmt.Sprintf("Test %d", i))
+		b := newTestSitesBuilder(t).WithLogger(loggers.NewErrorLogger())
+		b.WithConfigFile("toml", testConfig.siteConfig)
+
+		for _, tc := range themeConfigs {
+			var variationsNameBase = []string{"nproject", "ntheme", tc.name}
+
+			themeRoot := filepath.Join("themes", tc.name)
+			b.WithSourceFile(filepath.Join(themeRoot, "config.toml"), tc.config)
+
+			b.WithSourceFile(filepath.Join("layouts", "partials", "m.html"), `{{- range $k, $v := . }}{{ $k }}::{{ template "printv" $v }}
+{{ end }}	
+{{ define "printv" }}
+{{- $tp := printf "%T" . -}}
+{{- if (strings.HasSuffix $tp "map[string]interface {}") -}}
+{{- range $k, $v := . }}[{{ $k }}]{{ template "printv" $v }}{{ end -}}
+{{- else -}}
+{{- . }}|
+{{- end -}}
+{{ end }}
+`)
+
+			for _, nameVariaton := range variationsNameBase {
+				roots := []string{"", themeRoot}
+
+				for _, root := range roots {
+					name := tc.name
+					if root == "" {
+						name = "project"
+					}
+
+					if nameVariaton == "ntheme" && name == "project" {
+						continue
+					}
+
+					// static
+					b.WithSourceFile(filepath.Join(root, "static", nameVariaton+".txt"), name)
+
+					// layouts
+					if i == 1 {
+						b.WithSourceFile(filepath.Join(root, "layouts", "partials", "theme2o.html"), "Not Set")
+					}
+					b.WithSourceFile(filepath.Join(root, "layouts", "partials", nameVariaton+".html"), name)
+					if root != "" && testConfig.offset == tc.name {
+						for _, tc2 := range themeConfigs {
+							b.WithSourceFile(filepath.Join(root, "layouts", "partials", tc2.name+"o.html"), name)
+						}
+					}
+
+					// i18n + data
+
+					var dataContent string
+					if root == "" {
+						dataContent = fmt.Sprintf(`
+[%s]
+other = %q
+
+[inner]
+other = %q
+
+`, name, name, name)
+					} else {
+						dataContent = fmt.Sprintf(`
+[%s]
+other = %q
+
+[inner]
+other = %q
+
+[theme]
+other = %q
+
+`, name, name, name, name)
+					}
+
+					b.WithSourceFile(filepath.Join(root, "data", nameVariaton+".toml"), dataContent)
+					b.WithSourceFile(filepath.Join(root, "i18n", "en.toml"), dataContent)
+
+					// If an offset is set, duplicate a data key with a winner in the middle.
+					if root != "" && testConfig.offset == tc.name {
+						for _, tc2 := range themeConfigs {
+							dataContent := fmt.Sprintf(`
+[offset]
+v = %q
+`, tc.name)
+							b.WithSourceFile(filepath.Join(root, "data", tc2.name+"o.toml"), dataContent)
+						}
+					}
+				}
+
+			}
+
+		}
+
+		for _, themeConfig := range themeConfigs {
+			b.WithSourceFile(filepath.Join("themes", "config.toml"), themeConfig.config)
+		}
+
+		b.WithContent(filepath.Join("content", "page.md"), `---
+title: "Page"
+---
+
+`)
+
+		homeTpl := `
+data: {{ partial "m" .Site.Data }}
+i18n: {{ i18n "inner" }} {{ i18n "theme" }}
+partial ntheme: {{ partial "ntheme" . }}
+partial theme2o: {{ partial "theme2o" . }}
+params: {{ partial "m" .Site.Params }} 
+		
+`
+
+		b.WithTemplates(filepath.Join("layouts", "home.html"), homeTpl)
+
+		b.Build(BuildCfg{})
+
+		var _ = os.Stdout
+
+		//	printFs(b.H.Deps.BaseFs.LayoutsFs, "", os.Stdout)
+		testConfig.check(b)
+
+	}
+
+}

hugolib/multilingual.go

@@ -16,30 +16,33 @@ package hugolib
 import (
 	"sync"
 
+	"github.com/gohugoio/hugo/common/maps"
+
 	"sort"
 
 	"errors"
 	"fmt"
 
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/cast"
 )
 
 // Multilingual manages the all languages used in a multilingual site.
 type Multilingual struct {
-	Languages helpers.Languages
+	Languages langs.Languages
 
-	DefaultLang *helpers.Language
+	DefaultLang *langs.Language
 
-	langMap     map[string]*helpers.Language
+	langMap     map[string]*langs.Language
 	langMapInit sync.Once
 }
 
 // Language returns the Language associated with the given string.
-func (ml *Multilingual) Language(lang string) *helpers.Language {
+func (ml *Multilingual) Language(lang string) *langs.Language {
 	ml.langMapInit.Do(func() {
-		ml.langMap = make(map[string]*helpers.Language)
+		ml.langMap = make(map[string]*langs.Language)
 		for _, l := range ml.Languages {
 			ml.langMap[l.Lang] = l
 		}
@@ -47,16 +50,16 @@ func (ml *Multilingual) Language(lang string) *helpers.Language {
 	return ml.langMap[lang]
 }
 
-func getLanguages(cfg config.Provider) helpers.Languages {
+func getLanguages(cfg config.Provider) langs.Languages {
 	if cfg.IsSet("languagesSorted") {
-		return cfg.Get("languagesSorted").(helpers.Languages)
+		return cfg.Get("languagesSorted").(langs.Languages)
 	}
 
-	return helpers.Languages{helpers.NewDefaultLanguage(cfg)}
+	return langs.Languages{langs.NewDefaultLanguage(cfg)}
 }
 
 func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingual, error) {
-	languages := make(helpers.Languages, len(sites))
+	languages := make(langs.Languages, len(sites))
 
 	for i, s := range sites {
 		if s.Language == nil {
@@ -71,12 +74,12 @@ func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingua
 		defaultLang = "en"
 	}
 
-	return &Multilingual{Languages: languages, DefaultLang: helpers.NewLanguage(defaultLang, cfg)}, nil
+	return &Multilingual{Languages: languages, DefaultLang: langs.NewLanguage(defaultLang, cfg)}, nil
 
 }
 
-func newMultiLingualForLanguage(language *helpers.Language) *Multilingual {
+func newMultiLingualForLanguage(language *langs.Language) *Multilingual {
-	languages := helpers.Languages{language}
+	languages := langs.Languages{language}
 	return &Multilingual{Languages: languages, DefaultLang: language}
 }
 func (ml *Multilingual) enabled() bool {
@@ -90,8 +93,8 @@ func (s *Site) multilingualEnabled() bool {
 	return s.owner.multilingual != nil && s.owner.multilingual.enabled()
 }
 
-func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.Languages, error) {
+func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (langs.Languages, error) {
-	langs := make(helpers.Languages, len(l))
+	languages := make(langs.Languages, len(l))
 	i := 0
 
 	for lang, langConf := range l {
@@ -101,7 +104,7 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L
 			return nil, fmt.Errorf("Language config is not a map: %T", langConf)
 		}
 
-		language := helpers.NewLanguage(lang, cfg)
+		language := langs.NewLanguage(lang, cfg)
 
 		for loki, v := range langsMap {
 			switch loki {
@@ -118,7 +121,7 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L
 			case "params":
 				m := cast.ToStringMap(v)
 				// Needed for case insensitive fetching of params values
-				helpers.ToLowerMap(m)
+				maps.ToLower(m)
 				for k, vv := range m {
 					language.SetParam(k, vv)
 				}
@@ -131,11 +134,11 @@ func toSortedLanguages(cfg config.Provider, l map[string]interface{}) (helpers.L
 			language.Set(loki, v)
 		}
 
-		langs[i] = language
+		languages[i] = language
 		i++
 	}
 
-	sort.Sort(langs)
+	sort.Sort(languages)
 
-	return langs, nil
+	return languages, nil
 }

hugolib/page.go

@@ -21,6 +21,10 @@ import (
 	"reflect"
 	"unicode"
 
+	"github.com/gohugoio/hugo/common/maps"
+
+	"github.com/gohugoio/hugo/langs"
+
 	"github.com/gohugoio/hugo/related"
 
 	"github.com/bep/gitmap"
@@ -254,7 +258,7 @@ type Page struct {
 
 	// It would be tempting to use the language set on the Site, but in they way we do
 	// multi-site processing, these values may differ during the initial page processing.
-	language *helpers.Language
+	language *langs.Language
 
 	lang string
 
@@ -1281,7 +1285,7 @@ func (p *Page) update(frontmatter map[string]interface{}) error {
 		return errors.New("missing frontmatter data")
 	}
 	// Needed for case insensitive fetching of params values
-	helpers.ToLowerMap(frontmatter)
+	maps.ToLower(frontmatter)
 
 	var mtime time.Time
 	if p.Source.FileInfo() != nil {
@@ -2028,7 +2032,7 @@ func (p *Page) Scratch() *Scratch {
 	return p.scratch
 }
 
-func (p *Page) Language() *helpers.Language {
+func (p *Page) Language() *langs.Language {
 	p.initLanguage()
 	return p.language
 }

hugolib/page_bundler_capture.go

@@ -75,7 +75,7 @@ func newCapturer(
 		sem:            make(chan bool, numWorkers),
 		handler:        handler,
 		sourceSpec:     sourceSpec,
-		fs:             sourceSpec.Fs,
+		fs:             sourceSpec.SourceFs,
 		logger:         logger,
 		contentChanges: contentChanges,
 		seen:           make(map[string]bool),

hugolib/page_bundler_capture_test.go

@@ -20,6 +20,8 @@ import (
 	"path/filepath"
 	"sort"
 
+	"github.com/gohugoio/hugo/common/loggers"
+
 	jww "github.com/spf13/jwalterweatherman"
 
 	"runtime"
@@ -92,7 +94,7 @@ func TestPageBundlerCaptureSymlinks(t *testing.T) {
 	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
 
 	fileStore := &storeFilenames{}
-	logger := newErrorLogger()
+	logger := loggers.NewErrorLogger()
 	c := newCapturer(logger, sourceSpec, fileStore, nil)
 
 	assert.NoError(c.capture())
@@ -139,12 +141,10 @@ func TestPageBundlerCaptureBasic(t *testing.T) {
 
 	fileStore := &storeFilenames{}
 
-	c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil)
+	c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil)
 
 	assert.NoError(c.capture())
 
-	printFs(fs.Source, "", os.Stdout)
-
 	expected := `
 F:
 /work/base/_1.md
@@ -185,7 +185,7 @@ func TestPageBundlerCaptureMultilingual(t *testing.T) {
 
 	sourceSpec := source.NewSourceSpec(ps, ps.BaseFs.ContentFs)
 	fileStore := &storeFilenames{}
-	c := newCapturer(newErrorLogger(), sourceSpec, fileStore, nil)
+	c := newCapturer(loggers.NewErrorLogger(), sourceSpec, fileStore, nil)
 
 	assert.NoError(c.capture())
 
@@ -265,7 +265,7 @@ func BenchmarkPageBundlerCapture(b *testing.B) {
 			writeSource(b, fs, filepath.Join(base, "contentonly", fmt.Sprintf("c%d.md", i)), "content")
 		}
 
-		capturers[i] = newCapturer(newErrorLogger(), sourceSpec, new(noOpFileStore), nil, base)
+		capturers[i] = newCapturer(loggers.NewErrorLogger(), sourceSpec, new(noOpFileStore), nil, base)
 	}
 
 	b.ResetTimer()

hugolib/page_bundler_test.go

@@ -15,6 +15,9 @@ package hugolib
 
 import (
 	"io/ioutil"
+
+	"github.com/gohugoio/hugo/common/loggers"
+
 	"os"
 	"runtime"
 	"strings"
@@ -75,7 +78,7 @@ func TestPageBundlerSiteRegular(t *testing.T) {
 
 				cfg.Set("uglyURLs", ugly)
 
-				s := buildSingleSite(t, deps.DepsCfg{Logger: newWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{})
+				s := buildSingleSite(t, deps.DepsCfg{Logger: loggers.NewWarningLogger(), Fs: fs, Cfg: cfg}, BuildCfg{})
 
 				th := testHelper{s.Cfg, s.Fs, t}
 
@@ -158,7 +161,6 @@ func TestPageBundlerSiteRegular(t *testing.T) {
 				assert.Equal(filepath.FromSlash("/work/base/b/my-bundle/c/logo.png"), image.(resource.Source).AbsSourceFilename())
 				assert.Equal("https://example.com/2017/pageslug/c/logo.png", image.Permalink())
 
-				printFs(th.Fs.Destination, "", os.Stdout)
 				th.assertFileContent(filepath.FromSlash("/work/public/2017/pageslug/c/logo.png"), "content")
 				th.assertFileContent(filepath.FromSlash("/work/public/cpath/2017/pageslug/c/logo.png"), "content")
 
@@ -329,7 +331,7 @@ func TestPageBundlerSiteWitSymbolicLinksInContent(t *testing.T) {
 	cfg := ps.Cfg
 	fs := ps.Fs
 
-	s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: newErrorLogger()}, BuildCfg{})
+	s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg, Logger: loggers.NewErrorLogger()}, BuildCfg{})
 
 	th := testHelper{s.Cfg, s.Fs, t}
 

hugolib/pagination.go

@@ -532,7 +532,7 @@ func newPaginationURLFactory(d targetPathDescriptor) paginationURLFactory {
 		pathDescriptor := d
 		var rel string
 		if page > 1 {
-			rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath(), page)
+			rel = fmt.Sprintf("/%s/%d/", d.PathSpec.PaginatePath, page)
 			pathDescriptor.Addends = rel
 		}
 

hugolib/paths/baseURL.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package helpers
+package paths
 
 import (
 	"fmt"
@@ -30,6 +30,10 @@ func (b BaseURL) String() string {
 	return b.urlStr
 }
 
+func (b BaseURL) Path() string {
+	return b.url.Path
+}
+
 // WithProtocol returns the BaseURL prefixed with the given protocol.
 // The Protocol is normally of the form "scheme://", i.e. "webcal://".
 func (b BaseURL) WithProtocol(protocol string) (string, error) {

hugolib/paths/baseURL_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package helpers
+package paths
 
 import (
 	"testing"

hugolib/paths/paths.go

@@ -0,0 +1,231 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/langs"
+
+	"github.com/gohugoio/hugo/hugofs"
+)
+
+var FilePathSeparator = string(filepath.Separator)
+
+type Paths struct {
+	Fs  *hugofs.Fs
+	Cfg config.Provider
+
+	BaseURL
+
+	// If the baseURL contains a base path, e.g. https://example.com/docs, then "/docs" will be the BasePath.
+	// This will not be set if canonifyURLs is enabled.
+	BasePath string
+
+	// Directories
+	// TODO(bep) when we have trimmed down mos of the dirs usage outside of this package, make
+	// these into an interface.
+	ContentDir      string
+	ThemesDir       string
+	WorkingDir      string
+	AbsResourcesDir string
+	AbsPublishDir   string
+
+	// pagination path handling
+	PaginatePath string
+
+	PublishDir string
+
+	DisablePathToLower bool
+	RemovePathAccents  bool
+	UglyURLs           bool
+	CanonifyURLs       bool
+
+	Language  *langs.Language
+	Languages langs.Languages
+
+	// The PathSpec looks up its config settings in both the current language
+	// and then in the global Viper config.
+	// Some settings, the settings listed below, does not make sense to be set
+	// on per-language-basis. We have no good way of protecting against this
+	// other than a "white-list". See language.go.
+	defaultContentLanguageInSubdir bool
+	DefaultContentLanguage         string
+	multilingual                   bool
+
+	themes    []string
+	AllThemes []ThemeConfig
+}
+
+func New(fs *hugofs.Fs, cfg config.Provider) (*Paths, error) {
+	baseURLstr := cfg.GetString("baseURL")
+	baseURL, err := newBaseURLFromString(baseURLstr)
+
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err)
+	}
+
+	// TODO(bep)
+	contentDir := cfg.GetString("contentDir")
+	workingDir := cfg.GetString("workingDir")
+	resourceDir := cfg.GetString("resourceDir")
+	publishDir := cfg.GetString("publishDir")
+
+	defaultContentLanguage := cfg.GetString("defaultContentLanguage")
+
+	var (
+		language  *langs.Language
+		languages langs.Languages
+	)
+
+	if l, ok := cfg.(*langs.Language); ok {
+		language = l
+
+	}
+
+	if l, ok := cfg.Get("languagesSorted").(langs.Languages); ok {
+		languages = l
+	}
+
+	if len(languages) == 0 {
+		// We have some old tests that does not test the entire chain, hence
+		// they have no languages. So create one so we get the proper filesystem.
+		languages = langs.Languages{&langs.Language{Lang: "en", Cfg: cfg, ContentDir: contentDir}}
+	}
+
+	absPublishDir := AbsPathify(workingDir, publishDir)
+	if !strings.HasSuffix(absPublishDir, FilePathSeparator) {
+		absPublishDir += FilePathSeparator
+	}
+	// If root, remove the second '/'
+	if absPublishDir == "//" {
+		absPublishDir = FilePathSeparator
+	}
+	absResourcesDir := AbsPathify(workingDir, resourceDir)
+	if !strings.HasSuffix(absResourcesDir, FilePathSeparator) {
+		absResourcesDir += FilePathSeparator
+	}
+	if absResourcesDir == "//" {
+		absResourcesDir = FilePathSeparator
+	}
+
+	p := &Paths{
+		Fs:      fs,
+		Cfg:     cfg,
+		BaseURL: baseURL,
+
+		DisablePathToLower: cfg.GetBool("disablePathToLower"),
+		RemovePathAccents:  cfg.GetBool("removePathAccents"),
+		UglyURLs:           cfg.GetBool("uglyURLs"),
+		CanonifyURLs:       cfg.GetBool("canonifyURLs"),
+
+		ContentDir: contentDir,
+		ThemesDir:  cfg.GetString("themesDir"),
+		WorkingDir: workingDir,
+
+		AbsResourcesDir: absResourcesDir,
+		AbsPublishDir:   absPublishDir,
+
+		themes: config.GetStringSlicePreserveString(cfg, "theme"),
+
+		multilingual:                   cfg.GetBool("multilingual"),
+		defaultContentLanguageInSubdir: cfg.GetBool("defaultContentLanguageInSubdir"),
+		DefaultContentLanguage:         defaultContentLanguage,
+
+		Language:  language,
+		Languages: languages,
+
+		PaginatePath: cfg.GetString("paginatePath"),
+	}
+
+	if cfg.IsSet("allThemes") {
+		p.AllThemes = cfg.Get("allThemes").([]ThemeConfig)
+	} else {
+		p.AllThemes, err = collectThemeNames(p)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// TODO(bep) remove this, eventually
+	p.PublishDir = absPublishDir
+
+	return p, nil
+}
+
+func (p *Paths) Lang() string {
+	if p == nil || p.Language == nil {
+		return ""
+	}
+	return p.Language.Lang
+}
+
+// ThemeSet checks whether a theme is in use or not.
+func (p *Paths) ThemeSet() bool {
+	return len(p.themes) > 0
+}
+
+func (p *Paths) Themes() []string {
+	return p.themes
+}
+
+func (p *Paths) GetLanguagePrefix() string {
+	if !p.multilingual {
+		return ""
+	}
+
+	defaultLang := p.DefaultContentLanguage
+	defaultInSubDir := p.defaultContentLanguageInSubdir
+
+	currentLang := p.Language.Lang
+	if currentLang == "" || (currentLang == defaultLang && !defaultInSubDir) {
+		return ""
+	}
+	return currentLang
+}
+
+// GetLangSubDir returns the given language's subdir if needed.
+func (p *Paths) GetLangSubDir(lang string) string {
+	if !p.multilingual {
+		return ""
+	}
+
+	if p.Languages.IsMultihost() {
+		return ""
+	}
+
+	if lang == "" || (lang == p.DefaultContentLanguage && !p.defaultContentLanguageInSubdir) {
+		return ""
+	}
+
+	return lang
+}
+
+// AbsPathify creates an absolute path if given a relative path. If already
+// absolute, the path is just cleaned.
+func (p *Paths) AbsPathify(inPath string) string {
+	return AbsPathify(p.WorkingDir, inPath)
+}
+
+// AbsPathify creates an absolute path if given a working dir and arelative path.
+// If already absolute, the path is just cleaned.
+func AbsPathify(workingDir, inPath string) string {
+	if filepath.IsAbs(inPath) {
+		return filepath.Clean(inPath)
+	}
+	return filepath.Join(workingDir, inPath)
+}

hugolib/paths/paths_test.go

@@ -0,0 +1,40 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewPaths(t *testing.T) {
+	assert := require.New(t)
+
+	v := viper.New()
+	fs := hugofs.NewMem(v)
+
+	v.Set("defaultContentLanguageInSubdir", true)
+	v.Set("defaultContentLanguage", "no")
+	v.Set("multilingual", true)
+
+	p, err := New(fs, v)
+	assert.NoError(err)
+
+	assert.Equal(true, p.defaultContentLanguageInSubdir)
+	assert.Equal("no", p.DefaultContentLanguage)
+	assert.Equal(true, p.multilingual)
+}

hugolib/paths/themes.go

@@ -0,0 +1,162 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package paths
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+	"github.com/spf13/cast"
+	"github.com/spf13/viper"
+)
+
+type ThemeConfig struct {
+	// The theme name as provided by the folder name below /themes.
+	Name string
+
+	// Optional configuration filename (e.g. "/themes/mytheme/config.json").
+	ConfigFilename string
+
+	// Optional config read from the ConfigFile above.
+	Cfg config.Provider
+}
+
+// Create file system, an ordered theme list from left to right, no duplicates.
+type themesCollector struct {
+	themesDir string
+	fs        afero.Fs
+	seen      map[string]bool
+	themes    []ThemeConfig
+}
+
+func (c *themesCollector) isSeen(theme string) bool {
+	loki := strings.ToLower(theme)
+	if c.seen[loki] {
+		return true
+	}
+	c.seen[loki] = true
+	return false
+}
+
+func (c *themesCollector) addAndRecurse(themes ...string) error {
+	for i := 0; i < len(themes); i++ {
+		theme := themes[i]
+		configFilename := c.getConfigFileIfProvided(theme)
+		if !c.isSeen(theme) {
+			tc, err := c.add(theme, configFilename)
+			if err != nil {
+				return err
+			}
+			if err := c.addTemeNamesFromTheme(tc); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (c *themesCollector) add(name, configFilename string) (ThemeConfig, error) {
+	var cfg config.Provider
+	var tc ThemeConfig
+
+	if configFilename != "" {
+		v := viper.New()
+		v.SetFs(c.fs)
+		v.AutomaticEnv()
+		v.SetEnvPrefix("hugo")
+		v.SetConfigFile(configFilename)
+
+		err := v.ReadInConfig()
+		if err != nil {
+			return tc, err
+		}
+		cfg = v
+
+	}
+
+	tc = ThemeConfig{Name: name, ConfigFilename: configFilename, Cfg: cfg}
+	c.themes = append(c.themes, tc)
+	return tc, nil
+
+}
+
+func collectThemeNames(p *Paths) ([]ThemeConfig, error) {
+	return CollectThemes(p.Fs.Source, p.AbsPathify(p.ThemesDir), p.Themes())
+
+}
+
+func CollectThemes(fs afero.Fs, themesDir string, themes []string) ([]ThemeConfig, error) {
+	if len(themes) == 0 {
+		return nil, nil
+	}
+
+	c := &themesCollector{
+		fs:        fs,
+		themesDir: themesDir,
+		seen:      make(map[string]bool)}
+
+	for i := 0; i < len(themes); i++ {
+		theme := themes[i]
+		if err := c.addAndRecurse(theme); err != nil {
+			return nil, err
+		}
+	}
+
+	return c.themes, nil
+
+}
+
+func (c *themesCollector) getConfigFileIfProvided(theme string) string {
+	configDir := filepath.Join(c.themesDir, theme)
+
+	var (
+		configFilename string
+		exists         bool
+	)
+
+	// Viper supports more, but this is the sub-set supported by Hugo.
+	for _, configFormats := range []string{"toml", "yaml", "yml", "json"} {
+		configFilename = filepath.Join(configDir, "config."+configFormats)
+		exists, _ = afero.Exists(c.fs, configFilename)
+		if exists {
+			break
+		}
+	}
+
+	if !exists {
+		// No theme config set.
+		return ""
+	}
+
+	return configFilename
+
+}
+
+func (c *themesCollector) addTemeNamesFromTheme(theme ThemeConfig) error {
+	if theme.Cfg != nil && theme.Cfg.IsSet("theme") {
+		v := theme.Cfg.Get("theme")
+		switch vv := v.(type) {
+		case []string:
+			return c.addAndRecurse(vv...)
+		case []interface{}:
+			return c.addAndRecurse(cast.ToStringSlice(vv)...)
+		default:
+			return c.addAndRecurse(cast.ToString(vv))
+		}
+	}
+
+	return nil
+}

hugolib/shortcode_test.go

@@ -34,7 +34,9 @@ import (
 
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/tpl"
+
 	"github.com/stretchr/testify/require"
 )
 
@@ -46,7 +48,7 @@ func pageFromString(in, filename string, withTemplate ...func(templ tpl.Template
 		var err error
 		cfg, fs := newTestCfg()
 
-		d := deps.DepsCfg{Language: helpers.NewLanguage("en", cfg), Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]}
+		d := deps.DepsCfg{Language: langs.NewLanguage("en", cfg), Cfg: cfg, Fs: fs, WithTemplate: withTemplate[0]}
 
 		s, err = NewSiteForCfg(d)
 		if err != nil {

hugolib/site.go

@@ -27,6 +27,10 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/langs"
+
+	src "github.com/gohugoio/hugo/source"
+
 	"github.com/gohugoio/hugo/resource"
 
 	"golang.org/x/sync/errgroup"
@@ -107,7 +111,7 @@ type Site struct {
 	expiredCount int
 
 	Data     map[string]interface{}
-	Language *helpers.Language
+	Language *langs.Language
 
 	disabledKinds map[string]bool
 
@@ -175,7 +179,7 @@ func (s *Site) isEnabled(kind string) bool {
 // reset returns a new Site prepared for rebuild.
 func (s *Site) reset() *Site {
 	return &Site{Deps: s.Deps,
-		layoutHandler:       output.NewLayoutHandler(s.PathSpec.ThemeSet()),
+		layoutHandler:       output.NewLayoutHandler(),
 		disabledKinds:       s.disabledKinds,
 		titleFunc:           s.titleFunc,
 		relatedDocsHandler:  newSearchIndexHandler(s.relatedDocsHandler.cfg),
@@ -195,7 +199,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) {
 	c := newPageCollections()
 
 	if cfg.Language == nil {
-		cfg.Language = helpers.NewDefaultLanguage(cfg.Cfg)
+		cfg.Language = langs.NewDefaultLanguage(cfg.Cfg)
 	}
 
 	disabledKinds := make(map[string]bool)
@@ -261,7 +265,7 @@ func newSite(cfg deps.DepsCfg) (*Site, error) {
 
 	s := &Site{
 		PageCollections:     c,
-		layoutHandler:       output.NewLayoutHandler(cfg.Cfg.GetString("themesDir") != ""),
+		layoutHandler:       output.NewLayoutHandler(),
 		Language:            cfg.Language,
 		disabledKinds:       disabledKinds,
 		titleFunc:           titleFunc,
@@ -304,7 +308,7 @@ func NewSiteDefaultLang(withTemplate ...func(templ tpl.TemplateHandler) error) (
 	if err := loadDefaultSettingsFor(v); err != nil {
 		return nil, err
 	}
-	return newSiteForLang(helpers.NewDefaultLanguage(v), withTemplate...)
+	return newSiteForLang(langs.NewDefaultLanguage(v), withTemplate...)
 }
 
 // NewEnglishSite creates a new site in English language.
@@ -316,11 +320,11 @@ func NewEnglishSite(withTemplate ...func(templ tpl.TemplateHandler) error) (*Sit
 	if err := loadDefaultSettingsFor(v); err != nil {
 		return nil, err
 	}
-	return newSiteForLang(helpers.NewLanguage("en", v), withTemplate...)
+	return newSiteForLang(langs.NewLanguage("en", v), withTemplate...)
 }
 
 // newSiteForLang creates a new site in the given language.
-func newSiteForLang(lang *helpers.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) {
+func newSiteForLang(lang *langs.Language, withTemplate ...func(templ tpl.TemplateHandler) error) (*Site, error) {
 	withTemplates := func(templ tpl.TemplateHandler) error {
 		for _, wt := range withTemplate {
 			if err := wt(templ); err != nil {
@@ -389,9 +393,9 @@ type SiteInfo struct {
 	owner                          *HugoSites
 	s                              *Site
 	multilingual                   *Multilingual
-	Language                       *helpers.Language
+	Language                       *langs.Language
 	LanguagePrefix                 string
-	Languages                      helpers.Languages
+	Languages                      langs.Languages
 	defaultContentLanguageInSubdir bool
 	sectionPagesMenu               string
 }
@@ -431,7 +435,7 @@ func (s *SiteInfo) DisqusShortname() string {
 // Used in tests.
 
 type siteBuilderCfg struct {
-	language        *helpers.Language
+	language        *langs.Language
 	s               *Site
 	pageCollections *PageCollections
 }
@@ -805,15 +809,13 @@ func (s *Site) processPartial(events []fsnotify.Event) (whatChanged, error) {
 
 }
 
-func (s *Site) loadData(sourceDirs []string) (err error) {
+func (s *Site) loadData(fs afero.Fs) (err error) {
-	s.Log.DEBUG.Printf("Load Data from %d source(s)", len(sourceDirs))
+	spec := src.NewSourceSpec(s.PathSpec, fs)
+	fileSystem := spec.NewFilesystem("")
 	s.Data = make(map[string]interface{})
-	for _, sourceDir := range sourceDirs {
+	for _, r := range fileSystem.Files() {
-		fs := s.SourceSpec.NewFilesystem(sourceDir)
+		if err := s.handleDataFile(r); err != nil {
-		for _, r := range fs.Files() {
+			return err
-			if err := s.handleDataFile(r); err != nil {
-				return err
-			}
 		}
 	}
 
@@ -831,12 +833,17 @@ func (s *Site) handleDataFile(r source.ReadableFile) error {
 
 	// Crawl in data tree to insert data
 	current = s.Data
-	for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) {
+	keyParts := strings.Split(r.Dir(), helpers.FilePathSeparator)
-		if key != "" {
+	// The first path element is the virtual folder (typically theme name), which is
-			if _, ok := current[key]; !ok {
+	// not part of the key.
-				current[key] = make(map[string]interface{})
+	if len(keyParts) > 1 {
+		for _, key := range keyParts[1:] {
+			if key != "" {
+				if _, ok := current[key]; !ok {
+					current[key] = make(map[string]interface{})
+				}
+				current = current[key].(map[string]interface{})
 			}
-			current = current[key].(map[string]interface{})
 		}
 	}
 
@@ -919,18 +926,7 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) {
 }
 
 func (s *Site) readDataFromSourceFS() error {
-	var dataSourceDirs []string
+	err := s.loadData(s.PathSpec.BaseFs.Data.Fs)
-
-	// have to be last - duplicate keys in earlier entries will win
-	themeDataDir, err := s.PathSpec.GetThemeDataDirPath()
-	if err == nil {
-		dataSourceDirs = []string{s.absDataDir(), themeDataDir}
-	} else {
-		dataSourceDirs = []string{s.absDataDir()}
-
-	}
-
-	err = s.loadData(dataSourceDirs)
 	s.timerStep("load data")
 	return err
 }
@@ -1041,10 +1037,6 @@ func (s *Site) Initialise() (err error) {
 func (s *Site) initialize() (err error) {
 	s.Menus = Menus{}
 
-	if err = s.checkDirectories(); err != nil {
-		return err
-	}
-
 	return s.initializeSiteInfo()
 }
 
@@ -1071,7 +1063,7 @@ func (s *SiteInfo) SitemapAbsURL() string {
 func (s *Site) initializeSiteInfo() error {
 	var (
 		lang      = s.Language
-		languages helpers.Languages
+		languages langs.Languages
 	)
 
 	if s.owner != nil && s.owner.multilingual != nil {
@@ -1166,126 +1158,24 @@ func (s *Site) initializeSiteInfo() error {
 	return nil
 }
 
-func (s *Site) dataDir() string {
-	return s.Cfg.GetString("dataDir")
-}
-
-func (s *Site) absDataDir() string {
-	return s.PathSpec.AbsPathify(s.dataDir())
-}
-
-func (s *Site) i18nDir() string {
-	return s.Cfg.GetString("i18nDir")
-}
-
-func (s *Site) absI18nDir() string {
-	return s.PathSpec.AbsPathify(s.i18nDir())
-}
-
 func (s *Site) isI18nEvent(e fsnotify.Event) bool {
-	if s.getI18nDir(e.Name) != "" {
+	return s.BaseFs.SourceFilesystems.IsI18n(e.Name)
-		return true
-	}
-	return s.getThemeI18nDir(e.Name) != ""
-}
-
-func (s *Site) getI18nDir(path string) string {
-	return s.getRealDir(s.absI18nDir(), path)
-}
-
-func (s *Site) getThemeI18nDir(path string) string {
-	if !s.PathSpec.ThemeSet() {
-		return ""
-	}
-	return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.i18nDir()), path)
 }
 
 func (s *Site) isDataDirEvent(e fsnotify.Event) bool {
-	if s.getDataDir(e.Name) != "" {
+	return s.BaseFs.SourceFilesystems.IsData(e.Name)
-		return true
-	}
-	return s.getThemeDataDir(e.Name) != ""
-}
-
-func (s *Site) getDataDir(path string) string {
-	return s.getRealDir(s.absDataDir(), path)
-}
-
-func (s *Site) getThemeDataDir(path string) string {
-	if !s.PathSpec.ThemeSet() {
-		return ""
-	}
-	return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.dataDir()), path)
-}
-
-func (s *Site) layoutDir() string {
-	return s.Cfg.GetString("layoutDir")
 }
 
 func (s *Site) isLayoutDirEvent(e fsnotify.Event) bool {
-	if s.getLayoutDir(e.Name) != "" {
+	return s.BaseFs.SourceFilesystems.IsLayout(e.Name)
-		return true
-	}
-	return s.getThemeLayoutDir(e.Name) != ""
-}
-
-func (s *Site) getLayoutDir(path string) string {
-	return s.getRealDir(s.PathSpec.GetLayoutDirPath(), path)
-}
-
-func (s *Site) getThemeLayoutDir(path string) string {
-	if !s.PathSpec.ThemeSet() {
-		return ""
-	}
-	return s.getRealDir(filepath.Join(s.PathSpec.GetThemeDir(), s.layoutDir()), path)
 }
 
 func (s *Site) absContentDir() string {
-	return s.PathSpec.AbsPathify(s.PathSpec.ContentDir())
+	return s.PathSpec.AbsPathify(s.PathSpec.ContentDir)
 }
 
 func (s *Site) isContentDirEvent(e fsnotify.Event) bool {
-	relDir, _ := s.PathSpec.RelContentDir(e.Name)
+	return s.BaseFs.IsContent(e.Name)
-	return relDir != e.Name
-}
-
-func (s *Site) getContentDir(path string) string {
-	return s.getRealDir(s.absContentDir(), path)
-}
-
-// getRealDir gets the base path of the given path, also handling the case where
-// base is a symlinked folder.
-func (s *Site) getRealDir(base, path string) string {
-
-	if strings.HasPrefix(path, base) {
-		return base
-	}
-
-	realDir, err := helpers.GetRealPath(s.Fs.Source, base)
-
-	if err != nil {
-		if !os.IsNotExist(err) {
-			s.Log.ERROR.Printf("Failed to get real path for %s: %s", path, err)
-		}
-		return ""
-	}
-
-	if strings.HasPrefix(path, realDir) {
-		return realDir
-	}
-
-	return ""
-}
-
-func (s *Site) absPublishDir() string {
-	return s.PathSpec.AbsPathify(s.Cfg.GetString("publishDir"))
-}
-
-func (s *Site) checkDirectories() (err error) {
-	if b, _ := helpers.DirExists(s.absContentDir(), s.Fs.Source); !b {
-		return errors.New("No source directory found, expecting to find it at " + s.absContentDir())
-	}
-	return
 }
 
 type contentCaptureResultHandler struct {
@@ -1871,9 +1761,7 @@ func (s *Site) findFirstTemplate(layouts ...string) tpl.Template {
 func (s *Site) publish(statCounter *uint64, path string, r io.Reader) (err error) {
 	s.PathSpec.ProcessingStats.Incr(statCounter)
 
-	path = filepath.Join(s.absPublishDir(), path)
+	return helpers.WriteToDisk(filepath.Clean(path), r, s.BaseFs.PublishFs)
-
-	return helpers.WriteToDisk(path, r, s.Fs.Destination)
 }
 
 func getGoMaxProcs() int {

hugolib/testhelpers_test.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"text/template"
 
+	"github.com/gohugoio/hugo/langs"
 	"github.com/sanity-io/litter"
-
 	jww "github.com/spf13/jwalterweatherman"
 
 	"github.com/gohugoio/hugo/config"
@@ -22,11 +22,8 @@ import (
 	"github.com/gohugoio/hugo/tpl"
 	"github.com/spf13/viper"
 
-	"io/ioutil"
 	"os"
 
-	"log"
-
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -135,6 +132,11 @@ func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder {
 	return s
 }
 
+func (s *sitesBuilder) WithSourceFile(filename, content string) *sitesBuilder {
+	writeSource(s.T, s.Fs, filepath.FromSlash(filename), content)
+	return s
+}
+
 const commonConfigSections = `
 
 [services]
@@ -304,15 +306,17 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder {
 	s.writeFilePairs("i18n", s.i18nFilePairsAdded)
 
 	if s.Cfg == nil {
-		cfg, configFiles, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat})
+		cfg, _, err := LoadConfig(ConfigSourceDescriptor{Fs: s.Fs.Source, Filename: "config." + s.configFormat})
 		if err != nil {
 			s.Fatalf("Failed to load config: %s", err)
 		}
-		expectedConfigs := 1
+		// TODO(bep)
-		if s.theme != "" {
+		/*		expectedConfigs := 1
-			expectedConfigs = 2
+				if s.theme != "" {
-		}
+					expectedConfigs = 2
-		require.Equal(s.T, expectedConfigs, len(configFiles), fmt.Sprintf("Configs: %v", configFiles))
+				}
+				require.Equal(s.T, expectedConfigs, len(configFiles), fmt.Sprintf("Configs: %v", configFiles))
+		*/
 		s.Cfg = cfg
 	}
 
@@ -337,6 +341,7 @@ func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder {
 	if s.H == nil {
 		s.CreateSites()
 	}
+
 	err := s.H.Build(cfg)
 	if err == nil {
 		logErrorCount := s.H.NumLogErrors()
@@ -436,7 +441,7 @@ func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
 	content := readDestination(s.T, s.Fs, filename)
 	for _, match := range matches {
 		if !strings.Contains(content, match) {
-			s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
+			s.Fatalf("No match for %q in content for %s\n%s", match, filename, content)
 		}
 	}
 }
@@ -509,7 +514,7 @@ func (th testHelper) replaceDefaultContentLanguageValue(value string) string {
 }
 
 func newTestPathSpec(fs *hugofs.Fs, v *viper.Viper) *helpers.PathSpec {
-	l := helpers.NewDefaultLanguage(v)
+	l := langs.NewDefaultLanguage(v)
 	ps, _ := helpers.NewPathSpec(fs, l)
 	return ps
 }
@@ -519,6 +524,10 @@ func newTestDefaultPathSpec() *helpers.PathSpec {
 	// Easier to reason about in tests.
 	v.Set("disablePathToLower", true)
 	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
 	fs := hugofs.NewDefault(v)
 	ps, _ := helpers.NewPathSpec(fs, v)
 	return ps
@@ -551,7 +560,7 @@ func newTestSite(t testing.TB, configKeyValues ...interface{}) *Site {
 		cfg.Set(configKeyValues[i].(string), configKeyValues[i+1])
 	}
 
-	d := deps.DepsCfg{Language: helpers.NewLanguage("en", cfg), Fs: fs, Cfg: cfg}
+	d := deps.DepsCfg{Language: langs.NewLanguage("en", cfg), Fs: fs, Cfg: cfg}
 
 	s, err := NewSiteForCfg(d)
 
@@ -593,18 +602,6 @@ func newTestSitesFromConfigWithDefaultTemplates(t testing.TB, tomlConfig string)
 	)
 }
 
-func newDebugLogger() *jww.Notepad {
-	return jww.NewNotepad(jww.LevelDebug, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
-}
-
-func newErrorLogger() *jww.Notepad {
-	return jww.NewNotepad(jww.LevelError, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
-}
-
-func newWarningLogger() *jww.Notepad {
-	return jww.NewNotepad(jww.LevelWarn, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
-}
-
 func createWithTemplateFromNameValues(additionalTemplates ...string) func(templ tpl.TemplateHandler) error {
 
 	return func(templ tpl.TemplateHandler) error {

i18n/i18n_test.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/gohugoio/hugo/tpl/tplimpl"
 
+	"github.com/gohugoio/hugo/langs"
 	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/deps"
@@ -26,8 +27,6 @@ import (
 	"io/ioutil"
 	"os"
 
-	"github.com/gohugoio/hugo/helpers"
-
 	"log"
 
 	"github.com/gohugoio/hugo/config"
@@ -168,15 +167,16 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin
 	assert := require.New(t)
 	fs := hugofs.NewMem(cfg)
 	tp := NewTranslationProvider()
-	depsCfg := newDepsConfig(tp, cfg, fs)
-	d, err := deps.New(depsCfg)
-	assert.NoError(err)
 
 	for file, content := range test.data {
 		err := afero.WriteFile(fs.Source, filepath.Join("i18n", file), []byte(content), 0755)
 		assert.NoError(err)
 	}
 
+	depsCfg := newDepsConfig(tp, cfg, fs)
+	d, err := deps.New(depsCfg)
+	assert.NoError(err)
+
 	assert.NoError(d.LoadResources())
 	f := tp.t.Func(test.lang)
 	return f(test.id, test.args)
@@ -184,7 +184,7 @@ func doTestI18nTranslate(t *testing.T, test i18nTest, cfg config.Provider) strin
 }
 
 func newDepsConfig(tp *TranslationProvider, cfg config.Provider, fs *hugofs.Fs) deps.DepsCfg {
-	l := helpers.NewLanguage("en", cfg)
+	l := langs.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
 	return deps.DepsCfg{
 		Language:            l,
@@ -201,6 +201,10 @@ func TestI18nTranslate(t *testing.T) {
 	v := viper.New()
 	v.SetDefault("defaultContentLanguage", "en")
 	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
 
 	// Test without and with placeholders
 	for _, enablePlaceholders := range []bool{false, true} {

i18n/translationProvider.go

@@ -38,17 +38,8 @@ func NewTranslationProvider() *TranslationProvider {
 
 // Update updates the i18n func in the provided Deps.
 func (tp *TranslationProvider) Update(d *deps.Deps) error {
-	dir := d.PathSpec.AbsPathify(d.Cfg.GetString("i18nDir"))
+	sp := source.NewSourceSpec(d.PathSpec, d.BaseFs.SourceFilesystems.I18n.Fs)
-	sp := source.NewSourceSpec(d.PathSpec, d.Fs.Source)
+	src := sp.NewFilesystem("")
-	sources := []source.Input{sp.NewFilesystem(dir)}
-
-	themeI18nDir, err := d.PathSpec.GetThemeI18nDirPath()
-
-	if err == nil {
-		sources = []source.Input{sp.NewFilesystem(themeI18nDir), sources[0]}
-	}
-
-	d.Log.DEBUG.Printf("Load I18n from %q", sources)
 
 	i18nBundle := bundle.New()
 
@@ -58,14 +49,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
 	}
 	var newLangs []string
 
-	for _, currentSource := range sources {
+	for _, r := range src.Files() {
-		for _, r := range currentSource.Files() {
+		currentSpec := language.GetPluralSpec(r.BaseFileName())
-			currentSpec := language.GetPluralSpec(r.BaseFileName())
+		if currentSpec == nil {
-			if currentSpec == nil {
+			// This may is a language code not supported by go-i18n, it may be
-				// This may is a language code not supported by go-i18n, it may be
+			// Klingon or ... not even a fake language. Make sure it works.
-				// Klingon or ... not even a fake language. Make sure it works.
+			newLangs = append(newLangs, r.BaseFileName())
-				newLangs = append(newLangs, r.BaseFileName())
-			}
 		}
 	}
 
@@ -73,11 +62,12 @@ func (tp *TranslationProvider) Update(d *deps.Deps) error {
 		language.RegisterPluralSpec(newLangs, en)
 	}
 
-	for _, currentSource := range sources {
+	// The source files are ordered so the most important comes first. Since this is a
-		for _, r := range currentSource.Files() {
+	// last key win situation, we have to reverse the iteration order.
-			if err := addTranslationFile(i18nBundle, r); err != nil {
+	files := src.Files()
-				return err
+	for i := len(files) - 1; i >= 0; i-- {
-			}
+		if err := addTranslationFile(i18nBundle, files[i]); err != nil {
+			return err
 		}
 	}
 

langs/language.go

@@ -1,4 +1,4 @@
-// Copyright 2016-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,12 +11,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package helpers
+package langs
 
 import (
 	"sort"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/config"
 	"github.com/spf13/cast"
 )
@@ -73,7 +74,7 @@ func NewLanguage(lang string, cfg config.Provider) *Language {
 	for k, v := range cfg.GetStringMap("params") {
 		params[k] = v
 	}
-	ToLowerMap(params)
+	maps.ToLower(params)
 
 	defaultContentDir := cfg.GetString("contentDir")
 	if defaultContentDir == "" {

langs/language_test.go

@@ -1,4 +1,4 @@
-// Copyright 2016-present The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package helpers
+package langs
 
 import (
 	"testing"

output/docshelper.go

@@ -68,7 +68,7 @@ func createLayoutExamples() interface{} {
 		{"Taxonomy term in categories", LayoutDescriptor{Kind: "taxonomyTerm", Type: "categories", Section: "category"}, false, HTMLFormat},
 	} {
 
-		l := NewLayoutHandler(example.hasTheme)
+		l := NewLayoutHandler()
 		layouts, _ := l.For(example.d, example.f)
 
 		basicExamples = append(basicExamples, Example{

output/layout.go

@@ -41,8 +41,6 @@ type LayoutDescriptor struct {
 
 // LayoutHandler calculates the layout template to use to render a given output type.
 type LayoutHandler struct {
-	hasTheme bool
-
 	mu    sync.RWMutex
 	cache map[layoutCacheKey][]string
 }
@@ -53,8 +51,8 @@ type layoutCacheKey struct {
 }
 
 // NewLayoutHandler creates a new LayoutHandler.
-func NewLayoutHandler(hasTheme bool) *LayoutHandler {
+func NewLayoutHandler() *LayoutHandler {
-	return &LayoutHandler{hasTheme: hasTheme, cache: make(map[layoutCacheKey][]string)}
+	return &LayoutHandler{cache: make(map[layoutCacheKey][]string)}
 }
 
 // For returns a layout for the given LayoutDescriptor and options.
@@ -72,30 +70,6 @@ func (l *LayoutHandler) For(d LayoutDescriptor, f Format) ([]string, error) {
 
 	layouts := resolvePageTemplate(d, f)
 
-	if l.hasTheme {
-		// From Hugo 0.33 we interleave the project/theme templates. This was kind of a fundamental change, but the
-		// previous behaviour was surprising.
-		// As an example, an `index.html` in theme for the home page will now win over a `_default/list.html` in the project.
-		layoutsWithThemeLayouts := []string{}
-
-		// First place all non internal templates
-		for _, t := range layouts {
-			if !strings.HasPrefix(t, "_internal/") {
-				layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, t)
-				layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, "theme/"+t)
-			}
-		}
-
-		// Lastly place internal templates
-		for _, t := range layouts {
-			if strings.HasPrefix(t, "_internal/") {
-				layoutsWithThemeLayouts = append(layoutsWithThemeLayouts, t)
-			}
-		}
-
-		layouts = layoutsWithThemeLayouts
-	}
-
 	layouts = prependTextPrefixIfNeeded(f, layouts...)
 	layouts = helpers.UniqueStrings(layouts)
 

output/layout_base.go

@@ -40,26 +40,16 @@ type TemplateNames struct {
 }
 
 type TemplateLookupDescriptor struct {
-	// TemplateDir is the project or theme root of the current template.
-	// This will be the same as WorkingDir for non-theme templates.
-	TemplateDir string
-
 	// The full path to the site root.
 	WorkingDir string
 
-	// Main project layout dir, defaults to "layouts"
-	LayoutDir string
-
 	// The path to the template relative the the base.
 	//  I.e. shortcodes/youtube.html
 	RelPath string
 
-	// The template name prefix to look for, i.e. "theme".
+	// The template name prefix to look for.
 	Prefix string
 
-	// The theme dir if theme active.
-	ThemeDir string
-
 	// All the output formats in play. This is used to decide if text/template or
 	// html/template.
 	OutputFormats Formats
@@ -71,6 +61,7 @@ type TemplateLookupDescriptor struct {
 func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 
 	name := filepath.ToSlash(d.RelPath)
+	name = strings.TrimPrefix(name, "/")
 
 	if d.Prefix != "" {
 		name = strings.Trim(d.Prefix, "/") + "/" + name
@@ -78,22 +69,8 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 
 	var (
 		id TemplateNames
-
-		// This is the path to the actual template in process. This may
-		// be in the theme's or the project's /layouts.
-		baseLayoutDir = filepath.Join(d.TemplateDir, d.LayoutDir)
-		fullPath      = filepath.Join(baseLayoutDir, d.RelPath)
-
-		// This is always the project's layout dir.
-		baseWorkLayoutDir = filepath.Join(d.WorkingDir, d.LayoutDir)
-
-		baseThemeLayoutDir string
 	)
 
-	if d.ThemeDir != "" {
-		baseThemeLayoutDir = filepath.Join(d.ThemeDir, "layouts")
-	}
-
 	// The filename will have a suffix with an optional type indicator.
 	// Examples:
 	// index.html
@@ -119,7 +96,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 
 	filenameNoSuffix := parts[0]
 
-	id.OverlayFilename = fullPath
+	id.OverlayFilename = d.RelPath
 	id.Name = name
 
 	if isPlainText {
@@ -127,7 +104,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 	}
 
 	// Ace and Go templates may have both a base and inner template.
-	pathDir := filepath.Dir(fullPath)
+	pathDir := filepath.Dir(d.RelPath)
 
 	if ext == "amber" || strings.HasSuffix(pathDir, "partials") || strings.HasSuffix(pathDir, "shortcodes") {
 		// No base template support
@@ -150,7 +127,7 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 
 	// This may be a view that shouldn't have base template
 	// Have to look inside it to make sure
-	needsBase, err := d.ContainsAny(fullPath, innerMarkers)
+	needsBase, err := d.ContainsAny(d.RelPath, innerMarkers)
 	if err != nil {
 		return id, err
 	}
@@ -158,21 +135,14 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 	if needsBase {
 		currBaseFilename := fmt.Sprintf("%s-%s", filenameNoSuffix, baseFilename)
 
-		templateDir := filepath.Dir(fullPath)
-
-		// Find the base, e.g. "_default".
-		baseTemplatedDir := strings.TrimPrefix(templateDir, baseLayoutDir)
-		baseTemplatedDir = strings.TrimPrefix(baseTemplatedDir, helpers.FilePathSeparator)
-
 		// Look for base template in the follwing order:
 		//   1. <current-path>/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>.
 		//   2. <current-path>/baseof.<outputFormat>(optional).<suffix>
 		//   3. _default/<template-name>-baseof.<outputFormat>(optional).<suffix>, e.g. list-baseof.<outputFormat>(optional).<suffix>.
 		//   4. _default/baseof.<outputFormat>(optional).<suffix>
-		// For each of the steps above, it will first look in the project, then, if theme is set,
+		//
-		// in the theme's layouts folder.
+		// The filesystem it looks in a a composite of the project and potential theme(s).
-		// Also note that the <current-path> may be both the project's layout folder and the theme's.
+		pathsToCheck := createPathsToCheck(pathDir, baseFilename, currBaseFilename)
-		pairsToCheck := createPairsToCheck(baseTemplatedDir, baseFilename, currBaseFilename)
 
 		// We may have language code and/or "terms" in the template name. We want the most specific,
 		// but need to fall back to the baseof.html or baseof.ace if needed.
@@ -183,20 +153,15 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 		if len(p1) > 0 && len(p1) == len(p2) {
 			for i := len(p1); i > 0; i-- {
 				v1, v2 := strings.Join(p1[:i], ".")+"."+ext, strings.Join(p2[:i], ".")+"."+ext
-				pairsToCheck = append(pairsToCheck, createPairsToCheck(baseTemplatedDir, v1, v2)...)
+				pathsToCheck = append(pathsToCheck, createPathsToCheck(pathDir, v1, v2)...)
 
 			}
 		}
 
-	Loop:
+		for _, p := range pathsToCheck {
-		for _, pair := range pairsToCheck {
+			if ok, err := d.FileExists(p); err == nil && ok {
-			pathsToCheck := basePathsToCheck(pair, baseLayoutDir, baseWorkLayoutDir, baseThemeLayoutDir)
+				id.MasterFilename = p
-
+				break
-			for _, pathToCheck := range pathsToCheck {
-				if ok, err := d.FileExists(pathToCheck); err == nil && ok {
-					id.MasterFilename = pathToCheck
-					break Loop
-				}
 			}
 		}
 	}
@@ -205,29 +170,11 @@ func CreateTemplateNames(d TemplateLookupDescriptor) (TemplateNames, error) {
 
 }
 
-func createPairsToCheck(baseTemplatedDir, baseFilename, currBaseFilename string) [][]string {
+func createPathsToCheck(baseTemplatedDir, baseFilename, currBaseFilename string) []string {
-	return [][]string{
+	return []string{
-		{baseTemplatedDir, currBaseFilename},
+		filepath.Join(baseTemplatedDir, currBaseFilename),
-		{baseTemplatedDir, baseFilename},
+		filepath.Join(baseTemplatedDir, baseFilename),
-		{"_default", currBaseFilename},
+		filepath.Join("_default", currBaseFilename),
-		{"_default", baseFilename},
+		filepath.Join("_default", baseFilename),
 	}
 }
-
-func basePathsToCheck(path []string, layoutDir, workLayoutDir, themeLayoutDir string) []string {
-	// workLayoutDir will always be the most specific, so start there.
-	pathsToCheck := []string{filepath.Join((append([]string{workLayoutDir}, path...))...)}
-
-	if layoutDir != "" && layoutDir != workLayoutDir {
-		pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{layoutDir}, path...))...))
-	}
-
-	// May have a theme
-	if themeLayoutDir != "" && themeLayoutDir != layoutDir {
-		pathsToCheck = append(pathsToCheck, filepath.Join((append([]string{themeLayoutDir}, path...))...))
-
-	}
-
-	return pathsToCheck
-
-}

output/layout_base_test.go

@@ -25,8 +25,6 @@ func TestLayoutBase(t *testing.T) {
 
 	var (
 		workingDir     = "/sites/mysite/"
-		themeDir       = "/themes/mytheme/"
-		layoutBase1    = "layouts"
 		layoutPath1    = "_default/single.html"
 		layoutPathAmp  = "_default/single.amp.html"
 		layoutPathJSON = "_default/single.json"
@@ -39,108 +37,72 @@ func TestLayoutBase(t *testing.T) {
 		basePathMatchStrings string
 		expect               TemplateNames
 	}{
-		{"No base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, false, "",
+		{"No base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPath1}, false, "",
 			TemplateNames{
 				Name:            "_default/single.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+				OverlayFilename: "_default/single.html",
 			}},
-		{"Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1}, true, "",
+		{"Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPath1}, true, "",
 			TemplateNames{
 				Name:            "_default/single.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
+				OverlayFilename: "_default/single.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.html",
+				MasterFilename:  "_default/single-baseof.html",
 			}},
 		// Issue #3893
-		{"Base Lang, Default Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: "layouts", RelPath: "_default/list.en.html"}, true, "_default/baseof.html",
+		{"Base Lang, Default Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "_default/list.en.html"}, true, "_default/baseof.html",
 			TemplateNames{
 				Name:            "_default/list.en.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/list.en.html",
+				OverlayFilename: "_default/list.en.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/baseof.html",
+				MasterFilename:  "_default/baseof.html",
 			}},
-		{"Base Lang, Lang Base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: "layouts", RelPath: "_default/list.en.html"}, true, "_default/baseof.html|_default/baseof.en.html",
+		{"Base Lang, Lang Base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "_default/list.en.html"}, true, "_default/baseof.html|_default/baseof.en.html",
 			TemplateNames{
 				Name:            "_default/list.en.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/list.en.html",
+				OverlayFilename: "_default/list.en.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/baseof.en.html",
+				MasterFilename:  "_default/baseof.en.html",
 			}},
 		// Issue #3856
-		{"Base Taxonomy Term", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "taxonomy/tag.terms.html"}, true, "_default/baseof.html",
+		{"Base Taxonomy Term", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "taxonomy/tag.terms.html"}, true, "_default/baseof.html",
 			TemplateNames{
 				Name:            "taxonomy/tag.terms.html",
-				OverlayFilename: "/sites/mysite/layouts/taxonomy/tag.terms.html",
+				OverlayFilename: "taxonomy/tag.terms.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/baseof.html",
+				MasterFilename:  "_default/baseof.html",
 			}},
 
-		{"Base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true,
+		{"Partial", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: "partials/menu.html"}, true,
-			"mytheme/layouts/_default/baseof.html",
-			TemplateNames{
-				Name:            "_default/single.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
-				MasterFilename:  "/themes/mytheme/layouts/_default/baseof.html",
-			}},
-		{"Template in theme, base in theme", TemplateLookupDescriptor{TemplateDir: themeDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true,
-			"mytheme/layouts/_default/baseof.html",
-			TemplateNames{
-				Name:            "_default/single.html",
-				OverlayFilename: "/themes/mytheme/layouts/_default/single.html",
-				MasterFilename:  "/themes/mytheme/layouts/_default/baseof.html",
-			}},
-		{"Template in theme, base in site", TemplateLookupDescriptor{TemplateDir: themeDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true,
-			"/sites/mysite/layouts/_default/baseof.html",
-			TemplateNames{
-				Name:            "_default/single.html",
-				OverlayFilename: "/themes/mytheme/layouts/_default/single.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/baseof.html",
-			}},
-		{"Template in site, base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1, ThemeDir: themeDir}, true,
-			"/themes/mytheme",
-			TemplateNames{
-				Name:            "_default/single.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
-				MasterFilename:  "/themes/mytheme/layouts/_default/single-baseof.html",
-			}},
-		{"With prefix, base in theme", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPath1,
-			ThemeDir: themeDir, Prefix: "someprefix"}, true,
-			"mytheme/layouts/_default/baseof.html",
-			TemplateNames{
-				Name:            "someprefix/_default/single.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.html",
-				MasterFilename:  "/themes/mytheme/layouts/_default/baseof.html",
-			}},
-		{"Partial", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: "partials/menu.html"}, true,
 			"mytheme/layouts/_default/baseof.html",
 			TemplateNames{
 				Name:            "partials/menu.html",
-				OverlayFilename: "/sites/mysite/layouts/partials/menu.html",
+				OverlayFilename: "partials/menu.html",
 			}},
-		{"AMP, no base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, false, "",
+		{"AMP, no base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, false, "",
 			TemplateNames{
 				Name:            "_default/single.amp.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html",
+				OverlayFilename: "_default/single.amp.html",
 			}},
-		{"JSON, no base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, false, "",
+		{"JSON, no base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathJSON}, false, "",
 			TemplateNames{
 				Name:            "_default/single.json",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.json",
+				OverlayFilename: "_default/single.json",
 			}},
-		{"AMP with base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html",
+		{"AMP with base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, true, "single-baseof.html|single-baseof.amp.html",
 			TemplateNames{
 				Name:            "_default/single.amp.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html",
+				OverlayFilename: "_default/single.amp.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.amp.html",
+				MasterFilename:  "_default/single-baseof.amp.html",
 			}},
-		{"AMP with no AMP base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathAmp}, true, "single-baseof.html",
+		{"AMP with no AMP base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathAmp}, true, "single-baseof.html",
 			TemplateNames{
 				Name:            "_default/single.amp.html",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.amp.html",
+				OverlayFilename: "_default/single.amp.html",
-				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.html",
+				MasterFilename:  "_default/single-baseof.html",
 			}},
 
-		{"JSON with base", TemplateLookupDescriptor{TemplateDir: workingDir, WorkingDir: workingDir, LayoutDir: layoutBase1, RelPath: layoutPathJSON}, true, "single-baseof.json",
+		{"JSON with base", TemplateLookupDescriptor{WorkingDir: workingDir, RelPath: layoutPathJSON}, true, "single-baseof.json",
 			TemplateNames{
 				Name:            "_default/single.json",
-				OverlayFilename: "/sites/mysite/layouts/_default/single.json",
+				OverlayFilename: "_default/single.json",
-				MasterFilename:  "/sites/mysite/layouts/_default/single-baseof.json",
+				MasterFilename:  "_default/single-baseof.json",
 			}},
 	} {
 		t.Run(this.name, func(t *testing.T) {
@@ -164,7 +126,6 @@ func TestLayoutBase(t *testing.T) {
 
 			this.d.OutputFormats = Formats{AMPFormat, HTMLFormat, RSSFormat, JSONFormat}
 			this.d.WorkingDir = filepath.FromSlash(this.d.WorkingDir)
-			this.d.LayoutDir = filepath.FromSlash(this.d.LayoutDir)
 			this.d.RelPath = filepath.FromSlash(this.d.RelPath)
 			this.d.ContainsAny = needsBase
 			this.d.FileExists = fileExists

output/layout_test.go

@@ -57,62 +57,61 @@ func TestLayout(t *testing.T) {
 	for _, this := range []struct {
 		name           string
 		d              LayoutDescriptor
-		hasTheme       bool
 		layoutOverride string
 		tp             Format
 		expect         []string
 		expectCount    int
 	}{
-		{"Home", LayoutDescriptor{Kind: "home"}, true, "", ampType,
+		{"Home", LayoutDescriptor{Kind: "home"}, "", ampType,
-			[]string{"index.amp.html", "theme/index.amp.html", "home.amp.html", "theme/home.amp.html", "list.amp.html", "theme/list.amp.html", "index.html", "theme/index.html", "home.html", "theme/home.html", "list.html", "theme/list.html", "_default/index.amp.html"}, 24},
+			[]string{"index.amp.html", "home.amp.html", "list.amp.html", "index.html", "home.html", "list.html", "_default/index.amp.html"}, 12},
-		{"Home, HTML", LayoutDescriptor{Kind: "home"}, true, "", htmlFormat,
+		{"Home, HTML", LayoutDescriptor{Kind: "home"}, "", htmlFormat,
 			// We will eventually get to index.html. This looks stuttery, but makes the lookup logic easy to understand.
-			[]string{"index.html.html", "theme/index.html.html", "home.html.html"}, 24},
+			[]string{"index.html.html", "home.html.html"}, 12},
-		{"Home, french language", LayoutDescriptor{Kind: "home", Lang: "fr"}, true, "", ampType,
+		{"Home, french language", LayoutDescriptor{Kind: "home", Lang: "fr"}, "", ampType,
-			[]string{"index.fr.amp.html", "theme/index.fr.amp.html"},
+			[]string{"index.fr.amp.html"},
-			48},
+			24},
-		{"Home, no ext or delim", LayoutDescriptor{Kind: "home"}, true, "", noExtDelimFormat,
+		{"Home, no ext or delim", LayoutDescriptor{Kind: "home"}, "", noExtDelimFormat,
-			[]string{"index.nem", "theme/index.nem", "home.nem", "theme/home.nem", "list.nem"}, 12},
+			[]string{"index.nem", "home.nem", "list.nem"}, 6},
-		{"Home, no ext", LayoutDescriptor{Kind: "home"}, true, "", noExt,
+		{"Home, no ext", LayoutDescriptor{Kind: "home"}, "", noExt,
-			[]string{"index.nex", "theme/index.nex", "home.nex", "theme/home.nex", "list.nex"}, 12},
+			[]string{"index.nex", "home.nex", "list.nex"}, 6},
-		{"Page, no ext or delim", LayoutDescriptor{Kind: "page"}, true, "", noExtDelimFormat,
+		{"Page, no ext or delim", LayoutDescriptor{Kind: "page"}, "", noExtDelimFormat,
-			[]string{"_default/single.nem", "theme/_default/single.nem"}, 2},
+			[]string{"_default/single.nem"}, 1},
-		{"Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, false, "", ampType,
+		{"Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, "", ampType,
 			[]string{"sect1/sect1.amp.html", "sect1/section.amp.html", "sect1/list.amp.html", "sect1/sect1.html", "sect1/section.html", "sect1/list.html", "section/sect1.amp.html", "section/section.amp.html"}, 18},
-		{"Section with layout", LayoutDescriptor{Kind: "section", Section: "sect1", Layout: "mylayout"}, false, "", ampType,
+		{"Section with layout", LayoutDescriptor{Kind: "section", Section: "sect1", Layout: "mylayout"}, "", ampType,
 			[]string{"sect1/mylayout.amp.html", "sect1/sect1.amp.html", "sect1/section.amp.html", "sect1/list.amp.html", "sect1/mylayout.html", "sect1/sect1.html"}, 24},
-		{"Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, false, "", ampType,
+		{"Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, "", ampType,
 			[]string{"taxonomy/tag.amp.html", "taxonomy/taxonomy.amp.html", "taxonomy/list.amp.html", "taxonomy/tag.html", "taxonomy/taxonomy.html"}, 18},
-		{"Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}, false, "", ampType,
+		{"Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}, "", ampType,
 			[]string{"taxonomy/categories.terms.amp.html", "taxonomy/terms.amp.html", "taxonomy/list.amp.html", "taxonomy/categories.terms.html", "taxonomy/terms.html"}, 18},
-		{"Page", LayoutDescriptor{Kind: "page"}, true, "", ampType,
+		{"Page", LayoutDescriptor{Kind: "page"}, "", ampType,
-			[]string{"_default/single.amp.html", "theme/_default/single.amp.html", "_default/single.html", "theme/_default/single.html"}, 4},
+			[]string{"_default/single.amp.html", "_default/single.html"}, 2},
-		{"Page with layout", LayoutDescriptor{Kind: "page", Layout: "mylayout"}, false, "", ampType,
+		{"Page with layout", LayoutDescriptor{Kind: "page", Layout: "mylayout"}, "", ampType,
 			[]string{"_default/mylayout.amp.html", "_default/single.amp.html", "_default/mylayout.html", "_default/single.html"}, 4},
-		{"Page with layout and type", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype"}, false, "", ampType,
+		{"Page with layout and type", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype"}, "", ampType,
 			[]string{"myttype/mylayout.amp.html", "myttype/single.amp.html", "myttype/mylayout.html"}, 8},
-		{"Page with layout and type with subtype", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype/mysubtype"}, false, "", ampType,
+		{"Page with layout and type with subtype", LayoutDescriptor{Kind: "page", Layout: "mylayout", Type: "myttype/mysubtype"}, "", ampType,
 			[]string{"myttype/mysubtype/mylayout.amp.html", "myttype/mysubtype/single.amp.html", "myttype/mysubtype/mylayout.html"}, 8},
 		// RSS
-		{"RSS Home with theme", LayoutDescriptor{Kind: "home"}, true, "", RSSFormat,
+		{"RSS Home", LayoutDescriptor{Kind: "home"}, "", RSSFormat,
-			[]string{"index.rss.xml", "theme/index.rss.xml", "home.rss.xml", "theme/home.rss.xml", "rss.xml"}, 29},
+			[]string{"index.rss.xml", "home.rss.xml", "rss.xml"}, 15},
-		{"RSS Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, false, "", RSSFormat,
+		{"RSS Section", LayoutDescriptor{Kind: "section", Section: "sect1"}, "", RSSFormat,
 			[]string{"sect1/sect1.rss.xml", "sect1/section.rss.xml", "sect1/rss.xml", "sect1/list.rss.xml", "sect1/sect1.xml", "sect1/section.xml"}, 22},
-		{"RSS Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, false, "", RSSFormat,
+		{"RSS Taxonomy", LayoutDescriptor{Kind: "taxonomy", Section: "tag"}, "", RSSFormat,
 			[]string{"taxonomy/tag.rss.xml", "taxonomy/taxonomy.rss.xml", "taxonomy/rss.xml", "taxonomy/list.rss.xml", "taxonomy/tag.xml", "taxonomy/taxonomy.xml"}, 22},
-		{"RSS Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "tag"}, false, "", RSSFormat,
+		{"RSS Taxonomy term", LayoutDescriptor{Kind: "taxonomyTerm", Section: "tag"}, "", RSSFormat,
 			[]string{"taxonomy/tag.terms.rss.xml", "taxonomy/terms.rss.xml", "taxonomy/rss.xml", "taxonomy/list.rss.xml", "taxonomy/tag.terms.xml"}, 22},
-		{"Home plain text", LayoutDescriptor{Kind: "home"}, true, "", JSONFormat,
+		{"Home plain text", LayoutDescriptor{Kind: "home"}, "", JSONFormat,
-			[]string{"_text/index.json.json", "_text/theme/index.json.json", "_text/home.json.json", "_text/theme/home.json.json"}, 24},
+			[]string{"_text/index.json.json", "_text/home.json.json"}, 12},
-		{"Page plain text", LayoutDescriptor{Kind: "page"}, true, "", JSONFormat,
+		{"Page plain text", LayoutDescriptor{Kind: "page"}, "", JSONFormat,
-			[]string{"_text/_default/single.json.json", "_text/theme/_default/single.json.json", "_text/_default/single.json", "_text/theme/_default/single.json"}, 4},
+			[]string{"_text/_default/single.json.json", "_text/_default/single.json"}, 2},
-		{"Reserved section, shortcodes", LayoutDescriptor{Kind: "section", Section: "shortcodes", Type: "shortcodes"}, true, "", ampType,
+		{"Reserved section, shortcodes", LayoutDescriptor{Kind: "section", Section: "shortcodes", Type: "shortcodes"}, "", ampType,
-			[]string{"section/shortcodes.amp.html", "theme/section/shortcodes.amp.html"}, 24},
+			[]string{"section/shortcodes.amp.html"}, 12},
-		{"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, true, "", ampType,
+		{"Reserved section, partials", LayoutDescriptor{Kind: "section", Section: "partials", Type: "partials"}, "", ampType,
-			[]string{"section/partials.amp.html", "theme/section/partials.amp.html"}, 24},
+			[]string{"section/partials.amp.html"}, 12},
 	} {
 		t.Run(this.name, func(t *testing.T) {
-			l := NewLayoutHandler(this.hasTheme)
+			l := NewLayoutHandler()
 
 			layouts, err := l.For(this.d, this.tp)
 
@@ -130,11 +129,6 @@ func TestLayout(t *testing.T) {
 
 			}
 
-			if !this.hasTheme {
-				for _, layout := range layouts {
-					require.NotContains(t, layout, "theme")
-				}
-			}
 		})
 	}
 
@@ -142,7 +136,7 @@ func TestLayout(t *testing.T) {
 
 func BenchmarkLayout(b *testing.B) {
 	descriptor := LayoutDescriptor{Kind: "taxonomyTerm", Section: "categories"}
-	l := NewLayoutHandler(false)
+	l := NewLayoutHandler()
 
 	for i := 0; i < b.N; i++ {
 		layouts, err := l.For(descriptor, HTMLFormat)

resource/resource.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gohugoio/hugo/common/maps"
+
 	"github.com/spf13/afero"
 
 	"github.com/spf13/cast"
@@ -282,7 +284,6 @@ func NewSpec(s *helpers.PathSpec, mimeTypes media.Types) (*Spec, error) {
 	if err != nil {
 		return nil, err
 	}
-	s.GetLayoutDirPath()
 
 	genImagePath := filepath.FromSlash("_gen/images")
 
@@ -644,7 +645,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...Resource) er
 				if found {
 					m := cast.ToStringMap(params)
 					// Needed for case insensitive fetching of params values
-					helpers.ToLowerMap(m)
+					maps.ToLower(m)
 					ma.updateParams(m)
 				}
 			}

resource/testhelpers_test.go

@@ -30,6 +30,10 @@ func newTestResourceSpecForBaseURL(assert *require.Assertions, baseURL string) *
 	cfg.Set("baseURL", baseURL)
 	cfg.Set("resourceDir", "resources")
 	cfg.Set("contentDir", "content")
+	cfg.Set("dataDir", "data")
+	cfg.Set("i18nDir", "i18n")
+	cfg.Set("layoutDir", "layouts")
+	cfg.Set("archetypeDir", "archetypes")
 
 	imagingCfg := map[string]interface{}{
 		"resampleFilter": "linear",
@@ -63,8 +67,12 @@ func newTestResourceOsFs(assert *require.Assertions) *Spec {
 	}
 
 	cfg.Set("workingDir", workDir)
-	cfg.Set("contentDir", filepath.Join(workDir, "content"))
 	cfg.Set("resourceDir", filepath.Join(workDir, "res"))
+	cfg.Set("contentDir", "content")
+	cfg.Set("dataDir", "data")
+	cfg.Set("i18nDir", "i18n")
+	cfg.Set("layoutDir", "layouts")
+	cfg.Set("archetypeDir", "archetypes")
 
 	fs := hugofs.NewFrom(hugofs.Os, cfg)
 	fs.Destination = &afero.MemMapFs{}

source/content_directory_test.go

@@ -20,7 +20,6 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
 
@@ -52,9 +51,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) {
 	}
 
 	for i, test := range tests {
-
+		v := newTestConfig()
-		v := viper.New()
-		v.Set("contentDir", "content")
 		v.Set("ignoreFiles", test.ignoreFilesRegexpes)
 		fs := hugofs.NewMem(v)
 		ps, err := helpers.NewPathSpec(fs, v)

source/dirs.go

@@ -1,194 +0,0 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package source
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/spf13/afero"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/gohugoio/hugo/helpers"
-	"github.com/gohugoio/hugo/hugofs"
-	jww "github.com/spf13/jwalterweatherman"
-)
-
-// Dirs holds the source directories for a given build.
-// In case where there are more than one of a kind, the order matters:
-// It will be used to construct a union filesystem, so the right-most directory
-// will "win" on duplicates. Typically, the theme version will be the first.
-type Dirs struct {
-	logger   *jww.Notepad
-	pathSpec *helpers.PathSpec
-
-	staticDirs    []string
-	AbsStaticDirs []string
-
-	Language *helpers.Language
-}
-
-// NewDirs creates a new dirs with the given configuration and filesystem.
-func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) {
-	ps, err := helpers.NewPathSpec(fs, cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	var l *helpers.Language
-	if language, ok := cfg.(*helpers.Language); ok {
-		l = language
-	}
-
-	d := &Dirs{Language: l, pathSpec: ps, logger: logger}
-
-	return d, d.init(cfg)
-
-}
-
-func (d *Dirs) init(cfg config.Provider) error {
-
-	var (
-		statics []string
-	)
-
-	if d.pathSpec.Theme() != "" {
-		statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static"))
-	}
-
-	_, isLanguage := cfg.(*helpers.Language)
-	languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages)
-
-	if !isLanguage && !hasLanguages {
-		return errors.New("missing languagesSorted in config")
-	}
-
-	if !isLanguage {
-		// Merge all the static dirs.
-		for _, l := range languages {
-			addend, err := d.staticDirsFor(l)
-			if err != nil {
-				return err
-			}
-
-			statics = append(statics, addend...)
-		}
-	} else {
-		addend, err := d.staticDirsFor(cfg)
-		if err != nil {
-			return err
-		}
-
-		statics = append(statics, addend...)
-	}
-
-	d.staticDirs = removeDuplicatesKeepRight(statics)
-	d.AbsStaticDirs = make([]string, len(d.staticDirs))
-	for i, di := range d.staticDirs {
-		d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator
-	}
-
-	return nil
-}
-
-func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) {
-	var statics []string
-	ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg)
-	if err != nil {
-		return statics, err
-	}
-
-	statics = append(statics, ps.StaticDirs()...)
-
-	return statics, nil
-}
-
-// CreateStaticFs will create a union filesystem with the static paths configured.
-// Any missing directories will be logged as warnings.
-func (d *Dirs) CreateStaticFs() (afero.Fs, error) {
-	var (
-		source   = d.pathSpec.Fs.Source
-		absPaths []string
-	)
-
-	for _, staticDir := range d.AbsStaticDirs {
-		if _, err := source.Stat(staticDir); os.IsNotExist(err) {
-			d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir)
-		} else {
-			absPaths = append(absPaths, staticDir)
-		}
-
-	}
-
-	if len(absPaths) == 0 {
-		return nil, nil
-	}
-
-	return d.createOverlayFs(absPaths), nil
-
-}
-
-// IsStatic returns whether the given filename is located in one of the static
-// source dirs.
-func (d *Dirs) IsStatic(filename string) bool {
-	for _, absPath := range d.AbsStaticDirs {
-		if strings.HasPrefix(filename, absPath) {
-			return true
-		}
-	}
-	return false
-}
-
-// MakeStaticPathRelative creates a relative path from the given filename.
-// It will return an empty string if the filename is not a member of dirs.
-func (d *Dirs) MakeStaticPathRelative(filename string) string {
-	for _, currentPath := range d.AbsStaticDirs {
-		if strings.HasPrefix(filename, currentPath) {
-			return strings.TrimPrefix(filename, currentPath)
-		}
-	}
-
-	return ""
-
-}
-
-func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs {
-	source := d.pathSpec.Fs.Source
-
-	if len(absPaths) == 1 {
-		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
-	}
-
-	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
-	overlay := d.createOverlayFs(absPaths[1:])
-
-	return afero.NewCopyOnWriteFs(base, overlay)
-}
-
-func removeDuplicatesKeepRight(in []string) []string {
-	seen := make(map[string]bool)
-	var out []string
-	for i := len(in) - 1; i >= 0; i-- {
-		v := in[i]
-		if seen[v] {
-			continue
-		}
-		out = append([]string{v}, out...)
-		seen[v] = true
-	}
-
-	return out
-}

source/dirs_test.go

@@ -1,185 +0,0 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package source
-
-import (
-	"testing"
-
-	"github.com/gohugoio/hugo/helpers"
-
-	"fmt"
-
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-
-	"github.com/gohugoio/hugo/config"
-	"github.com/spf13/afero"
-
-	jww "github.com/spf13/jwalterweatherman"
-
-	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/viper"
-	"github.com/stretchr/testify/require"
-)
-
-var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
-
-func TestStaticDirs(t *testing.T) {
-	assert := require.New(t)
-
-	tests := []struct {
-		setup    func(cfg config.Provider, fs *hugofs.Fs) config.Provider
-		expected []string
-	}{
-
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-			return cfg
-		}, []string{"s1"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", []string{"s2", "s1", "s2"})
-			return cfg
-		}, []string{"s1", "s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("theme", "mytheme")
-			cfg.Set("themesDir", "themes")
-			cfg.Set("staticDir", []string{"s1", "s2"})
-			return cfg
-		}, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir", []string{"l1s1", "l1s2"})
-			return l1
-
-		}, []string{"l1s1", "l1s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
-			return l1
-
-		}, []string{"s1", "l1s1", "l1s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", []string{"s1", "s2"})
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
-			return l1
-
-		}, []string{"s1", "s2", "l1s1", "l1s2"}},
-		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
-			cfg.Set("staticDir", "s1")
-
-			l1 := helpers.NewLanguage("en", cfg)
-			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
-			l2 := helpers.NewLanguage("nn", cfg)
-			l2.Set("staticDir3", []string{"l2s1", "l2s2"})
-			l2.Set("staticDir", []string{"l2"})
-
-			cfg.Set("languagesSorted", helpers.Languages{l1, l2})
-			return cfg
-
-		}, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}},
-	}
-
-	for i, test := range tests {
-		msg := fmt.Sprintf("Test %d", i)
-		v := viper.New()
-		v.Set("contentDir", "content")
-
-		fs := hugofs.NewMem(v)
-		cfg := test.setup(v, fs)
-		cfg.Set("workingDir", filepath.FromSlash("/work"))
-		_, isLanguage := cfg.(*helpers.Language)
-		if !isLanguage && !cfg.IsSet("languagesSorted") {
-			cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)})
-		}
-		dirs, err := NewDirs(fs, cfg, logger)
-		assert.NoError(err)
-		assert.Equal(test.expected, dirs.staticDirs, msg)
-		assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs))
-
-		for i, d := range dirs.staticDirs {
-			abs := dirs.AbsStaticDirs[i]
-			assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs)
-			assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png")))
-			rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png"))
-			assert.Equal("logo.png", rel)
-		}
-
-		assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png")))
-
-	}
-
-}
-
-func TestStaticDirsFs(t *testing.T) {
-	assert := require.New(t)
-	v := viper.New()
-	fs := hugofs.NewMem(v)
-	v.Set("workingDir", filepath.FromSlash("/work"))
-	v.Set("theme", "mytheme")
-	v.Set("themesDir", "themes")
-	v.Set("contentDir", "content")
-	v.Set("staticDir", []string{"s1", "s2"})
-	v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
-
-	writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1")
-	writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2")
-	writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2")
-	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1")
-	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3")
-
-	dirs, err := NewDirs(fs, v, logger)
-	assert.NoError(err)
-
-	sfs, err := dirs.CreateStaticFs()
-	assert.NoError(err)
-
-	assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt"))
-	assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt"))
-	assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt"))
-
-}
-
-func TestRemoveDuplicatesKeepRight(t *testing.T) {
-	in := []string{"a", "b", "c", "a"}
-	out := removeDuplicatesKeepRight(in)
-
-	require.Equal(t, []string{"b", "c", "a"}, out)
-}
-
-func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
-	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {
-		t.Fatalf("Failed to write file: %s", err)
-	}
-}
-
-func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
-	filename = filepath.FromSlash(filename)
-	b, err := afero.ReadFile(fs, filename)
-	if err != nil {
-		afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
-			fmt.Println("    ", path, " ", info)
-			return nil
-		})
-		t.Fatalf("Failed to read file: %s", err)
-	}
-	return string(b)
-}

source/fileInfo.go

@@ -220,7 +220,7 @@ func (sp *SourceSpec) NewFileInfo(baseDir, filename string, isLeafBundle bool, f
 
 // Open implements ReadableFile.
 func (fi *FileInfo) Open() (io.ReadCloser, error) {
-	f, err := fi.sp.PathSpec.Fs.Source.Open(fi.Filename())
+	f, err := fi.sp.SourceFs.Open(fi.Filename())
 	return f, err
 }
 

source/fileInfo_test.go

@@ -19,8 +19,6 @@ import (
 
 	"github.com/gohugoio/hugo/helpers"
 
-	"github.com/spf13/viper"
-
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/require"
@@ -72,14 +70,13 @@ func TestFileInfoLanguage(t *testing.T) {
 
 	m := afero.NewMemMapFs()
 	lfs := hugofs.NewLanguageFs("sv", langs, m)
-	v := viper.New()
+	v := newTestConfig()
-	v.Set("contentDir", "content")
 
 	fs := hugofs.NewFrom(m, v)
 
 	ps, err := helpers.NewPathSpec(fs, v)
 	assert.NoError(err)
-	s := SourceSpec{Fs: lfs, PathSpec: ps}
+	s := SourceSpec{SourceFs: lfs, PathSpec: ps}
 	s.Languages = map[string]interface{}{
 		"en": true,
 	}

source/filesystem.go

@@ -79,16 +79,13 @@ func (f *Filesystem) captureFiles() {
 		return err
 	}
 
-	if f.Fs == nil {
+	if f.SourceFs == nil {
 		panic("Must have a fs")
 	}
-	err := helpers.SymbolicWalk(f.Fs, f.Base, walker)
+	err := helpers.SymbolicWalk(f.SourceFs, f.Base, walker)
 
 	if err != nil {
 		jww.ERROR.Println(err)
-		if err == helpers.ErrPathTooShort {
-			panic("The root path is too short. If this is a test, make sure to init the content paths.")
-		}
 	}
 
 }
@@ -100,7 +97,7 @@ func (f *Filesystem) shouldRead(filename string, fi os.FileInfo) (bool, error) {
 			jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filename, err)
 			return false, nil
 		}
-		linkfi, err := f.Fs.Stat(link)
+		linkfi, err := f.SourceFs.Stat(link)
 		if err != nil {
 			jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
 			return false, nil

source/filesystem_test.go

@@ -19,7 +19,6 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/helpers"
-
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/viper"
@@ -69,9 +68,19 @@ func TestUnicodeNorm(t *testing.T) {
 
 }
 
-func newTestSourceSpec() SourceSpec {
+func newTestConfig() *viper.Viper {
 	v := viper.New()
 	v.Set("contentDir", "content")
-	ps, _ := helpers.NewPathSpec(hugofs.NewMem(v), v)
+	v.Set("dataDir", "data")
-	return SourceSpec{Fs: hugofs.NewMem(v).Source, PathSpec: ps}
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
+	return v
+}
+
+func newTestSourceSpec() *SourceSpec {
+	v := newTestConfig()
+	fs := hugofs.NewMem(v)
+	ps, _ := helpers.NewPathSpec(fs, v)
+	return NewSourceSpec(ps, fs.Source)
 }

source/sourceSpec.go

@@ -18,6 +18,7 @@ import (
 	"path/filepath"
 	"regexp"
 
+	"github.com/gohugoio/hugo/langs"
 	"github.com/spf13/afero"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -29,7 +30,7 @@ import (
 type SourceSpec struct {
 	*helpers.PathSpec
 
-	Fs afero.Fs
+	SourceFs afero.Fs
 
 	// This is set if the ignoreFiles config is set.
 	ignoreFilesRe []*regexp.Regexp
@@ -52,7 +53,7 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
 	}
 
 	if len(languages) == 0 {
-		l := helpers.NewDefaultLanguage(cfg)
+		l := langs.NewDefaultLanguage(cfg)
 		languages[l.Lang] = l
 		defaultLang = l.Lang
 	}
@@ -71,12 +72,13 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec {
 		}
 	}
 
-	return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, Fs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+	return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet}
+
 }
 
 func (s *SourceSpec) IgnoreFile(filename string) bool {
 	if filename == "" {
-		if _, ok := s.Fs.(*afero.OsFs); ok {
+		if _, ok := s.SourceFs.(*afero.OsFs); ok {
 			return true
 		}
 		return false
@@ -108,7 +110,7 @@ func (s *SourceSpec) IgnoreFile(filename string) bool {
 }
 
 func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
-	fi, err := helpers.LstatIfPossible(s.Fs, filename)
+	fi, err := helpers.LstatIfPossible(s.SourceFs, filename)
 	if err != nil {
 		return false, err
 	}
@@ -119,7 +121,7 @@ func (s *SourceSpec) IsRegularSourceFile(filename string) (bool, error) {
 
 	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 		link, err := filepath.EvalSymlinks(filename)
-		fi, err = helpers.LstatIfPossible(s.Fs, link)
+		fi, err = helpers.LstatIfPossible(s.SourceFs, link)
 		if err != nil {
 			return false, err
 		}

tpl/collections/collections_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/langs"
 	jww "github.com/spf13/jwalterweatherman"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
@@ -774,7 +775,7 @@ type TstX struct {
 }
 
 func newDeps(cfg config.Provider) *deps.Deps {
-	l := helpers.NewLanguage("en", cfg)
+	l := langs.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
 	cs, err := helpers.NewContentSpec(l)
 	if err != nil {

tpl/data/resources_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/spf13/afero"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
@@ -164,7 +165,7 @@ func TestScpGetRemoteParallel(t *testing.T) {
 }
 
 func newDeps(cfg config.Provider) *deps.Deps {
-	l := helpers.NewLanguage("en", cfg)
+	l := langs.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
 	cs, err := helpers.NewContentSpec(l)
 	if err != nil {

tpl/template.go

@@ -35,7 +35,7 @@ type TemplateHandler interface {
 	TemplateFinder
 	AddTemplate(name, tpl string) error
 	AddLateTemplate(name, tpl string) error
-	LoadTemplates(absPath, prefix string)
+	LoadTemplates(prefix string)
 	PrintErrors()
 
 	MarkReady()

tpl/tplimpl/template.go

@@ -86,6 +86,10 @@ type templateHandler struct {
 
 	errors []*templateErr
 
+	// This is the filesystem to load the templates from. All the templates are
+	// stored in the root of this filesystem.
+	layoutsFs afero.Fs
+
 	*deps.Deps
 }
 
@@ -129,10 +133,11 @@ func (t *templateHandler) Lookup(name string) *tpl.TemplateAdapter {
 
 func (t *templateHandler) clone(d *deps.Deps) *templateHandler {
 	c := &templateHandler{
-		Deps:   d,
+		Deps:      d,
-		html:   &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)},
+		layoutsFs: d.BaseFs.Layouts.Fs,
-		text:   &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)},
+		html:      &htmlTemplates{t: template.Must(t.html.t.Clone()), overlays: make(map[string]*template.Template)},
-		errors: make([]*templateErr, 0),
+		text:      &textTemplates{t: texttemplate.Must(t.text.t.Clone()), overlays: make(map[string]*texttemplate.Template)},
+		errors:    make([]*templateErr, 0),
 	}
 
 	d.Tmpl = c
@@ -170,10 +175,11 @@ func newTemplateAdapter(deps *deps.Deps) *templateHandler {
 		overlays: make(map[string]*texttemplate.Template),
 	}
 	return &templateHandler{
-		Deps:   deps,
+		Deps:      deps,
-		html:   htmlT,
+		layoutsFs: deps.BaseFs.Layouts.Fs,
-		text:   textT,
+		html:      htmlT,
-		errors: make([]*templateErr, 0),
+		text:      textT,
+		errors:    make([]*templateErr, 0),
 	}
 
 }
@@ -208,15 +214,18 @@ func (t *htmlTemplates) Lookup(name string) *tpl.TemplateAdapter {
 }
 
 func (t *htmlTemplates) lookup(name string) *template.Template {
-	if templ := t.t.Lookup(name); templ != nil {
+
-		return templ
+	// Need to check in the overlay registry first as it will also be found below.
-	}
 	if t.overlays != nil {
 		if templ, ok := t.overlays[name]; ok {
 			return templ
 		}
 	}
 
+	if templ := t.t.Lookup(name); templ != nil {
+		return templ
+	}
+
 	if t.clone != nil {
 		return t.clone.Lookup(name)
 	}
@@ -248,15 +257,18 @@ func (t *textTemplates) Lookup(name string) *tpl.TemplateAdapter {
 }
 
 func (t *textTemplates) lookup(name string) *texttemplate.Template {
-	if templ := t.t.Lookup(name); templ != nil {
+
-		return templ
+	// Need to check in the overlay registry first as it will also be found below.
-	}
 	if t.overlays != nil {
 		if templ, ok := t.overlays[name]; ok {
 			return templ
 		}
 	}
 
+	if templ := t.t.Lookup(name); templ != nil {
+		return templ
+	}
+
 	if t.clone != nil {
 		return t.clone.Lookup(name)
 	}
@@ -287,11 +299,11 @@ func (t *textTemplates) setFuncs(funcMap map[string]interface{}) {
 	t.t.Funcs(funcMap)
 }
 
-// LoadTemplates loads the templates, starting from the given absolute path.
+// LoadTemplates loads the templates from the layouts filesystem.
 // A prefix can be given to indicate a template namespace to load the templates
 // into, i.e. "_internal" etc.
-func (t *templateHandler) LoadTemplates(absPath, prefix string) {
+func (t *templateHandler) LoadTemplates(prefix string) {
-	t.loadTemplates(absPath, prefix)
+	t.loadTemplates(prefix)
 
 }
 
@@ -406,85 +418,49 @@ func (t *templateHandler) RebuildClone() {
 	t.text.clone = texttemplate.Must(t.text.cloneClone.Clone())
 }
 
-func (t *templateHandler) loadTemplates(absPath string, prefix string) {
+func (t *templateHandler) loadTemplates(prefix string) {
-	t.Log.DEBUG.Printf("Load templates from path %q prefix %q", absPath, prefix)
 	walker := func(path string, fi os.FileInfo, err error) error {
+		if err != nil || fi.IsDir() {
+			return nil
+		}
+
+		if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) {
+			return nil
+		}
+
+		workingDir := t.PathSpec.WorkingDir
+
+		descriptor := output.TemplateLookupDescriptor{
+			WorkingDir:    workingDir,
+			RelPath:       path,
+			Prefix:        prefix,
+			OutputFormats: t.OutputFormatsConfig,
+			FileExists: func(filename string) (bool, error) {
+				return helpers.Exists(filename, t.Layouts.Fs)
+			},
+			ContainsAny: func(filename string, subslices [][]byte) (bool, error) {
+				return helpers.FileContainsAny(filename, subslices, t.Layouts.Fs)
+			},
+		}
+
+		tplID, err := output.CreateTemplateNames(descriptor)
 		if err != nil {
+			t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err)
+
 			return nil
 		}
 
-		t.Log.DEBUG.Println("Template path", path)
+		if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil {
-		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+			t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err)
-			link, err := filepath.EvalSymlinks(absPath)
-			if err != nil {
-				t.Log.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", absPath, err)
-				return nil
-			}
-
-			linkfi, err := t.Fs.Source.Stat(link)
-			if err != nil {
-				t.Log.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
-				return nil
-			}
-
-			if !linkfi.Mode().IsRegular() {
-				t.Log.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", absPath)
-			}
-			return nil
 		}
 
-		if !fi.IsDir() {
-			if isDotFile(path) || isBackupFile(path) || isBaseTemplate(path) {
-				return nil
-			}
-
-			var (
-				workingDir = t.PathSpec.WorkingDir()
-				themeDir   = t.PathSpec.GetThemeDir()
-				layoutDir  = t.PathSpec.LayoutDir()
-			)
-
-			if themeDir != "" && strings.HasPrefix(absPath, themeDir) {
-				layoutDir = "layouts"
-			}
-
-			li := strings.LastIndex(path, layoutDir) + len(layoutDir) + 1
-			relPath := path[li:]
-			templateDir := path[:li-len(layoutDir)-1]
-
-			descriptor := output.TemplateLookupDescriptor{
-				TemplateDir:   templateDir,
-				WorkingDir:    workingDir,
-				LayoutDir:     layoutDir,
-				RelPath:       relPath,
-				Prefix:        prefix,
-				ThemeDir:      themeDir,
-				OutputFormats: t.OutputFormatsConfig,
-				FileExists: func(filename string) (bool, error) {
-					return helpers.Exists(filename, t.Fs.Source)
-				},
-				ContainsAny: func(filename string, subslices [][]byte) (bool, error) {
-					return helpers.FileContainsAny(filename, subslices, t.Fs.Source)
-				},
-			}
-
-			tplID, err := output.CreateTemplateNames(descriptor)
-			if err != nil {
-				t.Log.ERROR.Printf("Failed to resolve template in path %q: %s", path, err)
-
-				return nil
-			}
-
-			if err := t.addTemplateFile(tplID.Name, tplID.MasterFilename, tplID.OverlayFilename); err != nil {
-				t.Log.ERROR.Printf("Failed to add template %q in path %q: %s", tplID.Name, path, err)
-			}
-
-		}
 		return nil
 	}
-	if err := helpers.SymbolicWalk(t.Fs.Source, absPath, walker); err != nil {
+
+	if err := helpers.SymbolicWalk(t.Layouts.Fs, "", walker); err != nil {
 		t.Log.ERROR.Printf("Failed to load templates: %s", err)
 	}
+
 }
 
 func (t *templateHandler) initFuncs() {
@@ -534,6 +510,7 @@ func (t *templateHandler) handleMaster(name, overlayFilename, masterFilename str
 }
 
 func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error {
+
 	masterTpl := t.lookup(masterFilename)
 
 	if masterTpl == nil {
@@ -565,6 +542,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin
 	if err := applyTemplateTransformersToHMLTTemplate(overlayTpl); err != nil {
 		return err
 	}
+
 	t.overlays[name] = overlayTpl
 
 	return err
@@ -572,6 +550,7 @@ func (t *htmlTemplates) handleMaster(name, overlayFilename, masterFilename strin
 }
 
 func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename string, onMissing func(filename string) (string, error)) error {
+
 	name = strings.TrimPrefix(name, textTmplNamePrefix)
 	masterTpl := t.lookup(masterFilename)
 
@@ -610,12 +589,16 @@ func (t *textTemplates) handleMaster(name, overlayFilename, masterFilename strin
 func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) error {
 	t.checkState()
 
+	t.Log.DEBUG.Printf("Add template file: name %q, baseTemplatePath %q, path %q", name, baseTemplatePath, path)
+
 	getTemplate := func(filename string) (string, error) {
-		b, err := afero.ReadFile(t.Fs.Source, filename)
+		b, err := afero.ReadFile(t.Layouts.Fs, filename)
 		if err != nil {
 			return "", err
 		}
-		return string(b), nil
+		s := string(b)
+
+		return s, nil
 	}
 
 	// get the suffix and switch on that
@@ -625,7 +608,7 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
 		//	Only HTML support for Amber
 		withoutExt := strings.TrimSuffix(name, filepath.Ext(name))
 		templateName := withoutExt + ".html"
-		b, err := afero.ReadFile(t.Fs.Source, path)
+		b, err := afero.ReadFile(t.Layouts.Fs, path)
 
 		if err != nil {
 			return err
@@ -654,14 +637,14 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
 	case ".ace":
 		//	Only HTML support for Ace
 		var innerContent, baseContent []byte
-		innerContent, err := afero.ReadFile(t.Fs.Source, path)
+		innerContent, err := afero.ReadFile(t.Layouts.Fs, path)
 
 		if err != nil {
 			return err
 		}
 
 		if baseTemplatePath != "" {
-			baseContent, err = afero.ReadFile(t.Fs.Source, baseTemplatePath)
+			baseContent, err = afero.ReadFile(t.Layouts.Fs, baseTemplatePath)
 			if err != nil {
 				return err
 			}
@@ -680,8 +663,6 @@ func (t *templateHandler) addTemplateFile(name, baseTemplatePath, path string) e
 			return err
 		}
 
-		t.Log.DEBUG.Printf("Add template file from path %s", path)
-
 		return t.AddTemplate(name, templ)
 	}
 }

tpl/tplimpl/template_funcs_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/i18n"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/gohugoio/hugo/tpl"
 	"github.com/gohugoio/hugo/tpl/internal"
 	"github.com/gohugoio/hugo/tpl/partials"
@@ -43,9 +44,18 @@ var (
 	logger = jww.NewNotepad(jww.LevelFatal, jww.LevelFatal, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
 )
 
+func newTestConfig() config.Provider {
+	v := viper.New()
+	v.Set("contentDir", "content")
+	v.Set("dataDir", "data")
+	v.Set("i18nDir", "i18n")
+	v.Set("layoutDir", "layouts")
+	v.Set("archetypeDir", "archetypes")
+	return v
+}
+
 func newDepsConfig(cfg config.Provider) deps.DepsCfg {
-	l := helpers.NewLanguage("en", cfg)
+	l := langs.NewLanguage("en", cfg)
-	l.Set("i18nDir", "i18n")
 	return deps.DepsCfg{
 		Language:            l,
 		Cfg:                 cfg,
@@ -61,13 +71,13 @@ func TestTemplateFuncsExamples(t *testing.T) {
 
 	workingDir := "/home/hugo"
 
-	v := viper.New()
+	v := newTestConfig()
 
 	v.Set("workingDir", workingDir)
 	v.Set("multilingual", true)
 	v.Set("contentDir", "content")
 	v.Set("baseURL", "http://mysite.com/hugo/")
-	v.Set("CurrentContentLanguage", helpers.NewLanguage("en", v))
+	v.Set("CurrentContentLanguage", langs.NewLanguage("en", v))
 
 	fs := hugofs.NewMem(v)
 
@@ -126,8 +136,7 @@ func TestPartialCached(t *testing.T) {
 	var data struct {
 	}
 
-	v := viper.New()
+	v := newTestConfig()
-	v.Set("contentDir", "content")
 
 	config := newDepsConfig(v)
 

tpl/tplimpl/template_test.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/hugofs"
-	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
 
@@ -34,8 +33,7 @@ func TestHTMLEscape(t *testing.T) {
 		"html":  "<h1>Hi!</h1>",
 		"other": "<h1>Hi!</h1>",
 	}
-	v := viper.New()
+	v := newTestConfig()
-	v.Set("contentDir", "content")
 	fs := hugofs.NewMem(v)
 
 	//afero.WriteFile(fs.Source, filepath.Join(workingDir, "README.txt"), []byte("Hugo Rocks!"), 0755)

tpl/transform/transform_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/langs"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -240,7 +241,7 @@ func TestPlainify(t *testing.T) {
 }
 
 func newDeps(cfg config.Provider) *deps.Deps {
-	l := helpers.NewLanguage("en", cfg)
+	l := langs.NewLanguage("en", cfg)
 	l.Set("i18nDir", "i18n")
 	cs, err := helpers.NewContentSpec(l)
 	if err != nil {