Add support for multiple staticDirs

This commit adds support for multiple statDirs both on the global and language level.

A simple `config.toml` example:

```bash
staticDir = ["static1", "static2"]
[languages]
[languages.no]
staticDir = ["staticDir_override", "static_no"]
baseURL = "https://example.no"
languageName = "Norsk"
weight = 1
title = "På norsk"

[languages.en]
staticDir2 = "static_en"
baseURL = "https://example.com"
languageName = "English"
weight = 2
title = "In English"
```

In the above, with no theme used:

the English site will get its static files as a union of "static1", "static2" and "static_en". On file duplicates, the right-most version will win.
the Norwegian site will get its static files as a union of "staticDir_override" and "static_no".

This commit also concludes the Multihost support in #4027.

Fixes #36
Closes #4027

Gopkg.lock

@@ -193,10 +193,10 @@
   revision = "86672fcb3f950f35f2e675df2240550f2a50762f"
 
 [[projects]]
-  branch = "master"
   name = "github.com/spf13/afero"
   packages = [".","mem"]
-  revision = "5660eeed305fe5f69c8fc6cf899132a459a97064"
+  revision = "8d919cbe7e2627e417f3e45c3c0e489a5b7e2536"
+  version = "v1.0.0"
 
 [[projects]]
   name = "github.com/spf13/cast"
@@ -285,6 +285,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "271e5ca84d4f9c63392ca282b940207c0c96995efb3a0a9fbc43114b0669bfa0"
+  inputs-digest = "a7cec7b1df49f84fdd4073cc70139d56c62c5fffcc7e3fcea5ca29615d4b9568"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -81,8 +81,8 @@
   version = "1.5.0"
 
 [[constraint]]
-  branch = "master"
   name = "github.com/spf13/afero"
+  version = "1.0.0"
 
 [[constraint]]
   name = "github.com/spf13/cast"

commands/commandeer.go

@@ -24,6 +24,7 @@ type commandeer struct {
 	*deps.DepsCfg
 	pathSpec    *helpers.PathSpec
 	visitedURLs *types.EvictingStringQueue
+
 	configured bool
 }
 

commands/hugo.go

@@ -22,7 +22,6 @@ import (
 	"github.com/gohugoio/hugo/hugofs"
 
 	"log"
-	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -30,6 +29,8 @@ import (
 	"sync"
 	"time"
 
+	src "github.com/gohugoio/hugo/source"
+
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/parser"
@@ -526,8 +527,7 @@ func (c *commandeer) watchConfig() {
 
 func (c *commandeer) build(watches ...bool) error {
 	if err := c.copyStatic(); err != nil {
-		// TODO(bep) multihost
+		return fmt.Errorf("Error copying static files: %s", err)
-		return fmt.Errorf("Error copying static files to %s: %s", c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")), err)
 	}
 	watch := false
 	if len(watches) > 0 && watches[0] {
@@ -538,88 +538,64 @@ func (c *commandeer) build(watches ...bool) error {
 	}
 
 	if buildWatch {
+		watchDirs, err := c.getDirList()
+		if err != nil {
+			return err
+		}
 		c.Logger.FEEDBACK.Println("Watching for changes in", c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")))
 		c.Logger.FEEDBACK.Println("Press Ctrl+C to stop")
-		utils.CheckErr(c.Logger, c.newWatcher(0))
+		utils.CheckErr(c.Logger, c.newWatcher(false, watchDirs...))
 	}
 
 	return nil
 }
 
-func (c *commandeer) getStaticSourceFs() afero.Fs {
-	source := c.Fs.Source
-	themeDir, err := c.PathSpec().GetThemeStaticDirPath()
-	staticDir := c.PathSpec().GetStaticDirPath() + helpers.FilePathSeparator
-	useTheme := true
-	useStatic := true
-
-	if err != nil {
-		if err != helpers.ErrThemeUndefined {
-			c.Logger.WARN.Println(err)
-		}
-		useTheme = false
-	} else {
-		if _, err := source.Stat(themeDir); os.IsNotExist(err) {
-			c.Logger.WARN.Println("Unable to find Theme Static Directory:", themeDir)
-			useTheme = false
-		}
-	}
-
-	if _, err := source.Stat(staticDir); os.IsNotExist(err) {
-		c.Logger.WARN.Println("Unable to find Static Directory:", staticDir)
-		useStatic = false
-	}
-
-	if !useStatic && !useTheme {
-		return nil
-	}
-
-	if !useStatic {
-		c.Logger.INFO.Println(themeDir, "is the only static directory available to sync from")
-		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, themeDir))
-	}
-
-	if !useTheme {
-		c.Logger.INFO.Println(staticDir, "is the only static directory available to sync from")
-		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, staticDir))
-	}
-
-	c.Logger.INFO.Println("using a UnionFS for static directory comprised of:")
-	c.Logger.INFO.Println("Base:", themeDir)
-	c.Logger.INFO.Println("Overlay:", staticDir)
-	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, themeDir))
-	overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(source, staticDir))
-	return afero.NewCopyOnWriteFs(base, overlay)
-}
-
 func (c *commandeer) copyStatic() error {
-	publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator
+	return c.doWithPublishDirs(c.copyStaticTo)
-	roots := c.roots()
-
-	if len(roots) == 0 {
-		return c.copyStaticTo(publishDir)
 }
 
-	for _, root := range roots {
+func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) error) error {
-		dir := filepath.Join(publishDir, root)
+	publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator
-		if err := c.copyStaticTo(dir); err != nil {
+	// If root, remove the second '/'
+	if publishDir == "//" {
+		publishDir = helpers.FilePathSeparator
+	}
+
+	languages := c.languages()
+
+	if !languages.IsMultihost() {
+		dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
+		if err != nil {
+			return err
+		}
+		return f(dirs, publishDir)
+	}
+
+	for _, l := range languages {
+		dir := filepath.Join(publishDir, l.Lang)
+		dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger)
+		if err != nil {
+			return err
+		}
+		if err := f(dirs, dir); err != nil {
 			return err
 		}
 	}
 
 	return nil
-
 }
 
-func (c *commandeer) copyStaticTo(publishDir string) error {
+func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) error {
 
 	// If root, remove the second '/'
 	if publishDir == "//" {
 		publishDir = helpers.FilePathSeparator
 	}
 
-	// Includes both theme/static & /static
+	staticSourceFs, err := dirs.CreateStaticFs()
-	staticSourceFs := c.getStaticSourceFs()
+	if err != nil {
+		return err
+	}
 
 	if staticSourceFs == nil {
 		c.Logger.WARN.Println("No static directories found to sync")
@@ -650,12 +626,17 @@ func (c *commandeer) copyStaticTo(publishDir string) error {
 }
 
 // getDirList provides NewWatcher() with a list of directories to watch for changes.
-func (c *commandeer) getDirList() []string {
+func (c *commandeer) getDirList() ([]string, error) {
 	var a []string
 	dataDir := c.PathSpec().AbsPathify(c.Cfg.GetString("dataDir"))
 	i18nDir := c.PathSpec().AbsPathify(c.Cfg.GetString("i18nDir"))
+	staticSyncer, err := newStaticSyncer(c)
+	if err != nil {
+		return nil, err
+	}
+
 	layoutDir := c.PathSpec().GetLayoutDirPath()
-	staticDir := c.PathSpec().GetStaticDirPath()
+	staticDirs := staticSyncer.d.AbsStaticDirs
 
 	walker := func(path string, fi os.FileInfo, err error) error {
 		if err != nil {
@@ -674,12 +655,12 @@ func (c *commandeer) getDirList() []string {
 				return nil
 			}
 
+			if os.IsNotExist(err) {
+				for _, staticDir := range staticDirs {
 					if path == staticDir && os.IsNotExist(err) {
 						c.Logger.WARN.Println("Skip staticDir:", err)
-				return nil
 					}
-
+				}
-			if os.IsNotExist(err) {
 				// Ignore.
 				return nil
 			}
@@ -726,17 +707,18 @@ func (c *commandeer) getDirList() []string {
 	_ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), walker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, walker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, walker)
+	for _, staticDir := range staticDirs {
 		_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, walker)
+	}
 
 	if c.PathSpec().ThemeSet() {
 		themesDir := c.PathSpec().GetThemeDir()
 		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "layouts"), walker)
-		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "static"), walker)
 		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "i18n"), walker)
 		_ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "data"), walker)
 	}
 
-	return a
+	return a, nil
 }
 
 func (c *commandeer) recreateAndBuildSites(watching bool) (err error) {
@@ -798,11 +780,18 @@ func (c *commandeer) rebuildSites(events []fsnotify.Event) error {
 }
 
 // newWatcher creates a new watcher to watch filesystem events.
-func (c *commandeer) newWatcher(port int) error {
+// if serve is set it will also start one or more HTTP servers to serve those
+// files.
+func (c *commandeer) newWatcher(serve bool, dirList ...string) error {
 	if runtime.GOOS == "darwin" {
 		tweakLimit()
 	}
 
+	staticSyncer, err := newStaticSyncer(c)
+	if err != nil {
+		return err
+	}
+
 	watcher, err := watcher.New(1 * time.Second)
 	var wg sync.WaitGroup
 
@@ -814,7 +803,7 @@ func (c *commandeer) newWatcher(port int) error {
 
 	wg.Add(1)
 
-	for _, d := range c.getDirList() {
+	for _, d := range dirList {
 		if d != "" {
 			_ = watcher.Add(d)
 		}
@@ -874,7 +863,7 @@ func (c *commandeer) newWatcher(port int) error {
 							if err := watcher.Add(path); err != nil {
 								return err
 							}
-						} else if !c.isStatic(path) {
+						} else if !staticSyncer.isStatic(path) {
 							// Hugo's rebuilding logic is entirely file based. When you drop a new folder into
 							// /content on OSX, the above logic will handle future watching of those files,
 							// but the initial CREATE is lost.
@@ -891,7 +880,7 @@ func (c *commandeer) newWatcher(port int) error {
 						}
 					}
 
-					if c.isStatic(ev.Name) {
+					if staticSyncer.isStatic(ev.Name) {
 						staticEvents = append(staticEvents, ev)
 					} else {
 						dynamicEvents = append(dynamicEvents, ev)
@@ -899,101 +888,21 @@ func (c *commandeer) newWatcher(port int) error {
 				}
 
 				if len(staticEvents) > 0 {
-					publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator
-
-					// If root, remove the second '/'
-					if publishDir == "//" {
-						publishDir = helpers.FilePathSeparator
-					}
-
 					c.Logger.FEEDBACK.Println("\nStatic file changes detected")
 					const layout = "2006-01-02 15:04:05.000 -0700"
 					c.Logger.FEEDBACK.Println(time.Now().Format(layout))
 
 					if c.Cfg.GetBool("forceSyncStatic") {
 						c.Logger.FEEDBACK.Printf("Syncing all static files\n")
-						// TODO(bep) multihost
 						err := c.copyStatic()
 						if err != nil {
-							utils.StopOnErr(c.Logger, err, fmt.Sprintf("Error copying static files to %s", publishDir))
+							utils.StopOnErr(c.Logger, err, "Error copying static files to publish dir")
 						}
 					} else {
-						staticSourceFs := c.getStaticSourceFs()
+						if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil {
-
-						if staticSourceFs == nil {
-							c.Logger.WARN.Println("No static directories found to sync")
-							return
-						}
-
-						syncer := fsync.NewSyncer()
-						syncer.NoTimes = c.Cfg.GetBool("noTimes")
-						syncer.NoChmod = c.Cfg.GetBool("noChmod")
-						syncer.SrcFs = staticSourceFs
-						syncer.DestFs = c.Fs.Destination
-
-						// prevent spamming the log on changes
-						logger := helpers.NewDistinctFeedbackLogger()
-
-						for _, ev := range staticEvents {
-							// Due to our approach of layering both directories and the content's rendered output
-							// into one we can't accurately remove a file not in one of the source directories.
-							// If a file is in the local static dir and also in the theme static dir and we remove
-							// it from one of those locations we expect it to still exist in the destination
-							//
-							// If Hugo generates a file (from the content dir) over a static file
-							// the content generated file should take precedence.
-							//
-							// Because we are now watching and handling individual events it is possible that a static
-							// event that occupies the same path as a content generated file will take precedence
-							// until a regeneration of the content takes places.
-							//
-							// Hugo assumes that these cases are very rare and will permit this bad behavior
-							// The alternative is to track every single file and which pipeline rendered it
-							// and then to handle conflict resolution on every event.
-
-							fromPath := ev.Name
-
-							// If we are here we already know the event took place in a static dir
-							relPath, err := c.PathSpec().MakeStaticPathRelative(fromPath)
-							if err != nil {
 							c.Logger.ERROR.Println(err)
 							continue
 						}
-
-							// Remove || rename is harder and will require an assumption.
-							// Hugo takes the following approach:
-							// If the static file exists in any of the static source directories after this event
-							// Hugo will re-sync it.
-							// If it does not exist in all of the static directories Hugo will remove it.
-							//
-							// This assumes that Hugo has not generated content on top of a static file and then removed
-							// the source of that static file. In this case Hugo will incorrectly remove that file
-							// from the published directory.
-							if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
-								if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {
-									// If file doesn't exist in any static dir, remove it
-									toRemove := filepath.Join(publishDir, relPath)
-									logger.Println("File no longer exists in static dir, removing", toRemove)
-									_ = c.Fs.Destination.RemoveAll(toRemove)
-								} else if err == nil {
-									// If file still exists, sync it
-									logger.Println("Syncing", relPath, "to", publishDir)
-									if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-										c.Logger.ERROR.Println(err)
-									}
-								} else {
-									c.Logger.ERROR.Println(err)
-								}
-
-								continue
-							}
-
-							// For all other event operations Hugo will sync static.
-							logger.Println("Syncing", relPath, "to", publishDir)
-							if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
-								c.Logger.ERROR.Println(err)
-							}
-						}
 					}
 
 					if !buildWatch && !c.Cfg.GetBool("disableLiveReload") {
@@ -1002,7 +911,7 @@ func (c *commandeer) newWatcher(port int) error {
 						// force refresh when more than one file
 						if len(staticEvents) > 0 {
 							for _, ev := range staticEvents {
-								path, _ := c.PathSpec().MakeStaticPathRelative(ev.Name)
+								path := staticSyncer.d.MakeStaticPathRelative(ev.Name)
 								livereload.RefreshPath(path)
 							}
 
@@ -1044,7 +953,7 @@ func (c *commandeer) newWatcher(port int) error {
 						}
 
 						if p != nil {
-							livereload.NavigateToPath(p.RelPermalink())
+							livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort())
 						} else {
 							livereload.ForceRefresh()
 						}
@@ -1058,14 +967,8 @@ func (c *commandeer) newWatcher(port int) error {
 		}
 	}()
 
-	if port > 0 {
+	if serve {
-		if !c.Cfg.GetBool("disableLiveReload") {
+		go c.serve()
-			livereload.Initialize()
-			http.HandleFunc("/livereload.js", livereload.ServeJS)
-			http.HandleFunc("/livereload", livereload.Handler)
-		}
-
-		go c.serve(port)
 	}
 
 	wg.Wait()
@@ -1084,10 +987,6 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
 	return name
 }
 
-func (c *commandeer) isStatic(path string) bool {
-	return strings.HasPrefix(path, c.PathSpec().GetStaticDirPath()) || (len(c.PathSpec().GetThemesDirPath()) > 0 && strings.HasPrefix(path, c.PathSpec().GetThemesDirPath()))
-}
-
 // isThemeVsHugoVersionMismatch returns whether the current Hugo version is
 // less than the theme's min_version.
 func (c *commandeer) isThemeVsHugoVersionMismatch() (mismatch bool, requiredMinVersion string) {

commands/server.go

@@ -25,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/livereload"
+
 	"github.com/gohugoio/hugo/config"
 
 	"github.com/gohugoio/hugo/helpers"
@@ -189,7 +191,7 @@ func server(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		c.Cfg.Set("baseURL", baseURL)
+		c.Set("baseURL", baseURL)
 	}
 
 	if err := memStats(); err != nil {
@@ -218,16 +220,22 @@ func server(cmd *cobra.Command, args []string) error {
 
 	// Watch runs its own server as part of the routine
 	if serverWatch {
-		watchDirs := c.getDirList()
+
-		baseWatchDir := c.Cfg.GetString("workingDir")
+		watchDirs, err := c.getDirList()
-		for i, dir := range watchDirs {
+		if err != nil {
-			watchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir)
+			return err
 		}
 
-		rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(watchDirs)), ",")
+		baseWatchDir := c.Cfg.GetString("workingDir")
+		relWatchDirs := make([]string, len(watchDirs))
+		for i, dir := range watchDirs {
+			relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir)
+		}
+
+		rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(relWatchDirs)), ",")
 
 		jww.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs)
-		err := c.newWatcher(serverPort)
+		err = c.newWatcher(true, watchDirs...)
 
 		if err != nil {
 			return err
@@ -238,7 +246,7 @@ func server(cmd *cobra.Command, args []string) error {
 }
 
 type fileServer struct {
-	basePort int
+	ports    []int
 	baseURLs []string
 	roots    []string
 	c        *commandeer
@@ -247,7 +255,7 @@ type fileServer struct {
 func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) {
 	baseURL := f.baseURLs[i]
 	root := f.roots[i]
-	port := f.basePort + i
+	port := f.ports[i]
 
 	publishDir := f.c.Cfg.GetString("publishDir")
 
@@ -257,12 +265,13 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) {
 
 	absPublishDir := f.c.PathSpec().AbsPathify(publishDir)
 
-	// TODO(bep) multihost unify feedback
+	if i == 0 {
 		if renderToDisk {
 			jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
 		} else {
 			jww.FEEDBACK.Println("Serving pages from memory")
 		}
+	}
 
 	httpFs := afero.NewHttpFs(f.c.Fs.Destination)
 	fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
@@ -270,7 +279,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) {
 	doLiveReload := !buildWatch && !f.c.Cfg.GetBool("disableLiveReload")
 	fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender")
 
-	if fastRenderMode {
+	if i == 0 && fastRenderMode {
 		jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
 	}
 
@@ -311,49 +320,50 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) {
 	return mu, endpoint, nil
 }
 
-func (c *commandeer) roots() []string {
+func (c *commandeer) serve() {
-	var roots []string
-	languages := c.languages()
-	isMultiHost := languages.IsMultihost()
-	if !isMultiHost {
-		return roots
-	}
 
-	for _, l := range languages {
-		roots = append(roots, l.Lang)
-	}
-	return roots
-}
-
-func (c *commandeer) serve(port int) {
-	// TODO(bep) multihost
 	isMultiHost := Hugo.IsMultihost()
 
 	var (
 		baseURLs []string
 		roots    []string
+		ports    []int
 	)
 
 	if isMultiHost {
 		for _, s := range Hugo.Sites {
 			baseURLs = append(baseURLs, s.BaseURL.String())
 			roots = append(roots, s.Language.Lang)
+			ports = append(ports, s.Info.ServerPort())
 		}
 	} else {
-		baseURLs = []string{Hugo.Sites[0].BaseURL.String()}
+		s := Hugo.Sites[0]
+		baseURLs = []string{s.BaseURL.String()}
 		roots = []string{""}
+		ports = append(ports, s.Info.ServerPort())
 	}
 
 	srv := &fileServer{
-		basePort: port,
+		ports:    ports,
 		baseURLs: baseURLs,
 		roots:    roots,
 		c:        c,
 	}
 
+	doLiveReload := !c.Cfg.GetBool("disableLiveReload")
+
+	if doLiveReload {
+		livereload.Initialize()
+	}
+
 	for i, _ := range baseURLs {
 		mu, endpoint, err := srv.createEndpoint(i)
 
+		if doLiveReload {
+			mu.HandleFunc("/livereload.js", livereload.ServeJS)
+			mu.HandleFunc("/livereload", livereload.Handler)
+		}
+		jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", endpoint, serverInterface)
 		go func() {
 			err = http.ListenAndServe(endpoint, mu)
 			if err != nil {
@@ -363,7 +373,6 @@ func (c *commandeer) serve(port int) {
 		}()
 	}
 
-	// TODO(bep) multihost		jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface)
 	jww.FEEDBACK.Println("Press Ctrl+C to stop")
 }
 

commands/static_syncer.go

@@ -0,0 +1,135 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package commands
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/gohugoio/hugo/helpers"
+	src "github.com/gohugoio/hugo/source"
+	"github.com/spf13/fsync"
+)
+
+type staticSyncer struct {
+	c *commandeer
+	d *src.Dirs
+}
+
+func newStaticSyncer(c *commandeer) (*staticSyncer, error) {
+	dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticSyncer{c: c, d: dirs}, nil
+}
+
+func (s *staticSyncer) isStatic(path string) bool {
+	return s.d.IsStatic(path)
+}
+
+func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
+	c := s.c
+
+	syncFn := func(dirs *src.Dirs, publishDir string) error {
+		staticSourceFs, err := dirs.CreateStaticFs()
+		if err != nil {
+			return err
+		}
+
+		if staticSourceFs == nil {
+			c.Logger.WARN.Println("No static directories found to sync")
+			return nil
+		}
+
+		syncer := fsync.NewSyncer()
+		syncer.NoTimes = c.Cfg.GetBool("noTimes")
+		syncer.NoChmod = c.Cfg.GetBool("noChmod")
+		syncer.SrcFs = staticSourceFs
+		syncer.DestFs = c.Fs.Destination
+
+		// prevent spamming the log on changes
+		logger := helpers.NewDistinctFeedbackLogger()
+
+		for _, ev := range staticEvents {
+			// Due to our approach of layering both directories and the content's rendered output
+			// into one we can't accurately remove a file not in one of the source directories.
+			// If a file is in the local static dir and also in the theme static dir and we remove
+			// it from one of those locations we expect it to still exist in the destination
+			//
+			// If Hugo generates a file (from the content dir) over a static file
+			// the content generated file should take precedence.
+			//
+			// Because we are now watching and handling individual events it is possible that a static
+			// event that occupies the same path as a content generated file will take precedence
+			// until a regeneration of the content takes places.
+			//
+			// Hugo assumes that these cases are very rare and will permit this bad behavior
+			// The alternative is to track every single file and which pipeline rendered it
+			// and then to handle conflict resolution on every event.
+
+			fromPath := ev.Name
+
+			// If we are here we already know the event took place in a static dir
+			relPath := dirs.MakeStaticPathRelative(fromPath)
+			if relPath == "" {
+				// Not member of this virtual host.
+				continue
+			}
+
+			// Remove || rename is harder and will require an assumption.
+			// Hugo takes the following approach:
+			// If the static file exists in any of the static source directories after this event
+			// Hugo will re-sync it.
+			// If it does not exist in all of the static directories Hugo will remove it.
+			//
+			// This assumes that Hugo has not generated content on top of a static file and then removed
+			// the source of that static file. In this case Hugo will incorrectly remove that file
+			// from the published directory.
+			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
+				if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) {
+					// If file doesn't exist in any static dir, remove it
+					toRemove := filepath.Join(publishDir, relPath)
+
+					logger.Println("File no longer exists in static dir, removing", toRemove)
+					_ = c.Fs.Destination.RemoveAll(toRemove)
+				} else if err == nil {
+					// If file still exists, sync it
+					logger.Println("Syncing", relPath, "to", publishDir)
+
+					if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
+						c.Logger.ERROR.Println(err)
+					}
+				} else {
+					c.Logger.ERROR.Println(err)
+				}
+
+				continue
+			}
+
+			// For all other event operations Hugo will sync static.
+			logger.Println("Syncing", relPath, "to", publishDir)
+			if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
+				c.Logger.ERROR.Println(err)
+			}
+		}
+
+		return nil
+	}
+
+	return c.doWithPublishDirs(syncFn)
+
+}

helpers/path.go

@@ -170,7 +170,7 @@ func (p *PathSpec) GetLayoutDirPath() string {
 // GetStaticDirPath returns the absolute path to the static file dir
 // for the current Hugo project.
 func (p *PathSpec) GetStaticDirPath() string {
-	return p.AbsPathify(p.staticDir)
+	return p.AbsPathify(p.StaticDir())
 }
 
 // GetThemeDir gets the root directory of the current theme, if there is one.

helpers/path_test.go

@@ -59,7 +59,8 @@ func TestMakePath(t *testing.T) {
 		v := viper.New()
 		l := NewDefaultLanguage(v)
 		v.Set("removePathAccents", test.removeAccents)
-		p, _ := NewPathSpec(hugofs.NewMem(v), l)
+		p, err := NewPathSpec(hugofs.NewMem(v), l)
+		require.NoError(t, err)
 
 		output := p.MakePath(test.input)
 		if output != test.expected {

helpers/pathspec.go

@@ -40,7 +40,7 @@ type PathSpec struct {
 	themesDir  string
 	layoutDir  string
 	workingDir string
-	staticDir  string
+	staticDirs []string
 
 	// The PathSpec looks up its config settings in both the current language
 	// and then in the global Viper config.
@@ -72,6 +72,12 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err)
 	}
 
+	var staticDirs []string
+
+	for i := -1; i <= 10; i++ {
+		staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...)
+	}
+
 	ps := &PathSpec{
 		Fs:                             fs,
 		Cfg:                            cfg,
@@ -87,7 +93,7 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 		themesDir:                      cfg.GetString("themesDir"),
 		layoutDir:                      cfg.GetString("layoutDir"),
 		workingDir:                     cfg.GetString("workingDir"),
-		staticDir:                      cfg.GetString("staticDir"),
+		staticDirs:                     staticDirs,
 		theme:                          cfg.GetString("theme"),
 	}
 
@@ -98,6 +104,25 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) {
 	return ps, nil
 }
 
+func getStringOrStringSlice(cfg config.Provider, key string, id int) []string {
+
+	if id > 0 {
+		key = fmt.Sprintf("%s%d", key, id)
+	}
+
+	var out []string
+
+	sd := cfg.Get(key)
+
+	if sds, ok := sd.(string); ok {
+		out = []string{sds}
+	} else if sdsl, ok := sd.([]string); ok {
+		out = sdsl
+	}
+
+	return out
+}
+
 // PaginatePath returns the configured root path used for paginator pages.
 func (p *PathSpec) PaginatePath() string {
 	return p.paginatePath
@@ -108,7 +133,17 @@ func (p *PathSpec) WorkingDir() string {
 	return p.workingDir
 }
 
-// LayoutDir returns the relative layout dir in the currenct Hugo project.
+// StaticDir returns the relative static dir in the current configuration.
+func (p *PathSpec) StaticDir() string {
+	return p.staticDirs[len(p.staticDirs)-1]
+}
+
+// StaticDirs returns the relative static dirs for the current configuration.
+func (p *PathSpec) StaticDirs() []string {
+	return p.staticDirs
+}
+
+// LayoutDir returns the relative layout dir in the current configuration.
 func (p *PathSpec) LayoutDir() string {
 	return p.layoutDir
 }
@@ -117,3 +152,8 @@ func (p *PathSpec) LayoutDir() string {
 func (p *PathSpec) Theme() string {
 	return p.theme
 }
+
+// Theme returns the theme relative theme dir.
+func (p *PathSpec) ThemesDir() string {
+	return p.themesDir
+}

helpers/pathspec_test.go

@@ -57,6 +57,6 @@ func TestNewPathSpecFromConfig(t *testing.T) {
 	require.Equal(t, "thethemes", p.themesDir)
 	require.Equal(t, "thelayouts", p.layoutDir)
 	require.Equal(t, "thework", p.workingDir)
-	require.Equal(t, "thestatic", p.staticDir)
+	require.Equal(t, "thestatic", p.StaticDir())
 	require.Equal(t, "thetheme", p.theme)
 }

hugolib/config.go

@@ -14,6 +14,7 @@
 package hugolib
 
 import (
+	"errors"
 	"fmt"
 
 	"io"
@@ -88,7 +89,7 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper.
 	return v, nil
 }
 
-func loadLanguageSettings(cfg config.Provider) error {
+func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error {
 	multilingual := cfg.GetStringMap("languages")
 	var (
 		langs helpers.Languages
@@ -104,7 +105,56 @@ func loadLanguageSettings(cfg config.Provider) error {
 		}
 	}
 
+	if oldLangs != nil {
+		// When in multihost mode, the languages are mapped to a server, so
+		// some structural language changes will need a restart of the dev server.
+		// The validation below isn't complete, but should cover the most
+		// important cases.
+		var invalid bool
+		if langs.IsMultihost() != oldLangs.IsMultihost() {
+			invalid = true
+		} else {
+			if langs.IsMultihost() && len(langs) != len(oldLangs) {
+				invalid = true
+			}
+		}
+
+		if invalid {
+			return errors.New("language change needing a server restart detected")
+		}
+
+		if langs.IsMultihost() {
+			// We need to transfer any server baseURL to the new language
+			for i, ol := range oldLangs {
+				nl := langs[i]
+				nl.Set("baseURL", ol.GetString("baseURL"))
+			}
+		}
+	}
+
 	cfg.Set("languagesSorted", langs)
+	cfg.Set("multilingual", len(langs) > 1)
+
+	// The baseURL may be provided at the language level. If that is true,
+	// then every language must have a baseURL. In this case we always render
+	// to a language sub folder, which is then stripped from all the Permalink URLs etc.
+	var baseURLFromLang bool
+
+	for _, l := range langs {
+		burl := l.GetLocal("baseURL")
+		if baseURLFromLang && burl == nil {
+			return errors.New("baseURL must be set on all or none of the languages")
+		}
+
+		if burl != nil {
+			baseURLFromLang = true
+		}
+	}
+
+	if baseURLFromLang {
+		cfg.Set("defaultContentLanguageInSubdir", true)
+		cfg.Set("multihost", true)
+	}
 
 	return nil
 }
@@ -178,5 +228,5 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("debug", false)
 	v.SetDefault("disableFastRender", false)
 
-	return loadLanguageSettings(v)
+	return loadLanguageSettings(v, nil)
 }

hugolib/hugo_sites.go

@@ -83,46 +83,19 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 
 	h := &HugoSites{
 		multilingual: langConfig,
+		multihost:    cfg.Cfg.GetBool("multihost"),
 		Sites:        sites}
 
 	for _, s := range sites {
 		s.owner = h
 	}
 
-	// TODO(bep)
-	cfg.Cfg.Set("multilingual", sites[0].multilingualEnabled())
-
 	if err := applyDepsIfNeeded(cfg, sites...); err != nil {
 		return nil, err
 	}
 
 	h.Deps = sites[0].Deps
 
-	// The baseURL may be provided at the language level. If that is true,
-	// then every language must have a baseURL. In this case we always render
-	// to a language sub folder, which is then stripped from all the Permalink URLs etc.
-	var baseURLFromLang bool
-
-	for _, s := range sites {
-		burl := s.Language.GetLocal("baseURL")
-		if baseURLFromLang && burl == nil {
-			return h, errors.New("baseURL must be set on all or none of the languages")
-		}
-
-		if burl != nil {
-			baseURLFromLang = true
-		}
-	}
-
-	if baseURLFromLang {
-		for _, s := range sites {
-			// TODO(bep) multihost check
-			s.Info.defaultContentLanguageInSubdir = true
-			s.Cfg.Set("defaultContentLanguageInSubdir", true)
-		}
-		h.multihost = true
-	}
-
 	return h, nil
 }
 
@@ -237,8 +210,9 @@ func (h *HugoSites) reset() {
 }
 
 func (h *HugoSites) createSitesFromConfig() error {
+	oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages)
 
-	if err := loadLanguageSettings(h.Cfg); err != nil {
+	if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil {
 		return err
 	}
 
@@ -269,6 +243,7 @@ func (h *HugoSites) createSitesFromConfig() error {
 	h.Deps = sites[0].Deps
 
 	h.multilingual = langConfig
+	h.multihost = h.Deps.Cfg.GetBool("multihost")
 
 	return nil
 }

hugolib/hugo_sites_build_test.go

@@ -1035,7 +1035,7 @@ func createMultiTestSitesForConfig(t *testing.T, siteConfig testSiteConfig, conf
 
 	if err := afero.WriteFile(mf,
 		filepath.Join("layouts", "_default/list.html"),
-		[]byte("{{ $p := .Paginator }}List Page {{ $p.PageNumber }}: {{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}"),
+		[]byte("{{ $p := .Paginator }}List Page {{ $p.PageNumber }}: {{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}"),
 		0755); err != nil {
 		t.Fatalf("Failed to write layout file: %s", err)
 	}

hugolib/hugo_sites_multihost_test.go

@@ -69,4 +69,10 @@ languageName = "Nynorsk"
 	th.assertFileContentStraight("public/fr/index.html", "French Home Page")
 	th.assertFileContentStraight("public/en/index.html", "Default Home Page")
 
+	// Check paginators
+	th.assertFileContent("public/en/page/1/index.html", `refresh" content="0; url=https://example.com/"`)
+	th.assertFileContent("public/nn/page/1/index.html", `refresh" content="0; url=https://example.no/"`)
+	th.assertFileContent("public/en/sect/page/2/index.html", "List Page 2", "Hello", "https://example.com/sect/", "\"/sect/page/3/")
+	th.assertFileContent("public/fr/sect/page/2/index.html", "List Page 2", "Bonjour", "https://example.fr/sect/")
+
 }

hugolib/page.go

@@ -1755,7 +1755,6 @@ func (p *Page) shouldAddLanguagePrefix() bool {
 	}
 
 	if p.s.owner.IsMultihost() {
-		// TODO(bep) multihost check vs lang below
 		return true
 	}
 

hugolib/page_output.go

@@ -41,7 +41,7 @@ type PageOutput struct {
 }
 
 func (p *PageOutput) targetPath(addends ...string) (string, error) {
-	tp, err := p.createTargetPath(p.outputFormat, addends...)
+	tp, err := p.createTargetPath(p.outputFormat, false, addends...)
 	if err != nil {
 		return "", err
 	}

hugolib/page_paths.go

@@ -125,12 +125,16 @@ func (p *Page) initTargetPathDescriptor() error {
 // createTargetPath creates the target filename for this Page for the given
 // output.Format. Some additional URL parts can also be provided, the typical
 // use case being pagination.
-func (p *Page) createTargetPath(t output.Format, addends ...string) (string, error) {
+func (p *Page) createTargetPath(t output.Format, noLangPrefix bool, addends ...string) (string, error) {
 	d, err := p.createTargetPathDescriptor(t)
 	if err != nil {
 		return "", nil
 	}
 
+	if noLangPrefix {
+		d.LangPrefix = ""
+	}
+
 	if len(addends) > 0 {
 		d.Addends = filepath.Join(addends...)
 	}
@@ -246,7 +250,7 @@ func (p *Page) createRelativePermalink() string {
 }
 
 func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string {
-	tp, err := p.createTargetPath(f)
+	tp, err := p.createTargetPath(f, p.s.owner.IsMultihost())
 
 	if err != nil {
 		p.s.Log.ERROR.Printf("Failed to create permalink for page %q: %s", p.FullFilePath(), err)
@@ -257,10 +261,6 @@ func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string {
 		tp = strings.TrimSuffix(tp, f.BaseFilename())
 	}
 
-	if p.s.owner.IsMultihost() {
-		tp = strings.TrimPrefix(tp, helpers.FilePathSeparator+p.s.Info.Language.Lang)
-	}
-
 	return p.s.PathSpec.URLizeFilename(tp)
 }
 

hugolib/pagination.go

@@ -285,7 +285,11 @@ func (p *PageOutput) Paginator(options ...interface{}) (*Pager, error) {
 			return
 		}
 
-		pagers, err := paginatePages(p.targetPathDescriptor, p.Data["Pages"], pagerSize)
+		pathDescriptor := p.targetPathDescriptor
+		if p.s.owner.IsMultihost() {
+			pathDescriptor.LangPrefix = ""
+		}
+		pagers, err := paginatePages(pathDescriptor, p.Data["Pages"], pagerSize)
 
 		if err != nil {
 			initError = err
@@ -333,7 +337,12 @@ func (p *PageOutput) Paginate(seq interface{}, options ...interface{}) (*Pager,
 		if p.paginator != nil {
 			return
 		}
-		pagers, err := paginatePages(p.targetPathDescriptor, seq, pagerSize)
+
+		pathDescriptor := p.targetPathDescriptor
+		if p.s.owner.IsMultihost() {
+			pathDescriptor.LangPrefix = ""
+		}
+		pagers, err := paginatePages(pathDescriptor, seq, pagerSize)
 
 		if err != nil {
 			initError = err
@@ -528,7 +537,6 @@ func newPaginationURLFactory(d targetPathDescriptor) paginationURLFactory {
 		targetPath := createTargetPath(pathDescriptor)
 		targetPath = strings.TrimSuffix(targetPath, d.Type.BaseFilename())
 		link := d.PathSpec.PrependBasePath(targetPath)
-
 		// Note: The targetPath is massaged with MakePathSanitized
 		return d.PathSpec.URLizeFilename(link)
 	}

hugolib/site.go

@@ -393,6 +393,19 @@ func (s *SiteInfo) BaseURL() template.URL {
 	return template.URL(s.s.PathSpec.BaseURL.String())
 }
 
+// ServerPort returns the port part of the BaseURL, 0 if none found.
+func (s *SiteInfo) ServerPort() int {
+	ps := s.s.PathSpec.BaseURL.URL().Port()
+	if ps == "" {
+		return 0
+	}
+	p, err := strconv.Atoi(ps)
+	if err != nil {
+		return 0
+	}
+	return p
+}
+
 // Used in tests.
 
 type siteBuilderCfg struct {
@@ -1806,7 +1819,7 @@ func (s *Site) renderAndWriteXML(name string, dest string, d interface{}, layout
 	if s.Info.relativeURLs {
 		path = []byte(helpers.GetDottedRelativePath(dest))
 	} else {
-		s := s.Cfg.GetString("baseURL")
+		s := s.PathSpec.BaseURL.String()
 		if !strings.HasSuffix(s, "/") {
 			s += "/"
 		}
@@ -1864,7 +1877,7 @@ func (s *Site) renderAndWritePage(name string, dest string, p *PageOutput, layou
 	if s.Info.relativeURLs {
 		path = []byte(helpers.GetDottedRelativePath(dest))
 	} else if s.Info.canonifyURLs {
-		url := s.Cfg.GetString("baseURL")
+		url := s.PathSpec.BaseURL.String()
 		if !strings.HasSuffix(url, "/") {
 			url += "/"
 		}

hugolib/site_render.go

@@ -147,7 +147,7 @@ func (s *Site) renderPaginator(p *PageOutput) error {
 
 		// write alias for page 1
 		addend := fmt.Sprintf("/%s/%d", paginatePath, 1)
-		target, err := p.createTargetPath(p.outputFormat, addend)
+		target, err := p.createTargetPath(p.outputFormat, false, addend)
 		if err != nil {
 			return err
 		}

livereload/livereload.go

@@ -38,7 +38,9 @@ package livereload
 
 import (
 	"fmt"
+	"net"
 	"net/http"
+	"net/url"
 	"path/filepath"
 
 	"github.com/gorilla/websocket"
@@ -47,7 +49,31 @@ import (
 // Prefix to signal to LiveReload that we need to navigate to another path.
 const hugoNavigatePrefix = "__hugo_navigate"
 
-var upgrader = &websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024}
+var upgrader = &websocket.Upgrader{
+	// Hugo may potentially spin up multiple HTTP servers, so we need to exclude the
+	// port when checking the origin.
+	CheckOrigin: func(r *http.Request) bool {
+		origin := r.Header["Origin"]
+		if len(origin) == 0 {
+			return true
+		}
+		u, err := url.Parse(origin[0])
+		if err != nil {
+			return false
+		}
+
+		h1, _, err := net.SplitHostPort(u.Host)
+		if err != nil {
+			return false
+		}
+		h2, _, err := net.SplitHostPort(r.Host)
+		if err != nil {
+			return false
+		}
+
+		return h1 == h2
+	},
+	ReadBufferSize: 1024, WriteBufferSize: 1024}
 
 // Handler is a HandlerFunc handling the livereload
 // Websocket interaction.
@@ -79,13 +105,28 @@ func NavigateToPath(path string) {
 	RefreshPath(hugoNavigatePrefix + path)
 }
 
+// NavigateToPathForPort is similar to NavigateToPath but will also
+// set window.location.port to the given port value.
+func NavigateToPathForPort(path string, port int) {
+	refreshPathForPort(hugoNavigatePrefix+path, port)
+}
+
 // RefreshPath tells livereload to refresh only the given path.
 // If that path points to a CSS stylesheet or an image, only the changes
 // will be updated in the browser, not the entire page.
 func RefreshPath(s string) {
+	refreshPathForPort(s, -1)
+}
+
+func refreshPathForPort(s string, port int) {
 	// Tell livereload a file has changed - will force a hard refresh if not CSS or an image
 	urlPath := filepath.ToSlash(s)
-	wsHub.broadcast <- []byte(`{"command":"reload","path":"` + urlPath + `","originalPath":"","liveCSS":true,"liveImg":true}`)
+	portStr := ""
+	if port > 0 {
+		portStr = fmt.Sprintf(`, "overrideURL": %d`, port)
+	}
+	msg := fmt.Sprintf(`{"command":"reload","path":%q,"originalPath":"","liveCSS":true,"liveImg":true%s}`, urlPath, portStr)
+	wsHub.broadcast <- []byte(msg)
 }
 
 // ServeJS serves the liverreload.js who's reference is injected into the page.
@@ -123,10 +164,14 @@ HugoReload.prototype.reload = function(path, options) {
 	
 	path = path.substring(prefix.length);
 	
-	if (window.location.pathname === path) {
+	if (!options.overrideURL && window.location.pathname === path) {
 		window.location.reload();
 	} else {
-		window.location.href = path;
+		if (options.overrideURL) {
+			window.location = location.protocol + "//" + location.hostname + ":" + options.overrideURL + path;
+		} else {
+			window.location.pathname = path;
+		}
 	}
 
 	return true;

source/dirs.go

@@ -0,0 +1,191 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/afero"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugofs"
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+// Dirs holds the source directories for a given build.
+// In case where there are more than one of a kind, the order matters:
+// It will be used to construct a union filesystem, so the right-most directory
+// will "win" on duplicates. Typically, the theme version will be the first.
+type Dirs struct {
+	logger   *jww.Notepad
+	pathSpec *helpers.PathSpec
+
+	staticDirs    []string
+	AbsStaticDirs []string
+
+	publishDir string
+}
+
+// NewDirs creates a new dirs with the given configuration and filesystem.
+func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) {
+	ps, err := helpers.NewPathSpec(fs, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	d := &Dirs{pathSpec: ps, logger: logger}
+
+	return d, d.init(cfg)
+
+}
+
+func (d *Dirs) init(cfg config.Provider) error {
+
+	var (
+		statics []string
+	)
+
+	if d.pathSpec.Theme() != "" {
+		statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static"))
+	}
+
+	_, isLanguage := cfg.(*helpers.Language)
+	languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages)
+
+	if !isLanguage && !hasLanguages {
+		return errors.New("missing languagesSorted in config")
+	}
+
+	if !isLanguage {
+		// Merge all the static dirs.
+		for _, l := range languages {
+			addend, err := d.staticDirsFor(l)
+			if err != nil {
+				return err
+			}
+
+			statics = append(statics, addend...)
+		}
+	} else {
+		addend, err := d.staticDirsFor(cfg)
+		if err != nil {
+			return err
+		}
+
+		statics = append(statics, addend...)
+	}
+
+	d.staticDirs = removeDuplicatesKeepRight(statics)
+	d.AbsStaticDirs = make([]string, len(d.staticDirs))
+	for i, di := range d.staticDirs {
+		d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator
+	}
+
+	d.publishDir = d.pathSpec.AbsPathify(cfg.GetString("publishDir")) + helpers.FilePathSeparator
+
+	return nil
+}
+
+func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) {
+	var statics []string
+	ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg)
+	if err != nil {
+		return statics, err
+	}
+
+	statics = append(statics, ps.StaticDirs()...)
+
+	return statics, nil
+}
+
+// CreateStaticFs will create a union filesystem with the static paths configured.
+// Any missing directories will be logged as warnings.
+func (d *Dirs) CreateStaticFs() (afero.Fs, error) {
+	var (
+		source   = d.pathSpec.Fs.Source
+		absPaths []string
+	)
+
+	for _, staticDir := range d.AbsStaticDirs {
+		if _, err := source.Stat(staticDir); os.IsNotExist(err) {
+			d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir)
+		} else {
+			absPaths = append(absPaths, staticDir)
+		}
+
+	}
+
+	if len(absPaths) == 0 {
+		return nil, nil
+	}
+
+	return d.createOverlayFs(absPaths), nil
+
+}
+
+// IsStatic returns whether the given filename is located in one of the static
+// source dirs.
+func (d *Dirs) IsStatic(filename string) bool {
+	for _, absPath := range d.AbsStaticDirs {
+		if strings.HasPrefix(filename, absPath) {
+			return true
+		}
+	}
+	return false
+}
+
+// MakeStaticPathRelative creates a relative path from the given filename.
+// It will return an empty string if the filename is not a member of dirs.
+func (d *Dirs) MakeStaticPathRelative(filename string) string {
+	for _, currentPath := range d.AbsStaticDirs {
+		if strings.HasPrefix(filename, currentPath) {
+			return strings.TrimPrefix(filename, currentPath)
+		}
+	}
+
+	return ""
+
+}
+
+func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs {
+	source := d.pathSpec.Fs.Source
+
+	if len(absPaths) == 1 {
+		return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
+	}
+
+	base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0]))
+	overlay := d.createOverlayFs(absPaths[1:])
+
+	return afero.NewCopyOnWriteFs(base, overlay)
+}
+
+func removeDuplicatesKeepRight(in []string) []string {
+	seen := make(map[string]bool)
+	var out []string
+	for i := len(in) - 1; i >= 0; i-- {
+		v := in[i]
+		if seen[v] {
+			continue
+		}
+		out = append([]string{v}, out...)
+		seen[v] = true
+	}
+
+	return out
+}

source/dirs_test.go

@@ -0,0 +1,177 @@
+// Copyright 2017 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package source
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/helpers"
+
+	"fmt"
+
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/afero"
+
+	jww "github.com/spf13/jwalterweatherman"
+
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime)
+
+func TestStaticDirs(t *testing.T) {
+	assert := require.New(t)
+
+	tests := []struct {
+		setup    func(cfg config.Provider, fs *hugofs.Fs) config.Provider
+		expected []string
+	}{
+
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+			return cfg
+		}, []string{"s1"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", []string{"s2", "s1", "s2"})
+			return cfg
+		}, []string{"s1", "s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("theme", "mytheme")
+			cfg.Set("themesDir", "themes")
+			cfg.Set("staticDir", []string{"s1", "s2"})
+			return cfg
+		}, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+
+			l1 := helpers.NewLanguage("en", cfg)
+			l1.Set("staticDir", []string{"l1s1", "l1s2"})
+			return l1
+
+		}, []string{"l1s1", "l1s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+
+			l1 := helpers.NewLanguage("en", cfg)
+			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
+			return l1
+
+		}, []string{"s1", "l1s1", "l1s2"}},
+		{func(cfg config.Provider, fs *hugofs.Fs) config.Provider {
+			cfg.Set("staticDir", "s1")
+
+			l1 := helpers.NewLanguage("en", cfg)
+			l1.Set("staticDir2", []string{"l1s1", "l1s2"})
+			l2 := helpers.NewLanguage("nn", cfg)
+			l2.Set("staticDir3", []string{"l2s1", "l2s2"})
+			l2.Set("staticDir", []string{"l2"})
+
+			cfg.Set("languagesSorted", helpers.Languages{l1, l2})
+			return cfg
+
+		}, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}},
+	}
+
+	for i, test := range tests {
+		if i != 0 {
+			break
+		}
+		msg := fmt.Sprintf("Test %d", i)
+		v := viper.New()
+		fs := hugofs.NewMem(v)
+		cfg := test.setup(v, fs)
+		cfg.Set("workingDir", filepath.FromSlash("/work"))
+		_, isLanguage := cfg.(*helpers.Language)
+		if !isLanguage && !cfg.IsSet("languagesSorted") {
+			cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)})
+		}
+		dirs, err := NewDirs(fs, cfg, logger)
+		assert.NoError(err)
+		assert.Equal(test.expected, dirs.staticDirs, msg)
+		assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs))
+
+		for i, d := range dirs.staticDirs {
+			abs := dirs.AbsStaticDirs[i]
+			assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs)
+			assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png")))
+			rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png"))
+			assert.Equal("logo.png", rel)
+		}
+
+		assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png")))
+
+	}
+
+}
+
+func TestStaticDirsFs(t *testing.T) {
+	assert := require.New(t)
+	v := viper.New()
+	fs := hugofs.NewMem(v)
+	v.Set("workingDir", filepath.FromSlash("/work"))
+	v.Set("theme", "mytheme")
+	v.Set("themesDir", "themes")
+	v.Set("staticDir", []string{"s1", "s2"})
+	v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)})
+
+	writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1")
+	writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2")
+	writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2")
+	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1")
+	writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3")
+
+	dirs, err := NewDirs(fs, v, logger)
+	assert.NoError(err)
+
+	sfs, err := dirs.CreateStaticFs()
+	assert.NoError(err)
+
+	assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt"))
+	assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt"))
+	assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt"))
+
+}
+
+func TestRemoveDuplicatesKeepRight(t *testing.T) {
+	in := []string{"a", "b", "c", "a"}
+	out := removeDuplicatesKeepRight(in)
+
+	require.Equal(t, []string{"b", "c", "a"}, out)
+}
+
+func writeToFs(t testing.TB, fs afero.Fs, filename, content string) {
+	if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil {
+		t.Fatalf("Failed to write file: %s", err)
+	}
+}
+
+func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string {
+	filename = filepath.FromSlash(filename)
+	b, err := afero.ReadFile(fs, filename)
+	if err != nil {
+		afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
+			fmt.Println("    ", path, " ", info)
+			return nil
+		})
+		t.Fatalf("Failed to read file: %s", err)
+	}
+	return string(b)
+}

tpl/urls/init_test.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/tpl/internal"
+	"github.com/spf13/viper"
 	"github.com/stretchr/testify/require"
 )
 
@@ -26,7 +27,7 @@ func TestInit(t *testing.T) {
 	var ns *internal.TemplateFuncsNamespace
 
 	for _, nsf := range internal.TemplateFuncsNamespaceRegistry {
-		ns = nsf(&deps.Deps{})
+		ns = nsf(&deps.Deps{Cfg: viper.New()})
 		if ns.Name == name {
 			found = true
 			break

tpl/urls/urls.go

@@ -27,12 +27,14 @@ import (
 func New(deps *deps.Deps) *Namespace {
 	return &Namespace{
 		deps:      deps,
+		multihost: deps.Cfg.GetBool("multihost"),
 	}
 }
 
 // Namespace provides template functions for the "urls" namespace.
 type Namespace struct {
 	deps      *deps.Deps
+	multihost bool
 }
 
 // AbsURL takes a given string and converts it to an absolute URL.
@@ -109,7 +111,7 @@ func (ns *Namespace) RelLangURL(a interface{}) (template.HTML, error) {
 		return "", err
 	}
 
-	return template.HTML(ns.deps.PathSpec.RelURL(s, true)), nil
+	return template.HTML(ns.deps.PathSpec.RelURL(s, !ns.multihost)), nil
 }
 
 // AbsLangURL takes a given string and converts it to an absolute URL according
@@ -121,5 +123,5 @@ func (ns *Namespace) AbsLangURL(a interface{}) (template.HTML, error) {
 		return "", err
 	}
 
-	return template.HTML(ns.deps.PathSpec.AbsURL(s, true)), nil
+	return template.HTML(ns.deps.PathSpec.AbsURL(s, !ns.multihost)), nil
 }