From 60dfb9a6e076200ab3ca3fd30e34bb3c14e0a893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 12 Nov 2017 10:03:56 +0100 Subject: [PATCH] Add support for multiple staticDirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds support for multiple statDirs both on the global and language level. A simple `config.toml` example: ```bash staticDir = ["static1", "static2"] [languages] [languages.no] staticDir = ["staticDir_override", "static_no"] baseURL = "https://example.no" languageName = "Norsk" weight = 1 title = "På norsk" [languages.en] staticDir2 = "static_en" baseURL = "https://example.com" languageName = "English" weight = 2 title = "In English" ``` In the above, with no theme used: the English site will get its static files as a union of "static1", "static2" and "static_en". On file duplicates, the right-most version will win. the Norwegian site will get its static files as a union of "staticDir_override" and "static_no". This commit also concludes the Multihost support in #4027. Fixes #36 Closes #4027 --- Gopkg.lock | 6 +- Gopkg.toml | 2 +- commands/commandeer.go | 3 +- commands/hugo.go | 243 ++++++++------------------- commands/server.go | 75 +++++---- commands/static_syncer.go | 135 +++++++++++++++ helpers/path.go | 2 +- helpers/path_test.go | 3 +- helpers/pathspec.go | 46 ++++- helpers/pathspec_test.go | 2 +- hugolib/config.go | 54 +++++- hugolib/hugo_sites.go | 33 +--- hugolib/hugo_sites_build_test.go | 2 +- hugolib/hugo_sites_multihost_test.go | 6 + hugolib/page.go | 1 - hugolib/page_output.go | 2 +- hugolib/page_paths.go | 12 +- hugolib/pagination.go | 14 +- hugolib/site.go | 17 +- hugolib/site_render.go | 2 +- livereload/livereload.go | 57 ++++++- source/dirs.go | 191 +++++++++++++++++++++ source/dirs_test.go | 177 +++++++++++++++++++ tpl/urls/init_test.go | 3 +- tpl/urls/urls.go | 10 +- 25 files changed, 825 insertions(+), 273 deletions(-) create mode 100644 commands/static_syncer.go create mode 100644 source/dirs.go create mode 100644 source/dirs_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 82698a6bb..dc63e7bd4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -193,10 +193,10 @@ revision = "86672fcb3f950f35f2e675df2240550f2a50762f" [[projects]] - branch = "master" name = "github.com/spf13/afero" packages = [".","mem"] - revision = "5660eeed305fe5f69c8fc6cf899132a459a97064" + revision = "8d919cbe7e2627e417f3e45c3c0e489a5b7e2536" + version = "v1.0.0" [[projects]] name = "github.com/spf13/cast" @@ -285,6 +285,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "271e5ca84d4f9c63392ca282b940207c0c96995efb3a0a9fbc43114b0669bfa0" + inputs-digest = "a7cec7b1df49f84fdd4073cc70139d56c62c5fffcc7e3fcea5ca29615d4b9568" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index e51766330..cf12080cc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -81,8 +81,8 @@ version = "1.5.0" [[constraint]] - branch = "master" name = "github.com/spf13/afero" + version = "1.0.0" [[constraint]] name = "github.com/spf13/cast" diff --git a/commands/commandeer.go b/commands/commandeer.go index 63fc0a663..b08566613 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -24,7 +24,8 @@ type commandeer struct { *deps.DepsCfg pathSpec *helpers.PathSpec visitedURLs *types.EvictingStringQueue - configured bool + + configured bool } func (c *commandeer) Set(key string, value interface{}) { diff --git a/commands/hugo.go b/commands/hugo.go index 1714c8035..7b50d0bb3 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -22,7 +22,6 @@ import ( "github.com/gohugoio/hugo/hugofs" "log" - "net/http" "os" "path/filepath" "runtime" @@ -30,6 +29,8 @@ import ( "sync" "time" + src "github.com/gohugoio/hugo/source" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/parser" @@ -526,8 +527,7 @@ func (c *commandeer) watchConfig() { func (c *commandeer) build(watches ...bool) error { if err := c.copyStatic(); err != nil { - // TODO(bep) multihost - return fmt.Errorf("Error copying static files to %s: %s", c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")), err) + return fmt.Errorf("Error copying static files: %s", err) } watch := false if len(watches) > 0 && watches[0] { @@ -538,88 +538,64 @@ func (c *commandeer) build(watches ...bool) error { } if buildWatch { + watchDirs, err := c.getDirList() + if err != nil { + return err + } c.Logger.FEEDBACK.Println("Watching for changes in", c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir"))) c.Logger.FEEDBACK.Println("Press Ctrl+C to stop") - utils.CheckErr(c.Logger, c.newWatcher(0)) + utils.CheckErr(c.Logger, c.newWatcher(false, watchDirs...)) } return nil } -func (c *commandeer) getStaticSourceFs() afero.Fs { - source := c.Fs.Source - themeDir, err := c.PathSpec().GetThemeStaticDirPath() - staticDir := c.PathSpec().GetStaticDirPath() + helpers.FilePathSeparator - useTheme := true - useStatic := true - - if err != nil { - if err != helpers.ErrThemeUndefined { - c.Logger.WARN.Println(err) - } - useTheme = false - } else { - if _, err := source.Stat(themeDir); os.IsNotExist(err) { - c.Logger.WARN.Println("Unable to find Theme Static Directory:", themeDir) - useTheme = false - } - } - - if _, err := source.Stat(staticDir); os.IsNotExist(err) { - c.Logger.WARN.Println("Unable to find Static Directory:", staticDir) - useStatic = false - } - - if !useStatic && !useTheme { - return nil - } - - if !useStatic { - c.Logger.INFO.Println(themeDir, "is the only static directory available to sync from") - return afero.NewReadOnlyFs(afero.NewBasePathFs(source, themeDir)) - } - - if !useTheme { - c.Logger.INFO.Println(staticDir, "is the only static directory available to sync from") - return afero.NewReadOnlyFs(afero.NewBasePathFs(source, staticDir)) - } - - c.Logger.INFO.Println("using a UnionFS for static directory comprised of:") - c.Logger.INFO.Println("Base:", themeDir) - c.Logger.INFO.Println("Overlay:", staticDir) - base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, themeDir)) - overlay := afero.NewReadOnlyFs(afero.NewBasePathFs(source, staticDir)) - return afero.NewCopyOnWriteFs(base, overlay) +func (c *commandeer) copyStatic() error { + return c.doWithPublishDirs(c.copyStaticTo) } -func (c *commandeer) copyStatic() error { +func (c *commandeer) doWithPublishDirs(f func(dirs *src.Dirs, publishDir string) error) error { publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator - roots := c.roots() - - if len(roots) == 0 { - return c.copyStaticTo(publishDir) + // If root, remove the second '/' + if publishDir == "//" { + publishDir = helpers.FilePathSeparator } - for _, root := range roots { - dir := filepath.Join(publishDir, root) - if err := c.copyStaticTo(dir); err != nil { + languages := c.languages() + + if !languages.IsMultihost() { + dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) + if err != nil { + return err + } + return f(dirs, publishDir) + } + + for _, l := range languages { + dir := filepath.Join(publishDir, l.Lang) + dirs, err := src.NewDirs(c.Fs, l, c.DepsCfg.Logger) + if err != nil { + return err + } + if err := f(dirs, dir); err != nil { return err } } return nil - } -func (c *commandeer) copyStaticTo(publishDir string) error { +func (c *commandeer) copyStaticTo(dirs *src.Dirs, publishDir string) error { // If root, remove the second '/' if publishDir == "//" { publishDir = helpers.FilePathSeparator } - // Includes both theme/static & /static - staticSourceFs := c.getStaticSourceFs() + staticSourceFs, err := dirs.CreateStaticFs() + if err != nil { + return err + } if staticSourceFs == nil { c.Logger.WARN.Println("No static directories found to sync") @@ -650,12 +626,17 @@ func (c *commandeer) copyStaticTo(publishDir string) error { } // getDirList provides NewWatcher() with a list of directories to watch for changes. -func (c *commandeer) getDirList() []string { +func (c *commandeer) getDirList() ([]string, error) { var a []string dataDir := c.PathSpec().AbsPathify(c.Cfg.GetString("dataDir")) i18nDir := c.PathSpec().AbsPathify(c.Cfg.GetString("i18nDir")) + staticSyncer, err := newStaticSyncer(c) + if err != nil { + return nil, err + } + layoutDir := c.PathSpec().GetLayoutDirPath() - staticDir := c.PathSpec().GetStaticDirPath() + staticDirs := staticSyncer.d.AbsStaticDirs walker := func(path string, fi os.FileInfo, err error) error { if err != nil { @@ -674,12 +655,12 @@ func (c *commandeer) getDirList() []string { return nil } - if path == staticDir && os.IsNotExist(err) { - c.Logger.WARN.Println("Skip staticDir:", err) - return nil - } - if os.IsNotExist(err) { + for _, staticDir := range staticDirs { + if path == staticDir && os.IsNotExist(err) { + c.Logger.WARN.Println("Skip staticDir:", err) + } + } // Ignore. return nil } @@ -726,17 +707,18 @@ func (c *commandeer) getDirList() []string { _ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), walker) _ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, walker) _ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, walker) - _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, walker) + for _, staticDir := range staticDirs { + _ = helpers.SymbolicWalk(c.Fs.Source, staticDir, walker) + } if c.PathSpec().ThemeSet() { themesDir := c.PathSpec().GetThemeDir() _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "layouts"), walker) - _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "static"), walker) _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "i18n"), walker) _ = helpers.SymbolicWalk(c.Fs.Source, filepath.Join(themesDir, "data"), walker) } - return a + return a, nil } func (c *commandeer) recreateAndBuildSites(watching bool) (err error) { @@ -798,11 +780,18 @@ func (c *commandeer) rebuildSites(events []fsnotify.Event) error { } // newWatcher creates a new watcher to watch filesystem events. -func (c *commandeer) newWatcher(port int) error { +// if serve is set it will also start one or more HTTP servers to serve those +// files. +func (c *commandeer) newWatcher(serve bool, dirList ...string) error { if runtime.GOOS == "darwin" { tweakLimit() } + staticSyncer, err := newStaticSyncer(c) + if err != nil { + return err + } + watcher, err := watcher.New(1 * time.Second) var wg sync.WaitGroup @@ -814,7 +803,7 @@ func (c *commandeer) newWatcher(port int) error { wg.Add(1) - for _, d := range c.getDirList() { + for _, d := range dirList { if d != "" { _ = watcher.Add(d) } @@ -874,7 +863,7 @@ func (c *commandeer) newWatcher(port int) error { if err := watcher.Add(path); err != nil { return err } - } else if !c.isStatic(path) { + } else if !staticSyncer.isStatic(path) { // Hugo's rebuilding logic is entirely file based. When you drop a new folder into // /content on OSX, the above logic will handle future watching of those files, // but the initial CREATE is lost. @@ -891,7 +880,7 @@ func (c *commandeer) newWatcher(port int) error { } } - if c.isStatic(ev.Name) { + if staticSyncer.isStatic(ev.Name) { staticEvents = append(staticEvents, ev) } else { dynamicEvents = append(dynamicEvents, ev) @@ -899,100 +888,20 @@ func (c *commandeer) newWatcher(port int) error { } if len(staticEvents) > 0 { - publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator - - // If root, remove the second '/' - if publishDir == "//" { - publishDir = helpers.FilePathSeparator - } - c.Logger.FEEDBACK.Println("\nStatic file changes detected") const layout = "2006-01-02 15:04:05.000 -0700" c.Logger.FEEDBACK.Println(time.Now().Format(layout)) if c.Cfg.GetBool("forceSyncStatic") { c.Logger.FEEDBACK.Printf("Syncing all static files\n") - // TODO(bep) multihost err := c.copyStatic() if err != nil { - utils.StopOnErr(c.Logger, err, fmt.Sprintf("Error copying static files to %s", publishDir)) + utils.StopOnErr(c.Logger, err, "Error copying static files to publish dir") } } else { - staticSourceFs := c.getStaticSourceFs() - - if staticSourceFs == nil { - c.Logger.WARN.Println("No static directories found to sync") - return - } - - syncer := fsync.NewSyncer() - syncer.NoTimes = c.Cfg.GetBool("noTimes") - syncer.NoChmod = c.Cfg.GetBool("noChmod") - syncer.SrcFs = staticSourceFs - syncer.DestFs = c.Fs.Destination - - // prevent spamming the log on changes - logger := helpers.NewDistinctFeedbackLogger() - - for _, ev := range staticEvents { - // Due to our approach of layering both directories and the content's rendered output - // into one we can't accurately remove a file not in one of the source directories. - // If a file is in the local static dir and also in the theme static dir and we remove - // it from one of those locations we expect it to still exist in the destination - // - // If Hugo generates a file (from the content dir) over a static file - // the content generated file should take precedence. - // - // Because we are now watching and handling individual events it is possible that a static - // event that occupies the same path as a content generated file will take precedence - // until a regeneration of the content takes places. - // - // Hugo assumes that these cases are very rare and will permit this bad behavior - // The alternative is to track every single file and which pipeline rendered it - // and then to handle conflict resolution on every event. - - fromPath := ev.Name - - // If we are here we already know the event took place in a static dir - relPath, err := c.PathSpec().MakeStaticPathRelative(fromPath) - if err != nil { - c.Logger.ERROR.Println(err) - continue - } - - // Remove || rename is harder and will require an assumption. - // Hugo takes the following approach: - // If the static file exists in any of the static source directories after this event - // Hugo will re-sync it. - // If it does not exist in all of the static directories Hugo will remove it. - // - // This assumes that Hugo has not generated content on top of a static file and then removed - // the source of that static file. In this case Hugo will incorrectly remove that file - // from the published directory. - if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { - if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) { - // If file doesn't exist in any static dir, remove it - toRemove := filepath.Join(publishDir, relPath) - logger.Println("File no longer exists in static dir, removing", toRemove) - _ = c.Fs.Destination.RemoveAll(toRemove) - } else if err == nil { - // If file still exists, sync it - logger.Println("Syncing", relPath, "to", publishDir) - if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { - c.Logger.ERROR.Println(err) - } - } else { - c.Logger.ERROR.Println(err) - } - - continue - } - - // For all other event operations Hugo will sync static. - logger.Println("Syncing", relPath, "to", publishDir) - if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { - c.Logger.ERROR.Println(err) - } + if err := staticSyncer.syncsStaticEvents(staticEvents); err != nil { + c.Logger.ERROR.Println(err) + continue } } @@ -1002,7 +911,7 @@ func (c *commandeer) newWatcher(port int) error { // force refresh when more than one file if len(staticEvents) > 0 { for _, ev := range staticEvents { - path, _ := c.PathSpec().MakeStaticPathRelative(ev.Name) + path := staticSyncer.d.MakeStaticPathRelative(ev.Name) livereload.RefreshPath(path) } @@ -1044,7 +953,7 @@ func (c *commandeer) newWatcher(port int) error { } if p != nil { - livereload.NavigateToPath(p.RelPermalink()) + livereload.NavigateToPathForPort(p.RelPermalink(), p.Site.ServerPort()) } else { livereload.ForceRefresh() } @@ -1058,14 +967,8 @@ func (c *commandeer) newWatcher(port int) error { } }() - if port > 0 { - if !c.Cfg.GetBool("disableLiveReload") { - livereload.Initialize() - http.HandleFunc("/livereload.js", livereload.ServeJS) - http.HandleFunc("/livereload", livereload.Handler) - } - - go c.serve(port) + if serve { + go c.serve() } wg.Wait() @@ -1084,10 +987,6 @@ func pickOneWriteOrCreatePath(events []fsnotify.Event) string { return name } -func (c *commandeer) isStatic(path string) bool { - return strings.HasPrefix(path, c.PathSpec().GetStaticDirPath()) || (len(c.PathSpec().GetThemesDirPath()) > 0 && strings.HasPrefix(path, c.PathSpec().GetThemesDirPath())) -} - // isThemeVsHugoVersionMismatch returns whether the current Hugo version is // less than the theme's min_version. func (c *commandeer) isThemeVsHugoVersionMismatch() (mismatch bool, requiredMinVersion string) { diff --git a/commands/server.go b/commands/server.go index bd45e7054..666f255e3 100644 --- a/commands/server.go +++ b/commands/server.go @@ -25,6 +25,8 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/livereload" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/helpers" @@ -189,7 +191,7 @@ func server(cmd *cobra.Command, args []string) error { if err != nil { return err } - c.Cfg.Set("baseURL", baseURL) + c.Set("baseURL", baseURL) } if err := memStats(); err != nil { @@ -218,16 +220,22 @@ func server(cmd *cobra.Command, args []string) error { // Watch runs its own server as part of the routine if serverWatch { - watchDirs := c.getDirList() - baseWatchDir := c.Cfg.GetString("workingDir") - for i, dir := range watchDirs { - watchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir) + + watchDirs, err := c.getDirList() + if err != nil { + return err } - rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(watchDirs)), ",") + baseWatchDir := c.Cfg.GetString("workingDir") + relWatchDirs := make([]string, len(watchDirs)) + for i, dir := range watchDirs { + relWatchDirs[i], _ = helpers.GetRelativePath(dir, baseWatchDir) + } + + rootWatchDirs := strings.Join(helpers.UniqueStrings(helpers.ExtractRootPaths(relWatchDirs)), ",") jww.FEEDBACK.Printf("Watching for changes in %s%s{%s}\n", baseWatchDir, helpers.FilePathSeparator, rootWatchDirs) - err := c.newWatcher(serverPort) + err = c.newWatcher(true, watchDirs...) if err != nil { return err @@ -238,7 +246,7 @@ func server(cmd *cobra.Command, args []string) error { } type fileServer struct { - basePort int + ports []int baseURLs []string roots []string c *commandeer @@ -247,7 +255,7 @@ type fileServer struct { func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) { baseURL := f.baseURLs[i] root := f.roots[i] - port := f.basePort + i + port := f.ports[i] publishDir := f.c.Cfg.GetString("publishDir") @@ -257,11 +265,12 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) { absPublishDir := f.c.PathSpec().AbsPathify(publishDir) - // TODO(bep) multihost unify feedback - if renderToDisk { - jww.FEEDBACK.Println("Serving pages from " + absPublishDir) - } else { - jww.FEEDBACK.Println("Serving pages from memory") + if i == 0 { + if renderToDisk { + jww.FEEDBACK.Println("Serving pages from " + absPublishDir) + } else { + jww.FEEDBACK.Println("Serving pages from memory") + } } httpFs := afero.NewHttpFs(f.c.Fs.Destination) @@ -270,7 +279,7 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) { doLiveReload := !buildWatch && !f.c.Cfg.GetBool("disableLiveReload") fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender") - if fastRenderMode { + if i == 0 && fastRenderMode { jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender") } @@ -311,49 +320,50 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) { return mu, endpoint, nil } -func (c *commandeer) roots() []string { - var roots []string - languages := c.languages() - isMultiHost := languages.IsMultihost() - if !isMultiHost { - return roots - } +func (c *commandeer) serve() { - for _, l := range languages { - roots = append(roots, l.Lang) - } - return roots -} - -func (c *commandeer) serve(port int) { - // TODO(bep) multihost isMultiHost := Hugo.IsMultihost() var ( baseURLs []string roots []string + ports []int ) if isMultiHost { for _, s := range Hugo.Sites { baseURLs = append(baseURLs, s.BaseURL.String()) roots = append(roots, s.Language.Lang) + ports = append(ports, s.Info.ServerPort()) } } else { - baseURLs = []string{Hugo.Sites[0].BaseURL.String()} + s := Hugo.Sites[0] + baseURLs = []string{s.BaseURL.String()} roots = []string{""} + ports = append(ports, s.Info.ServerPort()) } srv := &fileServer{ - basePort: port, + ports: ports, baseURLs: baseURLs, roots: roots, c: c, } + doLiveReload := !c.Cfg.GetBool("disableLiveReload") + + if doLiveReload { + livereload.Initialize() + } + for i, _ := range baseURLs { mu, endpoint, err := srv.createEndpoint(i) + if doLiveReload { + mu.HandleFunc("/livereload.js", livereload.ServeJS) + mu.HandleFunc("/livereload", livereload.Handler) + } + jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", endpoint, serverInterface) go func() { err = http.ListenAndServe(endpoint, mu) if err != nil { @@ -363,7 +373,6 @@ func (c *commandeer) serve(port int) { }() } - // TODO(bep) multihost jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface) jww.FEEDBACK.Println("Press Ctrl+C to stop") } diff --git a/commands/static_syncer.go b/commands/static_syncer.go new file mode 100644 index 000000000..98b745e4c --- /dev/null +++ b/commands/static_syncer.go @@ -0,0 +1,135 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commands + +import ( + "os" + "path/filepath" + + "github.com/fsnotify/fsnotify" + "github.com/gohugoio/hugo/helpers" + src "github.com/gohugoio/hugo/source" + "github.com/spf13/fsync" +) + +type staticSyncer struct { + c *commandeer + d *src.Dirs +} + +func newStaticSyncer(c *commandeer) (*staticSyncer, error) { + dirs, err := src.NewDirs(c.Fs, c.Cfg, c.DepsCfg.Logger) + if err != nil { + return nil, err + } + + return &staticSyncer{c: c, d: dirs}, nil +} + +func (s *staticSyncer) isStatic(path string) bool { + return s.d.IsStatic(path) +} + +func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error { + c := s.c + + syncFn := func(dirs *src.Dirs, publishDir string) error { + staticSourceFs, err := dirs.CreateStaticFs() + if err != nil { + return err + } + + if staticSourceFs == nil { + c.Logger.WARN.Println("No static directories found to sync") + return nil + } + + syncer := fsync.NewSyncer() + syncer.NoTimes = c.Cfg.GetBool("noTimes") + syncer.NoChmod = c.Cfg.GetBool("noChmod") + syncer.SrcFs = staticSourceFs + syncer.DestFs = c.Fs.Destination + + // prevent spamming the log on changes + logger := helpers.NewDistinctFeedbackLogger() + + for _, ev := range staticEvents { + // Due to our approach of layering both directories and the content's rendered output + // into one we can't accurately remove a file not in one of the source directories. + // If a file is in the local static dir and also in the theme static dir and we remove + // it from one of those locations we expect it to still exist in the destination + // + // If Hugo generates a file (from the content dir) over a static file + // the content generated file should take precedence. + // + // Because we are now watching and handling individual events it is possible that a static + // event that occupies the same path as a content generated file will take precedence + // until a regeneration of the content takes places. + // + // Hugo assumes that these cases are very rare and will permit this bad behavior + // The alternative is to track every single file and which pipeline rendered it + // and then to handle conflict resolution on every event. + + fromPath := ev.Name + + // If we are here we already know the event took place in a static dir + relPath := dirs.MakeStaticPathRelative(fromPath) + if relPath == "" { + // Not member of this virtual host. + continue + } + + // Remove || rename is harder and will require an assumption. + // Hugo takes the following approach: + // If the static file exists in any of the static source directories after this event + // Hugo will re-sync it. + // If it does not exist in all of the static directories Hugo will remove it. + // + // This assumes that Hugo has not generated content on top of a static file and then removed + // the source of that static file. In this case Hugo will incorrectly remove that file + // from the published directory. + if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove { + if _, err := staticSourceFs.Stat(relPath); os.IsNotExist(err) { + // If file doesn't exist in any static dir, remove it + toRemove := filepath.Join(publishDir, relPath) + + logger.Println("File no longer exists in static dir, removing", toRemove) + _ = c.Fs.Destination.RemoveAll(toRemove) + } else if err == nil { + // If file still exists, sync it + logger.Println("Syncing", relPath, "to", publishDir) + + if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { + c.Logger.ERROR.Println(err) + } + } else { + c.Logger.ERROR.Println(err) + } + + continue + } + + // For all other event operations Hugo will sync static. + logger.Println("Syncing", relPath, "to", publishDir) + if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil { + c.Logger.ERROR.Println(err) + } + } + + return nil + } + + return c.doWithPublishDirs(syncFn) + +} diff --git a/helpers/path.go b/helpers/path.go index a9e2567c6..a0b35e5ed 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -170,7 +170,7 @@ func (p *PathSpec) GetLayoutDirPath() string { // GetStaticDirPath returns the absolute path to the static file dir // for the current Hugo project. func (p *PathSpec) GetStaticDirPath() string { - return p.AbsPathify(p.staticDir) + return p.AbsPathify(p.StaticDir()) } // GetThemeDir gets the root directory of the current theme, if there is one. diff --git a/helpers/path_test.go b/helpers/path_test.go index 5c0ae10ea..8d895d762 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -59,7 +59,8 @@ func TestMakePath(t *testing.T) { v := viper.New() l := NewDefaultLanguage(v) v.Set("removePathAccents", test.removeAccents) - p, _ := NewPathSpec(hugofs.NewMem(v), l) + p, err := NewPathSpec(hugofs.NewMem(v), l) + require.NoError(t, err) output := p.MakePath(test.input) if output != test.expected { diff --git a/helpers/pathspec.go b/helpers/pathspec.go index 643d05646..5b7f534fe 100644 --- a/helpers/pathspec.go +++ b/helpers/pathspec.go @@ -40,7 +40,7 @@ type PathSpec struct { themesDir string layoutDir string workingDir string - staticDir string + staticDirs []string // The PathSpec looks up its config settings in both the current language // and then in the global Viper config. @@ -72,6 +72,12 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { return nil, fmt.Errorf("Failed to create baseURL from %q: %s", baseURLstr, err) } + var staticDirs []string + + for i := -1; i <= 10; i++ { + staticDirs = append(staticDirs, getStringOrStringSlice(cfg, "staticDir", i)...) + } + ps := &PathSpec{ Fs: fs, Cfg: cfg, @@ -87,7 +93,7 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { themesDir: cfg.GetString("themesDir"), layoutDir: cfg.GetString("layoutDir"), workingDir: cfg.GetString("workingDir"), - staticDir: cfg.GetString("staticDir"), + staticDirs: staticDirs, theme: cfg.GetString("theme"), } @@ -98,6 +104,25 @@ func NewPathSpec(fs *hugofs.Fs, cfg config.Provider) (*PathSpec, error) { return ps, nil } +func getStringOrStringSlice(cfg config.Provider, key string, id int) []string { + + if id > 0 { + key = fmt.Sprintf("%s%d", key, id) + } + + var out []string + + sd := cfg.Get(key) + + if sds, ok := sd.(string); ok { + out = []string{sds} + } else if sdsl, ok := sd.([]string); ok { + out = sdsl + } + + return out +} + // PaginatePath returns the configured root path used for paginator pages. func (p *PathSpec) PaginatePath() string { return p.paginatePath @@ -108,7 +133,17 @@ func (p *PathSpec) WorkingDir() string { return p.workingDir } -// LayoutDir returns the relative layout dir in the currenct Hugo project. +// StaticDir returns the relative static dir in the current configuration. +func (p *PathSpec) StaticDir() string { + return p.staticDirs[len(p.staticDirs)-1] +} + +// StaticDirs returns the relative static dirs for the current configuration. +func (p *PathSpec) StaticDirs() []string { + return p.staticDirs +} + +// LayoutDir returns the relative layout dir in the current configuration. func (p *PathSpec) LayoutDir() string { return p.layoutDir } @@ -117,3 +152,8 @@ func (p *PathSpec) LayoutDir() string { func (p *PathSpec) Theme() string { return p.theme } + +// Theme returns the theme relative theme dir. +func (p *PathSpec) ThemesDir() string { + return p.themesDir +} diff --git a/helpers/pathspec_test.go b/helpers/pathspec_test.go index 04ec7cac7..c251b6ba8 100644 --- a/helpers/pathspec_test.go +++ b/helpers/pathspec_test.go @@ -57,6 +57,6 @@ func TestNewPathSpecFromConfig(t *testing.T) { require.Equal(t, "thethemes", p.themesDir) require.Equal(t, "thelayouts", p.layoutDir) require.Equal(t, "thework", p.workingDir) - require.Equal(t, "thestatic", p.staticDir) + require.Equal(t, "thestatic", p.StaticDir()) require.Equal(t, "thetheme", p.theme) } diff --git a/hugolib/config.go b/hugolib/config.go index db59253cd..da84ab8b2 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -14,6 +14,7 @@ package hugolib import ( + "errors" "fmt" "io" @@ -88,7 +89,7 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper. return v, nil } -func loadLanguageSettings(cfg config.Provider) error { +func loadLanguageSettings(cfg config.Provider, oldLangs helpers.Languages) error { multilingual := cfg.GetStringMap("languages") var ( langs helpers.Languages @@ -104,7 +105,56 @@ func loadLanguageSettings(cfg config.Provider) error { } } + if oldLangs != nil { + // When in multihost mode, the languages are mapped to a server, so + // some structural language changes will need a restart of the dev server. + // The validation below isn't complete, but should cover the most + // important cases. + var invalid bool + if langs.IsMultihost() != oldLangs.IsMultihost() { + invalid = true + } else { + if langs.IsMultihost() && len(langs) != len(oldLangs) { + invalid = true + } + } + + if invalid { + return errors.New("language change needing a server restart detected") + } + + if langs.IsMultihost() { + // We need to transfer any server baseURL to the new language + for i, ol := range oldLangs { + nl := langs[i] + nl.Set("baseURL", ol.GetString("baseURL")) + } + } + } + cfg.Set("languagesSorted", langs) + cfg.Set("multilingual", len(langs) > 1) + + // The baseURL may be provided at the language level. If that is true, + // then every language must have a baseURL. In this case we always render + // to a language sub folder, which is then stripped from all the Permalink URLs etc. + var baseURLFromLang bool + + for _, l := range langs { + burl := l.GetLocal("baseURL") + if baseURLFromLang && burl == nil { + return errors.New("baseURL must be set on all or none of the languages") + } + + if burl != nil { + baseURLFromLang = true + } + } + + if baseURLFromLang { + cfg.Set("defaultContentLanguageInSubdir", true) + cfg.Set("multihost", true) + } return nil } @@ -178,5 +228,5 @@ func loadDefaultSettingsFor(v *viper.Viper) error { v.SetDefault("debug", false) v.SetDefault("disableFastRender", false) - return loadLanguageSettings(v) + return loadLanguageSettings(v, nil) } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index e0697507b..bf488b9be 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -83,46 +83,19 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { h := &HugoSites{ multilingual: langConfig, + multihost: cfg.Cfg.GetBool("multihost"), Sites: sites} for _, s := range sites { s.owner = h } - // TODO(bep) - cfg.Cfg.Set("multilingual", sites[0].multilingualEnabled()) - if err := applyDepsIfNeeded(cfg, sites...); err != nil { return nil, err } h.Deps = sites[0].Deps - // The baseURL may be provided at the language level. If that is true, - // then every language must have a baseURL. In this case we always render - // to a language sub folder, which is then stripped from all the Permalink URLs etc. - var baseURLFromLang bool - - for _, s := range sites { - burl := s.Language.GetLocal("baseURL") - if baseURLFromLang && burl == nil { - return h, errors.New("baseURL must be set on all or none of the languages") - } - - if burl != nil { - baseURLFromLang = true - } - } - - if baseURLFromLang { - for _, s := range sites { - // TODO(bep) multihost check - s.Info.defaultContentLanguageInSubdir = true - s.Cfg.Set("defaultContentLanguageInSubdir", true) - } - h.multihost = true - } - return h, nil } @@ -237,8 +210,9 @@ func (h *HugoSites) reset() { } func (h *HugoSites) createSitesFromConfig() error { + oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages) - if err := loadLanguageSettings(h.Cfg); err != nil { + if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil { return err } @@ -269,6 +243,7 @@ func (h *HugoSites) createSitesFromConfig() error { h.Deps = sites[0].Deps h.multilingual = langConfig + h.multihost = h.Deps.Cfg.GetBool("multihost") return nil } diff --git a/hugolib/hugo_sites_build_test.go b/hugolib/hugo_sites_build_test.go index 079f0fcfa..60c86d016 100644 --- a/hugolib/hugo_sites_build_test.go +++ b/hugolib/hugo_sites_build_test.go @@ -1035,7 +1035,7 @@ func createMultiTestSitesForConfig(t *testing.T, siteConfig testSiteConfig, conf if err := afero.WriteFile(mf, filepath.Join("layouts", "_default/list.html"), - []byte("{{ $p := .Paginator }}List Page {{ $p.PageNumber }}: {{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}"), + []byte("{{ $p := .Paginator }}List Page {{ $p.PageNumber }}: {{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}"), 0755); err != nil { t.Fatalf("Failed to write layout file: %s", err) } diff --git a/hugolib/hugo_sites_multihost_test.go b/hugolib/hugo_sites_multihost_test.go index 864d52c71..995d2407e 100644 --- a/hugolib/hugo_sites_multihost_test.go +++ b/hugolib/hugo_sites_multihost_test.go @@ -69,4 +69,10 @@ languageName = "Nynorsk" th.assertFileContentStraight("public/fr/index.html", "French Home Page") th.assertFileContentStraight("public/en/index.html", "Default Home Page") + // Check paginators + th.assertFileContent("public/en/page/1/index.html", `refresh" content="0; url=https://example.com/"`) + th.assertFileContent("public/nn/page/1/index.html", `refresh" content="0; url=https://example.no/"`) + th.assertFileContent("public/en/sect/page/2/index.html", "List Page 2", "Hello", "https://example.com/sect/", "\"/sect/page/3/") + th.assertFileContent("public/fr/sect/page/2/index.html", "List Page 2", "Bonjour", "https://example.fr/sect/") + } diff --git a/hugolib/page.go b/hugolib/page.go index 7da77f192..7c72fcb99 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -1755,7 +1755,6 @@ func (p *Page) shouldAddLanguagePrefix() bool { } if p.s.owner.IsMultihost() { - // TODO(bep) multihost check vs lang below return true } diff --git a/hugolib/page_output.go b/hugolib/page_output.go index 3b1e07907..4739e6936 100644 --- a/hugolib/page_output.go +++ b/hugolib/page_output.go @@ -41,7 +41,7 @@ type PageOutput struct { } func (p *PageOutput) targetPath(addends ...string) (string, error) { - tp, err := p.createTargetPath(p.outputFormat, addends...) + tp, err := p.createTargetPath(p.outputFormat, false, addends...) if err != nil { return "", err } diff --git a/hugolib/page_paths.go b/hugolib/page_paths.go index 993ad0780..083d6eb49 100644 --- a/hugolib/page_paths.go +++ b/hugolib/page_paths.go @@ -125,12 +125,16 @@ func (p *Page) initTargetPathDescriptor() error { // createTargetPath creates the target filename for this Page for the given // output.Format. Some additional URL parts can also be provided, the typical // use case being pagination. -func (p *Page) createTargetPath(t output.Format, addends ...string) (string, error) { +func (p *Page) createTargetPath(t output.Format, noLangPrefix bool, addends ...string) (string, error) { d, err := p.createTargetPathDescriptor(t) if err != nil { return "", nil } + if noLangPrefix { + d.LangPrefix = "" + } + if len(addends) > 0 { d.Addends = filepath.Join(addends...) } @@ -246,7 +250,7 @@ func (p *Page) createRelativePermalink() string { } func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string { - tp, err := p.createTargetPath(f) + tp, err := p.createTargetPath(f, p.s.owner.IsMultihost()) if err != nil { p.s.Log.ERROR.Printf("Failed to create permalink for page %q: %s", p.FullFilePath(), err) @@ -257,10 +261,6 @@ func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string { tp = strings.TrimSuffix(tp, f.BaseFilename()) } - if p.s.owner.IsMultihost() { - tp = strings.TrimPrefix(tp, helpers.FilePathSeparator+p.s.Info.Language.Lang) - } - return p.s.PathSpec.URLizeFilename(tp) } diff --git a/hugolib/pagination.go b/hugolib/pagination.go index 4733cf7c8..894f467a4 100644 --- a/hugolib/pagination.go +++ b/hugolib/pagination.go @@ -285,7 +285,11 @@ func (p *PageOutput) Paginator(options ...interface{}) (*Pager, error) { return } - pagers, err := paginatePages(p.targetPathDescriptor, p.Data["Pages"], pagerSize) + pathDescriptor := p.targetPathDescriptor + if p.s.owner.IsMultihost() { + pathDescriptor.LangPrefix = "" + } + pagers, err := paginatePages(pathDescriptor, p.Data["Pages"], pagerSize) if err != nil { initError = err @@ -333,7 +337,12 @@ func (p *PageOutput) Paginate(seq interface{}, options ...interface{}) (*Pager, if p.paginator != nil { return } - pagers, err := paginatePages(p.targetPathDescriptor, seq, pagerSize) + + pathDescriptor := p.targetPathDescriptor + if p.s.owner.IsMultihost() { + pathDescriptor.LangPrefix = "" + } + pagers, err := paginatePages(pathDescriptor, seq, pagerSize) if err != nil { initError = err @@ -528,7 +537,6 @@ func newPaginationURLFactory(d targetPathDescriptor) paginationURLFactory { targetPath := createTargetPath(pathDescriptor) targetPath = strings.TrimSuffix(targetPath, d.Type.BaseFilename()) link := d.PathSpec.PrependBasePath(targetPath) - // Note: The targetPath is massaged with MakePathSanitized return d.PathSpec.URLizeFilename(link) } diff --git a/hugolib/site.go b/hugolib/site.go index 28414c7d4..526ba285e 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -393,6 +393,19 @@ func (s *SiteInfo) BaseURL() template.URL { return template.URL(s.s.PathSpec.BaseURL.String()) } +// ServerPort returns the port part of the BaseURL, 0 if none found. +func (s *SiteInfo) ServerPort() int { + ps := s.s.PathSpec.BaseURL.URL().Port() + if ps == "" { + return 0 + } + p, err := strconv.Atoi(ps) + if err != nil { + return 0 + } + return p +} + // Used in tests. type siteBuilderCfg struct { @@ -1806,7 +1819,7 @@ func (s *Site) renderAndWriteXML(name string, dest string, d interface{}, layout if s.Info.relativeURLs { path = []byte(helpers.GetDottedRelativePath(dest)) } else { - s := s.Cfg.GetString("baseURL") + s := s.PathSpec.BaseURL.String() if !strings.HasSuffix(s, "/") { s += "/" } @@ -1864,7 +1877,7 @@ func (s *Site) renderAndWritePage(name string, dest string, p *PageOutput, layou if s.Info.relativeURLs { path = []byte(helpers.GetDottedRelativePath(dest)) } else if s.Info.canonifyURLs { - url := s.Cfg.GetString("baseURL") + url := s.PathSpec.BaseURL.String() if !strings.HasSuffix(url, "/") { url += "/" } diff --git a/hugolib/site_render.go b/hugolib/site_render.go index b4d688bda..2a5fec7ba 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -147,7 +147,7 @@ func (s *Site) renderPaginator(p *PageOutput) error { // write alias for page 1 addend := fmt.Sprintf("/%s/%d", paginatePath, 1) - target, err := p.createTargetPath(p.outputFormat, addend) + target, err := p.createTargetPath(p.outputFormat, false, addend) if err != nil { return err } diff --git a/livereload/livereload.go b/livereload/livereload.go index 74702175f..90096577d 100644 --- a/livereload/livereload.go +++ b/livereload/livereload.go @@ -38,7 +38,9 @@ package livereload import ( "fmt" + "net" "net/http" + "net/url" "path/filepath" "github.com/gorilla/websocket" @@ -47,7 +49,31 @@ import ( // Prefix to signal to LiveReload that we need to navigate to another path. const hugoNavigatePrefix = "__hugo_navigate" -var upgrader = &websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024} +var upgrader = &websocket.Upgrader{ + // Hugo may potentially spin up multiple HTTP servers, so we need to exclude the + // port when checking the origin. + CheckOrigin: func(r *http.Request) bool { + origin := r.Header["Origin"] + if len(origin) == 0 { + return true + } + u, err := url.Parse(origin[0]) + if err != nil { + return false + } + + h1, _, err := net.SplitHostPort(u.Host) + if err != nil { + return false + } + h2, _, err := net.SplitHostPort(r.Host) + if err != nil { + return false + } + + return h1 == h2 + }, + ReadBufferSize: 1024, WriteBufferSize: 1024} // Handler is a HandlerFunc handling the livereload // Websocket interaction. @@ -79,13 +105,28 @@ func NavigateToPath(path string) { RefreshPath(hugoNavigatePrefix + path) } +// NavigateToPathForPort is similar to NavigateToPath but will also +// set window.location.port to the given port value. +func NavigateToPathForPort(path string, port int) { + refreshPathForPort(hugoNavigatePrefix+path, port) +} + // RefreshPath tells livereload to refresh only the given path. // If that path points to a CSS stylesheet or an image, only the changes // will be updated in the browser, not the entire page. func RefreshPath(s string) { + refreshPathForPort(s, -1) +} + +func refreshPathForPort(s string, port int) { // Tell livereload a file has changed - will force a hard refresh if not CSS or an image urlPath := filepath.ToSlash(s) - wsHub.broadcast <- []byte(`{"command":"reload","path":"` + urlPath + `","originalPath":"","liveCSS":true,"liveImg":true}`) + portStr := "" + if port > 0 { + portStr = fmt.Sprintf(`, "overrideURL": %d`, port) + } + msg := fmt.Sprintf(`{"command":"reload","path":%q,"originalPath":"","liveCSS":true,"liveImg":true%s}`, urlPath, portStr) + wsHub.broadcast <- []byte(msg) } // ServeJS serves the liverreload.js who's reference is injected into the page. @@ -120,13 +161,17 @@ HugoReload.prototype.reload = function(path, options) { if (path.lastIndexOf(prefix, 0) !== 0) { return false } - + path = path.substring(prefix.length); - - if (window.location.pathname === path) { + + if (!options.overrideURL && window.location.pathname === path) { window.location.reload(); } else { - window.location.href = path; + if (options.overrideURL) { + window.location = location.protocol + "//" + location.hostname + ":" + options.overrideURL + path; + } else { + window.location.pathname = path; + } } return true; diff --git a/source/dirs.go b/source/dirs.go new file mode 100644 index 000000000..1e6850da7 --- /dev/null +++ b/source/dirs.go @@ -0,0 +1,191 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "github.com/spf13/afero" + + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugofs" + jww "github.com/spf13/jwalterweatherman" +) + +// Dirs holds the source directories for a given build. +// In case where there are more than one of a kind, the order matters: +// It will be used to construct a union filesystem, so the right-most directory +// will "win" on duplicates. Typically, the theme version will be the first. +type Dirs struct { + logger *jww.Notepad + pathSpec *helpers.PathSpec + + staticDirs []string + AbsStaticDirs []string + + publishDir string +} + +// NewDirs creates a new dirs with the given configuration and filesystem. +func NewDirs(fs *hugofs.Fs, cfg config.Provider, logger *jww.Notepad) (*Dirs, error) { + ps, err := helpers.NewPathSpec(fs, cfg) + if err != nil { + return nil, err + } + + d := &Dirs{pathSpec: ps, logger: logger} + + return d, d.init(cfg) + +} + +func (d *Dirs) init(cfg config.Provider) error { + + var ( + statics []string + ) + + if d.pathSpec.Theme() != "" { + statics = append(statics, filepath.Join(d.pathSpec.ThemesDir(), d.pathSpec.Theme(), "static")) + } + + _, isLanguage := cfg.(*helpers.Language) + languages, hasLanguages := cfg.Get("languagesSorted").(helpers.Languages) + + if !isLanguage && !hasLanguages { + return errors.New("missing languagesSorted in config") + } + + if !isLanguage { + // Merge all the static dirs. + for _, l := range languages { + addend, err := d.staticDirsFor(l) + if err != nil { + return err + } + + statics = append(statics, addend...) + } + } else { + addend, err := d.staticDirsFor(cfg) + if err != nil { + return err + } + + statics = append(statics, addend...) + } + + d.staticDirs = removeDuplicatesKeepRight(statics) + d.AbsStaticDirs = make([]string, len(d.staticDirs)) + for i, di := range d.staticDirs { + d.AbsStaticDirs[i] = d.pathSpec.AbsPathify(di) + helpers.FilePathSeparator + } + + d.publishDir = d.pathSpec.AbsPathify(cfg.GetString("publishDir")) + helpers.FilePathSeparator + + return nil +} + +func (d *Dirs) staticDirsFor(cfg config.Provider) ([]string, error) { + var statics []string + ps, err := helpers.NewPathSpec(d.pathSpec.Fs, cfg) + if err != nil { + return statics, err + } + + statics = append(statics, ps.StaticDirs()...) + + return statics, nil +} + +// CreateStaticFs will create a union filesystem with the static paths configured. +// Any missing directories will be logged as warnings. +func (d *Dirs) CreateStaticFs() (afero.Fs, error) { + var ( + source = d.pathSpec.Fs.Source + absPaths []string + ) + + for _, staticDir := range d.AbsStaticDirs { + if _, err := source.Stat(staticDir); os.IsNotExist(err) { + d.logger.WARN.Printf("Unable to find Static Directory: %s", staticDir) + } else { + absPaths = append(absPaths, staticDir) + } + + } + + if len(absPaths) == 0 { + return nil, nil + } + + return d.createOverlayFs(absPaths), nil + +} + +// IsStatic returns whether the given filename is located in one of the static +// source dirs. +func (d *Dirs) IsStatic(filename string) bool { + for _, absPath := range d.AbsStaticDirs { + if strings.HasPrefix(filename, absPath) { + return true + } + } + return false +} + +// MakeStaticPathRelative creates a relative path from the given filename. +// It will return an empty string if the filename is not a member of dirs. +func (d *Dirs) MakeStaticPathRelative(filename string) string { + for _, currentPath := range d.AbsStaticDirs { + if strings.HasPrefix(filename, currentPath) { + return strings.TrimPrefix(filename, currentPath) + } + } + + return "" + +} + +func (d *Dirs) createOverlayFs(absPaths []string) afero.Fs { + source := d.pathSpec.Fs.Source + + if len(absPaths) == 1 { + return afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) + } + + base := afero.NewReadOnlyFs(afero.NewBasePathFs(source, absPaths[0])) + overlay := d.createOverlayFs(absPaths[1:]) + + return afero.NewCopyOnWriteFs(base, overlay) +} + +func removeDuplicatesKeepRight(in []string) []string { + seen := make(map[string]bool) + var out []string + for i := len(in) - 1; i >= 0; i-- { + v := in[i] + if seen[v] { + continue + } + out = append([]string{v}, out...) + seen[v] = true + } + + return out +} diff --git a/source/dirs_test.go b/source/dirs_test.go new file mode 100644 index 000000000..0d8eacf56 --- /dev/null +++ b/source/dirs_test.go @@ -0,0 +1,177 @@ +// Copyright 2017 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "testing" + + "github.com/gohugoio/hugo/helpers" + + "fmt" + + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/gohugoio/hugo/config" + "github.com/spf13/afero" + + jww "github.com/spf13/jwalterweatherman" + + "github.com/gohugoio/hugo/hugofs" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +var logger = jww.NewNotepad(jww.LevelInfo, jww.LevelError, os.Stdout, ioutil.Discard, "", log.Ldate|log.Ltime) + +func TestStaticDirs(t *testing.T) { + assert := require.New(t) + + tests := []struct { + setup func(cfg config.Provider, fs *hugofs.Fs) config.Provider + expected []string + }{ + + {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { + cfg.Set("staticDir", "s1") + return cfg + }, []string{"s1"}}, + {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { + cfg.Set("staticDir", []string{"s2", "s1", "s2"}) + return cfg + }, []string{"s1", "s2"}}, + {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { + cfg.Set("theme", "mytheme") + cfg.Set("themesDir", "themes") + cfg.Set("staticDir", []string{"s1", "s2"}) + return cfg + }, []string{filepath.FromSlash("themes/mytheme/static"), "s1", "s2"}}, + {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { + cfg.Set("staticDir", "s1") + + l1 := helpers.NewLanguage("en", cfg) + l1.Set("staticDir", []string{"l1s1", "l1s2"}) + return l1 + + }, []string{"l1s1", "l1s2"}}, + {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { + cfg.Set("staticDir", "s1") + + l1 := helpers.NewLanguage("en", cfg) + l1.Set("staticDir2", []string{"l1s1", "l1s2"}) + return l1 + + }, []string{"s1", "l1s1", "l1s2"}}, + {func(cfg config.Provider, fs *hugofs.Fs) config.Provider { + cfg.Set("staticDir", "s1") + + l1 := helpers.NewLanguage("en", cfg) + l1.Set("staticDir2", []string{"l1s1", "l1s2"}) + l2 := helpers.NewLanguage("nn", cfg) + l2.Set("staticDir3", []string{"l2s1", "l2s2"}) + l2.Set("staticDir", []string{"l2"}) + + cfg.Set("languagesSorted", helpers.Languages{l1, l2}) + return cfg + + }, []string{"s1", "l1s1", "l1s2", "l2", "l2s1", "l2s2"}}, + } + + for i, test := range tests { + if i != 0 { + break + } + msg := fmt.Sprintf("Test %d", i) + v := viper.New() + fs := hugofs.NewMem(v) + cfg := test.setup(v, fs) + cfg.Set("workingDir", filepath.FromSlash("/work")) + _, isLanguage := cfg.(*helpers.Language) + if !isLanguage && !cfg.IsSet("languagesSorted") { + cfg.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(cfg)}) + } + dirs, err := NewDirs(fs, cfg, logger) + assert.NoError(err) + assert.Equal(test.expected, dirs.staticDirs, msg) + assert.Len(dirs.AbsStaticDirs, len(dirs.staticDirs)) + + for i, d := range dirs.staticDirs { + abs := dirs.AbsStaticDirs[i] + assert.Equal(filepath.Join("/work", d)+helpers.FilePathSeparator, abs) + assert.True(dirs.IsStatic(filepath.Join(abs, "logo.png"))) + rel := dirs.MakeStaticPathRelative(filepath.Join(abs, "logo.png")) + assert.Equal("logo.png", rel) + } + + assert.False(dirs.IsStatic(filepath.FromSlash("/some/other/dir/logo.png"))) + + } + +} + +func TestStaticDirsFs(t *testing.T) { + assert := require.New(t) + v := viper.New() + fs := hugofs.NewMem(v) + v.Set("workingDir", filepath.FromSlash("/work")) + v.Set("theme", "mytheme") + v.Set("themesDir", "themes") + v.Set("staticDir", []string{"s1", "s2"}) + v.Set("languagesSorted", helpers.Languages{helpers.NewDefaultLanguage(v)}) + + writeToFs(t, fs.Source, "/work/s1/f1.txt", "s1-f1") + writeToFs(t, fs.Source, "/work/s2/f2.txt", "s2-f2") + writeToFs(t, fs.Source, "/work/s1/f2.txt", "s1-f2") + writeToFs(t, fs.Source, "/work/themes/mytheme/static/f1.txt", "theme-f1") + writeToFs(t, fs.Source, "/work/themes/mytheme/static/f3.txt", "theme-f3") + + dirs, err := NewDirs(fs, v, logger) + assert.NoError(err) + + sfs, err := dirs.CreateStaticFs() + assert.NoError(err) + + assert.Equal("s1-f1", readFileFromFs(t, sfs, "f1.txt")) + assert.Equal("s2-f2", readFileFromFs(t, sfs, "f2.txt")) + assert.Equal("theme-f3", readFileFromFs(t, sfs, "f3.txt")) + +} + +func TestRemoveDuplicatesKeepRight(t *testing.T) { + in := []string{"a", "b", "c", "a"} + out := removeDuplicatesKeepRight(in) + + require.Equal(t, []string{"b", "c", "a"}, out) +} + +func writeToFs(t testing.TB, fs afero.Fs, filename, content string) { + if err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte(content), 0755); err != nil { + t.Fatalf("Failed to write file: %s", err) + } +} + +func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { + filename = filepath.FromSlash(filename) + b, err := afero.ReadFile(fs, filename) + if err != nil { + afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error { + fmt.Println(" ", path, " ", info) + return nil + }) + t.Fatalf("Failed to read file: %s", err) + } + return string(b) +} diff --git a/tpl/urls/init_test.go b/tpl/urls/init_test.go index 6630f13d3..a678ee6b1 100644 --- a/tpl/urls/init_test.go +++ b/tpl/urls/init_test.go @@ -18,6 +18,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/tpl/internal" + "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -26,7 +27,7 @@ func TestInit(t *testing.T) { var ns *internal.TemplateFuncsNamespace for _, nsf := range internal.TemplateFuncsNamespaceRegistry { - ns = nsf(&deps.Deps{}) + ns = nsf(&deps.Deps{Cfg: viper.New()}) if ns.Name == name { found = true break diff --git a/tpl/urls/urls.go b/tpl/urls/urls.go index d89069901..a9f8f4f76 100644 --- a/tpl/urls/urls.go +++ b/tpl/urls/urls.go @@ -26,13 +26,15 @@ import ( // New returns a new instance of the urls-namespaced template functions. func New(deps *deps.Deps) *Namespace { return &Namespace{ - deps: deps, + deps: deps, + multihost: deps.Cfg.GetBool("multihost"), } } // Namespace provides template functions for the "urls" namespace. type Namespace struct { - deps *deps.Deps + deps *deps.Deps + multihost bool } // AbsURL takes a given string and converts it to an absolute URL. @@ -109,7 +111,7 @@ func (ns *Namespace) RelLangURL(a interface{}) (template.HTML, error) { return "", err } - return template.HTML(ns.deps.PathSpec.RelURL(s, true)), nil + return template.HTML(ns.deps.PathSpec.RelURL(s, !ns.multihost)), nil } // AbsLangURL takes a given string and converts it to an absolute URL according @@ -121,5 +123,5 @@ func (ns *Namespace) AbsLangURL(a interface{}) (template.HTML, error) { return "", err } - return template.HTML(ns.deps.PathSpec.AbsURL(s, true)), nil + return template.HTML(ns.deps.PathSpec.AbsURL(s, !ns.multihost)), nil }