Add multilingual multihost support

This commit adds multihost support when more than one language is configured and `baseURL` is set per language.

Updates #4027

commands/commandeer.go

@@ -41,6 +41,10 @@ func (c *commandeer) PathSpec() *helpers.PathSpec {
 	return c.pathSpec
 }
 
+func (c *commandeer) languages() helpers.Languages {
+	return c.Cfg.Get("languagesSorted").(helpers.Languages)
+}
+
 func (c *commandeer) initFs(fs *hugofs.Fs) error {
 	c.DepsCfg.Fs = fs
 	ps, err := helpers.NewPathSpec(fs, c.Cfg)

commands/hugo.go

@@ -526,6 +526,7 @@ func (c *commandeer) watchConfig() {
 
 func (c *commandeer) build(watches ...bool) error {
 	if err := c.copyStatic(); err != nil {
+		// TODO(bep) multihost
 		return fmt.Errorf("Error copying static files to %s: %s", c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")), err)
 	}
 	watch := false
@@ -593,6 +594,24 @@ func (c *commandeer) getStaticSourceFs() afero.Fs {
 
 func (c *commandeer) copyStatic() error {
 	publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator
+	roots := c.roots()
+
+	if len(roots) == 0 {
+		return c.copyStaticTo(publishDir)
+	}
+
+	for _, root := range roots {
+		dir := filepath.Join(publishDir, root)
+		if err := c.copyStaticTo(dir); err != nil {
+			return err
+		}
+	}
+
+	return nil
+
+}
+
+func (c *commandeer) copyStaticTo(publishDir string) error {
 
 	// If root, remove the second '/'
 	if publishDir == "//" {
@@ -893,6 +912,7 @@ func (c *commandeer) newWatcher(port int) error {
 
 					if c.Cfg.GetBool("forceSyncStatic") {
 						c.Logger.FEEDBACK.Printf("Syncing all static files\n")
+						// TODO(bep) multihost
 						err := c.copyStatic()
 						if err != nil {
 							utils.StopOnErr(c.Logger, err, fmt.Sprintf("Error copying static files to %s", publishDir))

commands/server.go

@@ -19,12 +19,14 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/gohugoio/hugo/config"
+
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
@@ -137,11 +139,22 @@ func server(cmd *cobra.Command, args []string) error {
 		c.watchConfig()
 	}
 
-	l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(serverPort)))
+	languages := c.languages()
+	serverPorts := make([]int, 1)
+
+	if languages.IsMultihost() {
+		serverPorts = make([]int, len(languages))
+	}
+
+	currentServerPort := serverPort
+
+	for i := 0; i < len(serverPorts); i++ {
+		l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(currentServerPort)))
 		if err == nil {
 			l.Close()
+			serverPorts[i] = currentServerPort
 		} else {
-		if serverCmd.Flags().Changed("port") {
+			if i == 0 && serverCmd.Flags().Changed("port") {
 				// port set explicitly by user -- he/she probably meant it!
 				return newSystemErrorF("Server startup failed: %s", err)
 			}
@@ -150,21 +163,34 @@ func server(cmd *cobra.Command, args []string) error {
 			if err != nil {
 				return newSystemError("Unable to find alternative port to use:", err)
 			}
-		serverPort = sp.Port
+			serverPorts[i] = sp.Port
+		}
+
+		currentServerPort = serverPorts[i] + 1
 	}
 
 	c.Set("port", serverPort)
 	if liveReloadPort != -1 {
 		c.Set("liveReloadPort", liveReloadPort)
 	} else {
-		c.Set("liveReloadPort", serverPort)
+		c.Set("liveReloadPort", serverPorts[0])
 	}
 
-	baseURL, err = fixURL(c.Cfg, baseURL)
+	if languages.IsMultihost() {
+		for i, language := range languages {
+			baseURL, err = fixURL(language, baseURL, serverPorts[i])
 			if err != nil {
 				return err
 			}
-	c.Set("baseURL", baseURL)
+			language.Set("baseURL", baseURL)
+		}
+	} else {
+		baseURL, err = fixURL(c.Cfg, baseURL, serverPorts[0])
+		if err != nil {
+			return err
+		}
+		c.Cfg.Set("baseURL", baseURL)
+	}
 
 	if err := memStats(); err != nil {
 		jww.ERROR.Println("memstats error:", err)
@@ -208,28 +234,52 @@ func server(cmd *cobra.Command, args []string) error {
 		}
 	}
 
-	c.serve(serverPort)
-
 	return nil
 }
 
-func (c *commandeer) serve(port int) {
+type fileServer struct {
+	basePort int
+	baseURLs []string
+	roots    []string
+	c        *commandeer
+}
+
+func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) {
+	baseURL := f.baseURLs[i]
+	root := f.roots[i]
+	port := f.basePort + i
+
+	publishDir := f.c.Cfg.GetString("publishDir")
+
+	if root != "" {
+		publishDir = filepath.Join(publishDir, root)
+	}
+
+	absPublishDir := f.c.PathSpec().AbsPathify(publishDir)
+
+	// TODO(bep) multihost unify feedback
 	if renderToDisk {
-		jww.FEEDBACK.Println("Serving pages from " + c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")))
+		jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
 	} else {
 		jww.FEEDBACK.Println("Serving pages from memory")
 	}
 
-	httpFs := afero.NewHttpFs(c.Fs.Destination)
+	httpFs := afero.NewHttpFs(f.c.Fs.Destination)
-	fs := filesOnlyFs{httpFs.Dir(c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")))}
+	fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
 
-	doLiveReload := !buildWatch && !c.Cfg.GetBool("disableLiveReload")
+	doLiveReload := !buildWatch && !f.c.Cfg.GetBool("disableLiveReload")
-	fastRenderMode := doLiveReload && !c.Cfg.GetBool("disableFastRender")
+	fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender")
 
 	if fastRenderMode {
 		jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
 	}
 
+	// We're only interested in the path
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, "", fmt.Errorf("Invalid baseURL: %s", err)
+	}
+
 	decorate := func(h http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			if noHTTPCache {
@@ -240,7 +290,7 @@ func (c *commandeer) serve(port int) {
 			if fastRenderMode {
 				p := r.RequestURI
 				if strings.HasSuffix(p, "/") || strings.HasSuffix(p, "html") || strings.HasSuffix(p, "htm") {
-					c.visitedURLs.Add(p)
+					f.c.visitedURLs.Add(p)
 				}
 			}
 			h.ServeHTTP(w, r)
@@ -248,32 +298,78 @@ func (c *commandeer) serve(port int) {
 	}
 
 	fileserver := decorate(http.FileServer(fs))
+	mu := http.NewServeMux()
 
-	// We're only interested in the path
-	u, err := url.Parse(c.Cfg.GetString("baseURL"))
-	if err != nil {
-		jww.ERROR.Fatalf("Invalid baseURL: %s", err)
-	}
 	if u.Path == "" || u.Path == "/" {
-		http.Handle("/", fileserver)
+		mu.Handle("/", fileserver)
 	} else {
-		http.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
+		mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
 	}
 
-	jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface)
-	jww.FEEDBACK.Println("Press Ctrl+C to stop")
-
 	endpoint := net.JoinHostPort(serverInterface, strconv.Itoa(port))
-	err = http.ListenAndServe(endpoint, nil)
+
+	return mu, endpoint, nil
+}
+
+func (c *commandeer) roots() []string {
+	var roots []string
+	languages := c.languages()
+	isMultiHost := languages.IsMultihost()
+	if !isMultiHost {
+		return roots
+	}
+
+	for _, l := range languages {
+		roots = append(roots, l.Lang)
+	}
+	return roots
+}
+
+func (c *commandeer) serve(port int) {
+	// TODO(bep) multihost
+	isMultiHost := Hugo.IsMultihost()
+
+	var (
+		baseURLs []string
+		roots    []string
+	)
+
+	if isMultiHost {
+		for _, s := range Hugo.Sites {
+			baseURLs = append(baseURLs, s.BaseURL.String())
+			roots = append(roots, s.Language.Lang)
+		}
+	} else {
+		baseURLs = []string{Hugo.Sites[0].BaseURL.String()}
+		roots = []string{""}
+	}
+
+	srv := &fileServer{
+		basePort: port,
+		baseURLs: baseURLs,
+		roots:    roots,
+		c:        c,
+	}
+
+	for i, _ := range baseURLs {
+		mu, endpoint, err := srv.createEndpoint(i)
+
+		go func() {
+			err = http.ListenAndServe(endpoint, mu)
 			if err != nil {
 				jww.ERROR.Printf("Error: %s\n", err.Error())
 				os.Exit(1)
 			}
+		}()
+	}
+
+	// TODO(bep) multihost		jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface)
+	jww.FEEDBACK.Println("Press Ctrl+C to stop")
 }
 
 // fixURL massages the baseURL into a form needed for serving
 // all pages correctly.
-func fixURL(cfg config.Provider, s string) (string, error) {
+func fixURL(cfg config.Provider, s string, port int) (string, error) {
 	useLocalhost := false
 	if s == "" {
 		s = cfg.GetString("baseURL")
@@ -315,7 +411,7 @@ func fixURL(cfg config.Provider, s string) (string, error) {
 				return "", fmt.Errorf("Failed to split baseURL hostpost: %s", err)
 			}
 		}
-		u.Host += fmt.Sprintf(":%d", serverPort)
+		u.Host += fmt.Sprintf(":%d", port)
 	}
 
 	return u.String(), nil

commands/server_test.go

@@ -47,7 +47,7 @@ func TestFixURL(t *testing.T) {
 		v.Set("baseURL", test.CfgBaseURL)
 		serverAppend = test.AppendPort
 		serverPort = test.Port
-		result, err := fixURL(v, baseURL)
+		result, err := fixURL(v, baseURL, serverPort)
 		if err != nil {
 			t.Errorf("Test #%d %s: unexpected error %s", i, test.TestName, err)
 		}

helpers/language.go

@@ -102,6 +102,17 @@ func (l *Language) Params() map[string]interface{} {
 	return l.params
 }
 
+// IsMultihost returns whether the languages has baseURL specificed on the
+// language level.
+func (l Languages) IsMultihost() bool {
+	for _, lang := range l {
+		if lang.GetLocal("baseURL") != nil {
+			return true
+		}
+	}
+	return false
+}
+
 // SetParam sets param with the given key and value.
 // SetParam is case-insensitive.
 func (l *Language) SetParam(k string, v interface{}) {
@@ -132,6 +143,17 @@ func (l *Language) GetStringMapString(key string) map[string]string {
 //
 // Get returns an interface. For a specific value use one of the Get____ methods.
 func (l *Language) Get(key string) interface{} {
+	local := l.GetLocal(key)
+	if local != nil {
+		return local
+	}
+	return l.Cfg.Get(key)
+}
+
+// GetLocal gets a configuration value set on language level. It will
+// not fall back to any global value.
+// It will return nil if a value with the given key cannot be found.
+func (l *Language) GetLocal(key string) interface{} {
 	if l == nil {
 		panic("language not set")
 	}
@@ -141,7 +163,7 @@ func (l *Language) Get(key string) interface{} {
 			return v
 		}
 	}
-	return l.Cfg.Get(key)
+	return nil
 }
 
 // Set sets the value for the key in the language's params.

helpers/path.go

@@ -158,7 +158,6 @@ func (p *PathSpec) AbsPathify(inPath string) string {
 		return filepath.Clean(inPath)
 	}
 
-	// TODO(bep): Consider moving workingDir to argument list
 	return filepath.Join(p.workingDir, inPath)
 }
 

hugolib/config.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"strings"
 
+	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/afero"
 	"github.com/spf13/viper"
@@ -80,11 +81,34 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper.
 		helpers.Deprecated("site config", "disableRobotsTXT", "Use disableKinds= [\"robotsTXT\"]", false)
 	}
 
-	loadDefaultSettingsFor(v)
+	if err := loadDefaultSettingsFor(v); err != nil {
+		return v, err
+	}
 
 	return v, nil
 }
 
+func loadLanguageSettings(cfg config.Provider) error {
+	multilingual := cfg.GetStringMap("languages")
+	var (
+		langs helpers.Languages
+		err   error
+	)
+
+	if len(multilingual) == 0 {
+		langs = append(langs, helpers.NewDefaultLanguage(cfg))
+	} else {
+		langs, err = toSortedLanguages(cfg, multilingual)
+		if err != nil {
+			return fmt.Errorf("Failed to parse multilingual config: %s", err)
+		}
+	}
+
+	cfg.Set("languagesSorted", langs)
+
+	return nil
+}
+
 func loadDefaultSettingsFor(v *viper.Viper) error {
 
 	c, err := helpers.NewContentSpec(v)
@@ -154,5 +178,5 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("debug", false)
 	v.SetDefault("disableFastRender", false)
 
-	return nil
+	return loadLanguageSettings(v)
 }

hugolib/hugo_sites.go

@@ -15,7 +15,6 @@ package hugolib
 
 import (
 	"errors"
-	"fmt"
 	"strings"
 	"sync"
 
@@ -37,9 +36,16 @@ type HugoSites struct {
 
 	multilingual *Multilingual
 
+	// Multihost is set if multilingual and baseURL set on the language level.
+	multihost bool
+
 	*deps.Deps
 }
 
+func (h *HugoSites) IsMultihost() bool {
+	return h != nil && h.multihost
+}
+
 // GetContentPage finds a Page with content given the absolute filename.
 // Returns nil if none found.
 func (h *HugoSites) GetContentPage(filename string) *Page {
@@ -92,6 +98,31 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
 
 	h.Deps = sites[0].Deps
 
+	// The baseURL may be provided at the language level. If that is true,
+	// then every language must have a baseURL. In this case we always render
+	// to a language sub folder, which is then stripped from all the Permalink URLs etc.
+	var baseURLFromLang bool
+
+	for _, s := range sites {
+		burl := s.Language.GetLocal("baseURL")
+		if baseURLFromLang && burl == nil {
+			return h, errors.New("baseURL must be set on all or none of the languages")
+		}
+
+		if burl != nil {
+			baseURLFromLang = true
+		}
+	}
+
+	if baseURLFromLang {
+		for _, s := range sites {
+			// TODO(bep) multihost check
+			s.Info.defaultContentLanguageInSubdir = true
+			s.Cfg.Set("defaultContentLanguageInSubdir", true)
+		}
+		h.multihost = true
+	}
+
 	return h, nil
 }
 
@@ -180,26 +211,7 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
 		sites []*Site
 	)
 
-	multilingual := cfg.Cfg.GetStringMap("languages")
+	languages := getLanguages(cfg.Cfg)
-
-	if len(multilingual) == 0 {
-		l := helpers.NewDefaultLanguage(cfg.Cfg)
-		cfg.Language = l
-		s, err := newSite(cfg)
-		if err != nil {
-			return nil, err
-		}
-		sites = append(sites, s)
-	}
-
-	if len(multilingual) > 0 {
-		var err error
-
-		languages, err := toSortedLanguages(cfg.Cfg, multilingual)
-
-		if err != nil {
-			return nil, fmt.Errorf("Failed to parse multilingual config: %s", err)
-		}
 
 	for _, lang := range languages {
 		var s *Site
@@ -213,7 +225,6 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
 
 		sites = append(sites, s)
 	}
-	}
 
 	return sites, nil
 }
@@ -227,7 +238,12 @@ func (h *HugoSites) reset() {
 
 func (h *HugoSites) createSitesFromConfig() error {
 
+	if err := loadLanguageSettings(h.Cfg); err != nil {
+		return err
+	}
+
 	depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg}
+
 	sites, err := createSitesFromConfig(depsCfg)
 
 	if err != nil {
@@ -286,7 +302,7 @@ type BuildCfg struct {
 
 func (h *HugoSites) renderCrossSitesArtifacts() error {
 
-	if !h.multilingual.enabled() {
+	if !h.multilingual.enabled() || h.IsMultihost() {
 		return nil
 	}
 

hugolib/hugo_sites_build_test.go

@@ -1269,7 +1269,7 @@ lag:
 		t.Fatalf("Failed to create sites: %s", err)
 	}
 
-	if len(sites.Sites) != 4 {
+	if len(sites.Sites) == 0 {
 		t.Fatalf("Got %d sites", len(sites.Sites))
 	}
 

hugolib/hugo_sites_multihost_test.go

@@ -0,0 +1,72 @@
+package hugolib
+
+import (
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMultihosts(t *testing.T) {
+	t.Parallel()
+
+	var multiSiteTOMLConfigTemplate = `
+paginate = 1
+disablePathToLower = true
+defaultContentLanguage = "{{ .DefaultContentLanguage }}"
+defaultContentLanguageInSubdir = {{ .DefaultContentLanguageInSubdir }}
+
+[permalinks]
+other = "/somewhere/else/:filename"
+
+[Taxonomies]
+tag = "tags"
+
+[Languages]
+[Languages.en]
+baseURL = "https://example.com"
+weight = 10
+title = "In English"
+languageName = "English"
+
+[Languages.fr]
+baseURL = "https://example.fr"
+weight = 20
+title = "Le Français"
+languageName = "Français"
+
+[Languages.nn]
+baseURL = "https://example.no"
+weight = 30
+title = "På nynorsk"
+languageName = "Nynorsk"
+
+`
+
+	siteConfig := testSiteConfig{Fs: afero.NewMemMapFs(), DefaultContentLanguage: "fr", DefaultContentLanguageInSubdir: false}
+	sites := createMultiTestSites(t, siteConfig, multiSiteTOMLConfigTemplate)
+	fs := sites.Fs
+	cfg := BuildCfg{Watching: true}
+	th := testHelper{sites.Cfg, fs, t}
+	assert := require.New(t)
+
+	err := sites.Build(cfg)
+	assert.NoError(err)
+
+	th.assertFileContent("public/en/sect/doc1-slug/index.html", "Hello")
+
+	s1 := sites.Sites[0]
+
+	s1h := s1.getPage(KindHome)
+	assert.True(s1h.IsTranslated())
+	assert.Len(s1h.Translations(), 2)
+	assert.Equal("https://example.com/", s1h.Permalink())
+
+	s2 := sites.Sites[1]
+	s2h := s2.getPage(KindHome)
+	assert.Equal("https://example.fr/", s2h.Permalink())
+
+	th.assertFileContentStraight("public/fr/index.html", "French Home Page")
+	th.assertFileContentStraight("public/en/index.html", "Default Home Page")
+
+}

hugolib/multilingual.go

@@ -47,6 +47,14 @@ func (ml *Multilingual) Language(lang string) *helpers.Language {
 	return ml.langMap[lang]
 }
 
+func getLanguages(cfg config.Provider) helpers.Languages {
+	if cfg.IsSet("languagesSorted") {
+		return cfg.Get("languagesSorted").(helpers.Languages)
+	}
+
+	return helpers.Languages{helpers.NewDefaultLanguage(cfg)}
+}
+
 func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingual, error) {
 	languages := make(helpers.Languages, len(sites))
 

hugolib/page.go

@@ -1754,6 +1754,11 @@ func (p *Page) shouldAddLanguagePrefix() bool {
 		return false
 	}
 
+	if p.s.owner.IsMultihost() {
+		// TODO(bep) multihost check vs lang below
+		return true
+	}
+
 	if p.Lang() == "" {
 		return false
 	}

hugolib/page_paths.go

@@ -257,6 +257,10 @@ func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string {
 		tp = strings.TrimSuffix(tp, f.BaseFilename())
 	}
 
+	if p.s.owner.IsMultihost() {
+		tp = strings.TrimPrefix(tp, helpers.FilePathSeparator+p.s.Info.Language.Lang)
+	}
+
 	return p.s.PathSpec.URLizeFilename(tp)
 }
 

hugolib/site_render.go

@@ -387,7 +387,7 @@ func (s *Site) renderAliases() error {
 		}
 	}
 
-	if s.owner.multilingual.enabled() {
+	if s.owner.multilingual.enabled() && !s.owner.IsMultihost() {
 		mainLang := s.owner.multilingual.DefaultLang
 		if s.Info.defaultContentLanguageInSubdir {
 			mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false)