Add multilingual multihost support

This commit adds multihost support when more than one language is configured and `baseURL` is set per language.

Updates #4027
This commit is contained in:
Bjørn Erik Pedersen 2017-11-02 08:25:20 +01:00
parent 6233ddf9d1
commit 2e0465764b
14 changed files with 350 additions and 80 deletions

View file

@ -41,6 +41,10 @@ func (c *commandeer) PathSpec() *helpers.PathSpec {
return c.pathSpec
}
func (c *commandeer) languages() helpers.Languages {
return c.Cfg.Get("languagesSorted").(helpers.Languages)
}
func (c *commandeer) initFs(fs *hugofs.Fs) error {
c.DepsCfg.Fs = fs
ps, err := helpers.NewPathSpec(fs, c.Cfg)

View file

@ -526,6 +526,7 @@ func (c *commandeer) watchConfig() {
func (c *commandeer) build(watches ...bool) error {
if err := c.copyStatic(); err != nil {
// TODO(bep) multihost
return fmt.Errorf("Error copying static files to %s: %s", c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")), err)
}
watch := false
@ -593,6 +594,24 @@ func (c *commandeer) getStaticSourceFs() afero.Fs {
func (c *commandeer) copyStatic() error {
publishDir := c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")) + helpers.FilePathSeparator
roots := c.roots()
if len(roots) == 0 {
return c.copyStaticTo(publishDir)
}
for _, root := range roots {
dir := filepath.Join(publishDir, root)
if err := c.copyStaticTo(dir); err != nil {
return err
}
}
return nil
}
func (c *commandeer) copyStaticTo(publishDir string) error {
// If root, remove the second '/'
if publishDir == "//" {
@ -893,6 +912,7 @@ func (c *commandeer) newWatcher(port int) error {
if c.Cfg.GetBool("forceSyncStatic") {
c.Logger.FEEDBACK.Printf("Syncing all static files\n")
// TODO(bep) multihost
err := c.copyStatic()
if err != nil {
utils.StopOnErr(c.Logger, err, fmt.Sprintf("Error copying static files to %s", publishDir))

View file

@ -19,12 +19,14 @@ import (
"net/http"
"net/url"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
"time"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/afero"
"github.com/spf13/cobra"
@ -137,34 +139,58 @@ func server(cmd *cobra.Command, args []string) error {
c.watchConfig()
}
l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(serverPort)))
if err == nil {
l.Close()
} else {
if serverCmd.Flags().Changed("port") {
// port set explicitly by user -- he/she probably meant it!
return newSystemErrorF("Server startup failed: %s", err)
languages := c.languages()
serverPorts := make([]int, 1)
if languages.IsMultihost() {
serverPorts = make([]int, len(languages))
}
currentServerPort := serverPort
for i := 0; i < len(serverPorts); i++ {
l, err := net.Listen("tcp", net.JoinHostPort(serverInterface, strconv.Itoa(currentServerPort)))
if err == nil {
l.Close()
serverPorts[i] = currentServerPort
} else {
if i == 0 && serverCmd.Flags().Changed("port") {
// port set explicitly by user -- he/she probably meant it!
return newSystemErrorF("Server startup failed: %s", err)
}
jww.ERROR.Println("port", serverPort, "already in use, attempting to use an available port")
sp, err := helpers.FindAvailablePort()
if err != nil {
return newSystemError("Unable to find alternative port to use:", err)
}
serverPorts[i] = sp.Port
}
jww.ERROR.Println("port", serverPort, "already in use, attempting to use an available port")
sp, err := helpers.FindAvailablePort()
if err != nil {
return newSystemError("Unable to find alternative port to use:", err)
}
serverPort = sp.Port
currentServerPort = serverPorts[i] + 1
}
c.Set("port", serverPort)
if liveReloadPort != -1 {
c.Set("liveReloadPort", liveReloadPort)
} else {
c.Set("liveReloadPort", serverPort)
c.Set("liveReloadPort", serverPorts[0])
}
baseURL, err = fixURL(c.Cfg, baseURL)
if err != nil {
return err
if languages.IsMultihost() {
for i, language := range languages {
baseURL, err = fixURL(language, baseURL, serverPorts[i])
if err != nil {
return err
}
language.Set("baseURL", baseURL)
}
} else {
baseURL, err = fixURL(c.Cfg, baseURL, serverPorts[0])
if err != nil {
return err
}
c.Cfg.Set("baseURL", baseURL)
}
c.Set("baseURL", baseURL)
if err := memStats(); err != nil {
jww.ERROR.Println("memstats error:", err)
@ -208,28 +234,52 @@ func server(cmd *cobra.Command, args []string) error {
}
}
c.serve(serverPort)
return nil
}
func (c *commandeer) serve(port int) {
type fileServer struct {
basePort int
baseURLs []string
roots []string
c *commandeer
}
func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, error) {
baseURL := f.baseURLs[i]
root := f.roots[i]
port := f.basePort + i
publishDir := f.c.Cfg.GetString("publishDir")
if root != "" {
publishDir = filepath.Join(publishDir, root)
}
absPublishDir := f.c.PathSpec().AbsPathify(publishDir)
// TODO(bep) multihost unify feedback
if renderToDisk {
jww.FEEDBACK.Println("Serving pages from " + c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")))
jww.FEEDBACK.Println("Serving pages from " + absPublishDir)
} else {
jww.FEEDBACK.Println("Serving pages from memory")
}
httpFs := afero.NewHttpFs(c.Fs.Destination)
fs := filesOnlyFs{httpFs.Dir(c.PathSpec().AbsPathify(c.Cfg.GetString("publishDir")))}
httpFs := afero.NewHttpFs(f.c.Fs.Destination)
fs := filesOnlyFs{httpFs.Dir(absPublishDir)}
doLiveReload := !buildWatch && !c.Cfg.GetBool("disableLiveReload")
fastRenderMode := doLiveReload && !c.Cfg.GetBool("disableFastRender")
doLiveReload := !buildWatch && !f.c.Cfg.GetBool("disableLiveReload")
fastRenderMode := doLiveReload && !f.c.Cfg.GetBool("disableFastRender")
if fastRenderMode {
jww.FEEDBACK.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
}
// We're only interested in the path
u, err := url.Parse(baseURL)
if err != nil {
return nil, "", fmt.Errorf("Invalid baseURL: %s", err)
}
decorate := func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if noHTTPCache {
@ -240,7 +290,7 @@ func (c *commandeer) serve(port int) {
if fastRenderMode {
p := r.RequestURI
if strings.HasSuffix(p, "/") || strings.HasSuffix(p, "html") || strings.HasSuffix(p, "htm") {
c.visitedURLs.Add(p)
f.c.visitedURLs.Add(p)
}
}
h.ServeHTTP(w, r)
@ -248,32 +298,78 @@ func (c *commandeer) serve(port int) {
}
fileserver := decorate(http.FileServer(fs))
mu := http.NewServeMux()
// We're only interested in the path
u, err := url.Parse(c.Cfg.GetString("baseURL"))
if err != nil {
jww.ERROR.Fatalf("Invalid baseURL: %s", err)
}
if u.Path == "" || u.Path == "/" {
http.Handle("/", fileserver)
mu.Handle("/", fileserver)
} else {
http.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
}
jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface)
jww.FEEDBACK.Println("Press Ctrl+C to stop")
endpoint := net.JoinHostPort(serverInterface, strconv.Itoa(port))
err = http.ListenAndServe(endpoint, nil)
if err != nil {
jww.ERROR.Printf("Error: %s\n", err.Error())
os.Exit(1)
return mu, endpoint, nil
}
func (c *commandeer) roots() []string {
var roots []string
languages := c.languages()
isMultiHost := languages.IsMultihost()
if !isMultiHost {
return roots
}
for _, l := range languages {
roots = append(roots, l.Lang)
}
return roots
}
func (c *commandeer) serve(port int) {
// TODO(bep) multihost
isMultiHost := Hugo.IsMultihost()
var (
baseURLs []string
roots []string
)
if isMultiHost {
for _, s := range Hugo.Sites {
baseURLs = append(baseURLs, s.BaseURL.String())
roots = append(roots, s.Language.Lang)
}
} else {
baseURLs = []string{Hugo.Sites[0].BaseURL.String()}
roots = []string{""}
}
srv := &fileServer{
basePort: port,
baseURLs: baseURLs,
roots: roots,
c: c,
}
for i, _ := range baseURLs {
mu, endpoint, err := srv.createEndpoint(i)
go func() {
err = http.ListenAndServe(endpoint, mu)
if err != nil {
jww.ERROR.Printf("Error: %s\n", err.Error())
os.Exit(1)
}
}()
}
// TODO(bep) multihost jww.FEEDBACK.Printf("Web Server is available at %s (bind address %s)\n", u.String(), serverInterface)
jww.FEEDBACK.Println("Press Ctrl+C to stop")
}
// fixURL massages the baseURL into a form needed for serving
// all pages correctly.
func fixURL(cfg config.Provider, s string) (string, error) {
func fixURL(cfg config.Provider, s string, port int) (string, error) {
useLocalhost := false
if s == "" {
s = cfg.GetString("baseURL")
@ -315,7 +411,7 @@ func fixURL(cfg config.Provider, s string) (string, error) {
return "", fmt.Errorf("Failed to split baseURL hostpost: %s", err)
}
}
u.Host += fmt.Sprintf(":%d", serverPort)
u.Host += fmt.Sprintf(":%d", port)
}
return u.String(), nil

View file

@ -47,7 +47,7 @@ func TestFixURL(t *testing.T) {
v.Set("baseURL", test.CfgBaseURL)
serverAppend = test.AppendPort
serverPort = test.Port
result, err := fixURL(v, baseURL)
result, err := fixURL(v, baseURL, serverPort)
if err != nil {
t.Errorf("Test #%d %s: unexpected error %s", i, test.TestName, err)
}

View file

@ -102,6 +102,17 @@ func (l *Language) Params() map[string]interface{} {
return l.params
}
// IsMultihost returns whether the languages has baseURL specificed on the
// language level.
func (l Languages) IsMultihost() bool {
for _, lang := range l {
if lang.GetLocal("baseURL") != nil {
return true
}
}
return false
}
// SetParam sets param with the given key and value.
// SetParam is case-insensitive.
func (l *Language) SetParam(k string, v interface{}) {
@ -132,6 +143,17 @@ func (l *Language) GetStringMapString(key string) map[string]string {
//
// Get returns an interface. For a specific value use one of the Get____ methods.
func (l *Language) Get(key string) interface{} {
local := l.GetLocal(key)
if local != nil {
return local
}
return l.Cfg.Get(key)
}
// GetLocal gets a configuration value set on language level. It will
// not fall back to any global value.
// It will return nil if a value with the given key cannot be found.
func (l *Language) GetLocal(key string) interface{} {
if l == nil {
panic("language not set")
}
@ -141,7 +163,7 @@ func (l *Language) Get(key string) interface{} {
return v
}
}
return l.Cfg.Get(key)
return nil
}
// Set sets the value for the key in the language's params.

View file

@ -158,7 +158,6 @@ func (p *PathSpec) AbsPathify(inPath string) string {
return filepath.Clean(inPath)
}
// TODO(bep): Consider moving workingDir to argument list
return filepath.Join(p.workingDir, inPath)
}

View file

@ -19,6 +19,7 @@ import (
"io"
"strings"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/afero"
"github.com/spf13/viper"
@ -80,11 +81,34 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper.
helpers.Deprecated("site config", "disableRobotsTXT", "Use disableKinds= [\"robotsTXT\"]", false)
}
loadDefaultSettingsFor(v)
if err := loadDefaultSettingsFor(v); err != nil {
return v, err
}
return v, nil
}
func loadLanguageSettings(cfg config.Provider) error {
multilingual := cfg.GetStringMap("languages")
var (
langs helpers.Languages
err error
)
if len(multilingual) == 0 {
langs = append(langs, helpers.NewDefaultLanguage(cfg))
} else {
langs, err = toSortedLanguages(cfg, multilingual)
if err != nil {
return fmt.Errorf("Failed to parse multilingual config: %s", err)
}
}
cfg.Set("languagesSorted", langs)
return nil
}
func loadDefaultSettingsFor(v *viper.Viper) error {
c, err := helpers.NewContentSpec(v)
@ -154,5 +178,5 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
v.SetDefault("debug", false)
v.SetDefault("disableFastRender", false)
return nil
return loadLanguageSettings(v)
}

View file

@ -15,7 +15,6 @@ package hugolib
import (
"errors"
"fmt"
"strings"
"sync"
@ -37,9 +36,16 @@ type HugoSites struct {
multilingual *Multilingual
// Multihost is set if multilingual and baseURL set on the language level.
multihost bool
*deps.Deps
}
func (h *HugoSites) IsMultihost() bool {
return h != nil && h.multihost
}
// GetContentPage finds a Page with content given the absolute filename.
// Returns nil if none found.
func (h *HugoSites) GetContentPage(filename string) *Page {
@ -92,6 +98,31 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
h.Deps = sites[0].Deps
// The baseURL may be provided at the language level. If that is true,
// then every language must have a baseURL. In this case we always render
// to a language sub folder, which is then stripped from all the Permalink URLs etc.
var baseURLFromLang bool
for _, s := range sites {
burl := s.Language.GetLocal("baseURL")
if baseURLFromLang && burl == nil {
return h, errors.New("baseURL must be set on all or none of the languages")
}
if burl != nil {
baseURLFromLang = true
}
}
if baseURLFromLang {
for _, s := range sites {
// TODO(bep) multihost check
s.Info.defaultContentLanguageInSubdir = true
s.Cfg.Set("defaultContentLanguageInSubdir", true)
}
h.multihost = true
}
return h, nil
}
@ -180,41 +211,21 @@ func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
sites []*Site
)
multilingual := cfg.Cfg.GetStringMap("languages")
languages := getLanguages(cfg.Cfg)
for _, lang := range languages {
var s *Site
var err error
cfg.Language = lang
s, err = newSite(cfg)
if len(multilingual) == 0 {
l := helpers.NewDefaultLanguage(cfg.Cfg)
cfg.Language = l
s, err := newSite(cfg)
if err != nil {
return nil, err
}
sites = append(sites, s)
}
if len(multilingual) > 0 {
var err error
languages, err := toSortedLanguages(cfg.Cfg, multilingual)
if err != nil {
return nil, fmt.Errorf("Failed to parse multilingual config: %s", err)
}
for _, lang := range languages {
var s *Site
var err error
cfg.Language = lang
s, err = newSite(cfg)
if err != nil {
return nil, err
}
sites = append(sites, s)
}
}
return sites, nil
}
@ -227,7 +238,12 @@ func (h *HugoSites) reset() {
func (h *HugoSites) createSitesFromConfig() error {
if err := loadLanguageSettings(h.Cfg); err != nil {
return err
}
depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg}
sites, err := createSitesFromConfig(depsCfg)
if err != nil {
@ -286,7 +302,7 @@ type BuildCfg struct {
func (h *HugoSites) renderCrossSitesArtifacts() error {
if !h.multilingual.enabled() {
if !h.multilingual.enabled() || h.IsMultihost() {
return nil
}

View file

@ -1269,7 +1269,7 @@ lag:
t.Fatalf("Failed to create sites: %s", err)
}
if len(sites.Sites) != 4 {
if len(sites.Sites) == 0 {
t.Fatalf("Got %d sites", len(sites.Sites))
}

View file

@ -0,0 +1,72 @@
package hugolib
import (
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
func TestMultihosts(t *testing.T) {
t.Parallel()
var multiSiteTOMLConfigTemplate = `
paginate = 1
disablePathToLower = true
defaultContentLanguage = "{{ .DefaultContentLanguage }}"
defaultContentLanguageInSubdir = {{ .DefaultContentLanguageInSubdir }}
[permalinks]
other = "/somewhere/else/:filename"
[Taxonomies]
tag = "tags"
[Languages]
[Languages.en]
baseURL = "https://example.com"
weight = 10
title = "In English"
languageName = "English"
[Languages.fr]
baseURL = "https://example.fr"
weight = 20
title = "Le Français"
languageName = "Français"
[Languages.nn]
baseURL = "https://example.no"
weight = 30
title = "På nynorsk"
languageName = "Nynorsk"
`
siteConfig := testSiteConfig{Fs: afero.NewMemMapFs(), DefaultContentLanguage: "fr", DefaultContentLanguageInSubdir: false}
sites := createMultiTestSites(t, siteConfig, multiSiteTOMLConfigTemplate)
fs := sites.Fs
cfg := BuildCfg{Watching: true}
th := testHelper{sites.Cfg, fs, t}
assert := require.New(t)
err := sites.Build(cfg)
assert.NoError(err)
th.assertFileContent("public/en/sect/doc1-slug/index.html", "Hello")
s1 := sites.Sites[0]
s1h := s1.getPage(KindHome)
assert.True(s1h.IsTranslated())
assert.Len(s1h.Translations(), 2)
assert.Equal("https://example.com/", s1h.Permalink())
s2 := sites.Sites[1]
s2h := s2.getPage(KindHome)
assert.Equal("https://example.fr/", s2h.Permalink())
th.assertFileContentStraight("public/fr/index.html", "French Home Page")
th.assertFileContentStraight("public/en/index.html", "Default Home Page")
}

View file

@ -47,6 +47,14 @@ func (ml *Multilingual) Language(lang string) *helpers.Language {
return ml.langMap[lang]
}
func getLanguages(cfg config.Provider) helpers.Languages {
if cfg.IsSet("languagesSorted") {
return cfg.Get("languagesSorted").(helpers.Languages)
}
return helpers.Languages{helpers.NewDefaultLanguage(cfg)}
}
func newMultiLingualFromSites(cfg config.Provider, sites ...*Site) (*Multilingual, error) {
languages := make(helpers.Languages, len(sites))

View file

@ -1754,6 +1754,11 @@ func (p *Page) shouldAddLanguagePrefix() bool {
return false
}
if p.s.owner.IsMultihost() {
// TODO(bep) multihost check vs lang below
return true
}
if p.Lang() == "" {
return false
}

View file

@ -257,6 +257,10 @@ func (p *Page) createRelativePermalinkForOutputFormat(f output.Format) string {
tp = strings.TrimSuffix(tp, f.BaseFilename())
}
if p.s.owner.IsMultihost() {
tp = strings.TrimPrefix(tp, helpers.FilePathSeparator+p.s.Info.Language.Lang)
}
return p.s.PathSpec.URLizeFilename(tp)
}

View file

@ -387,7 +387,7 @@ func (s *Site) renderAliases() error {
}
}
if s.owner.multilingual.enabled() {
if s.owner.multilingual.enabled() && !s.owner.IsMultihost() {
mainLang := s.owner.multilingual.DefaultLang
if s.Info.defaultContentLanguageInSubdir {
mainLangURL := s.PathSpec.AbsURL(mainLang.Lang, false)