hugo/hugolib/hugo_sites.go
Bjørn Erik Pedersen eb42774e58
Add support for a content dir set per language
A sample config:

```toml
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true

[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
contentDir = "content/english"

[Languages.nn]
weight = 20
title = "På Norsk"
languageName = "Norsk"
contentDir = "content/norwegian"
```

The value of `contentDir` can be any valid path, even absolute path references. The only restriction is that the content dirs cannot overlap.

The content files will be assigned a language by

1. The placement: `content/norwegian/post/my-post.md` will be read as Norwegian content.
2. The filename: `content/english/post/my-post.nn.md` will be read as Norwegian even if it lives in the English content folder.

The content directories will be merged into a big virtual filesystem with one simple rule: The most specific language file will win.
This means that if both `content/norwegian/post/my-post.md` and `content/english/post/my-post.nn.md` exists, they will be considered duplicates and the version inside `content/norwegian` will win.

Note that translations will be automatically assigned by Hugo by the content file's relative placement, so `content/norwegian/post/my-post.md` will be a translation of `content/english/post/my-post.md`.

If this does not work for you, you can connect the translations together by setting a `translationKey` in the content files' front matter.

Fixes #4523
Fixes #4552
Fixes #4553
2018-04-02 08:06:21 +02:00

760 lines
18 KiB
Go

// Copyright 2016-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"errors"
"io"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/gohugoio/hugo/resource"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/i18n"
"github.com/gohugoio/hugo/tpl"
"github.com/gohugoio/hugo/tpl/tplimpl"
)
// HugoSites represents the sites to build. Each site represents a language.
type HugoSites struct {
Sites []*Site
multilingual *Multilingual
// Multihost is set if multilingual and baseURL set on the language level.
multihost bool
// If this is running in the dev server.
running bool
*deps.Deps
// Keeps track of bundle directories and symlinks to enable partial rebuilding.
ContentChanges *contentChangeMap
// If enabled, keeps a revision map for all content.
gitInfo *gitInfo
}
func (h *HugoSites) IsMultihost() bool {
return h != nil && h.multihost
}
func (h *HugoSites) PrintProcessingStats(w io.Writer) {
stats := make([]*helpers.ProcessingStats, len(h.Sites))
for i := 0; i < len(h.Sites); i++ {
stats[i] = h.Sites[i].PathSpec.ProcessingStats
}
helpers.ProcessingStatsTable(w, stats...)
}
func (h *HugoSites) langSite() map[string]*Site {
m := make(map[string]*Site)
for _, s := range h.Sites {
m[s.Language.Lang] = s
}
return m
}
// GetContentPage finds a Page with content given the absolute filename.
// Returns nil if none found.
func (h *HugoSites) GetContentPage(filename string) *Page {
for _, s := range h.Sites {
pos := s.rawAllPages.findPagePosByFilename(filename)
if pos == -1 {
continue
}
return s.rawAllPages[pos]
}
// If not found already, this may be bundled in another content file.
dir := filepath.Dir(filename)
for _, s := range h.Sites {
pos := s.rawAllPages.findPagePosByFilnamePrefix(dir)
if pos == -1 {
continue
}
return s.rawAllPages[pos]
}
return nil
}
// NewHugoSites creates a new collection of sites given the input sites, building
// a language configuration based on those.
func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {
if cfg.Language != nil {
return nil, errors.New("Cannot provide Language in Cfg when sites are provided")
}
langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)
if err != nil {
return nil, err
}
var contentChangeTracker *contentChangeMap
h := &HugoSites{
running: cfg.Running,
multilingual: langConfig,
multihost: cfg.Cfg.GetBool("multihost"),
Sites: sites}
for _, s := range sites {
s.owner = h
}
if err := applyDepsIfNeeded(cfg, sites...); err != nil {
return nil, err
}
h.Deps = sites[0].Deps
// Only needed in server mode.
// TODO(bep) clean up the running vs watching terms
if cfg.Running {
contentChangeTracker = &contentChangeMap{pathSpec: h.PathSpec, symContent: make(map[string]map[string]bool)}
h.ContentChanges = contentChangeTracker
}
if err := h.initGitInfo(); err != nil {
return nil, err
}
return h, nil
}
func (h *HugoSites) initGitInfo() error {
if h.Cfg.GetBool("enableGitInfo") {
gi, err := newGitInfo(h.Cfg)
if err != nil {
h.Log.ERROR.Println("Failed to read Git log:", err)
} else {
h.gitInfo = gi
}
}
return nil
}
func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error {
if cfg.TemplateProvider == nil {
cfg.TemplateProvider = tplimpl.DefaultTemplateProvider
}
if cfg.TranslationProvider == nil {
cfg.TranslationProvider = i18n.NewTranslationProvider()
}
var (
d *deps.Deps
err error
)
for _, s := range sites {
if s.Deps != nil {
continue
}
if d == nil {
cfg.Language = s.Language
cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)
var err error
d, err = deps.New(cfg)
if err != nil {
return err
}
d.OutputFormatsConfig = s.outputFormatsConfig
s.Deps = d
if err = d.LoadResources(); err != nil {
return err
}
} else {
d, err = d.ForLanguage(s.Language)
if err != nil {
return err
}
d.OutputFormatsConfig = s.outputFormatsConfig
s.Deps = d
}
s.resourceSpec, err = resource.NewSpec(s.Deps.PathSpec, s.mediaTypesConfig)
if err != nil {
return err
}
}
return nil
}
// NewHugoSites creates HugoSites from the given config.
func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
sites, err := createSitesFromConfig(cfg)
if err != nil {
return nil, err
}
return newHugoSites(cfg, sites...)
}
func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error {
return func(templ tpl.TemplateHandler) error {
templ.LoadTemplates(s.PathSpec.GetLayoutDirPath(), "")
if s.PathSpec.ThemeSet() {
templ.LoadTemplates(s.PathSpec.GetThemeDir()+"/layouts", "theme")
}
for _, wt := range withTemplates {
if wt == nil {
continue
}
if err := wt(templ); err != nil {
return err
}
}
return nil
}
}
func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {
var (
sites []*Site
)
languages := getLanguages(cfg.Cfg)
for _, lang := range languages {
if lang.Disabled {
continue
}
var s *Site
var err error
cfg.Language = lang
s, err = newSite(cfg)
if err != nil {
return nil, err
}
sites = append(sites, s)
}
return sites, nil
}
// Reset resets the sites and template caches, making it ready for a full rebuild.
func (h *HugoSites) reset() {
for i, s := range h.Sites {
h.Sites[i] = s.reset()
}
}
func (h *HugoSites) createSitesFromConfig() error {
oldLangs, _ := h.Cfg.Get("languagesSorted").(helpers.Languages)
if err := loadLanguageSettings(h.Cfg, oldLangs); err != nil {
return err
}
depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg}
sites, err := createSitesFromConfig(depsCfg)
if err != nil {
return err
}
langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...)
if err != nil {
return err
}
h.Sites = sites
for _, s := range sites {
s.owner = h
}
if err := applyDepsIfNeeded(depsCfg, sites...); err != nil {
return err
}
h.Deps = sites[0].Deps
h.multilingual = langConfig
h.multihost = h.Deps.Cfg.GetBool("multihost")
return nil
}
func (h *HugoSites) toSiteInfos() []*SiteInfo {
infos := make([]*SiteInfo, len(h.Sites))
for i, s := range h.Sites {
infos[i] = &s.Info
}
return infos
}
// BuildCfg holds build options used to, as an example, skip the render step.
type BuildCfg struct {
// Reset site state before build. Use to force full rebuilds.
ResetState bool
// Re-creates the sites from configuration before a build.
// This is needed if new languages are added.
CreateSitesFromConfig bool
// Skip rendering. Useful for testing.
SkipRender bool
// Use this to indicate what changed (for rebuilds).
whatChanged *whatChanged
// Recently visited URLs. This is used for partial re-rendering.
RecentlyVisited map[string]bool
}
// shouldRender is used in the Fast Render Mode to determine if we need to re-render
// a Page: If it is recently visited (the home pages will always be in this set) or changed.
// Note that a page does not have to have a content page / file.
// For regular builds, this will allways return true.
func (cfg *BuildCfg) shouldRender(p *Page) bool {
if len(cfg.RecentlyVisited) == 0 {
return true
}
if cfg.RecentlyVisited[p.RelPermalink()] {
return true
}
if cfg.whatChanged != nil && p.File != nil && cfg.whatChanged.files[p.File.Filename()] {
return true
}
return false
}
func (h *HugoSites) renderCrossSitesArtifacts() error {
if !h.multilingual.enabled() || h.IsMultihost() {
return nil
}
sitemapEnabled := false
for _, s := range h.Sites {
if s.isEnabled(kindSitemap) {
sitemapEnabled = true
break
}
}
if !sitemapEnabled {
return nil
}
// TODO(bep) DRY
sitemapDefault := parseSitemap(h.Cfg.GetStringMap("sitemap"))
s := h.Sites[0]
smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"}
return s.renderAndWriteXML(&s.PathSpec.ProcessingStats.Sitemaps, "sitemapindex",
sitemapDefault.Filename, h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...)
}
func (h *HugoSites) assignMissingTranslations() error {
// This looks heavy, but it should be a small number of nodes by now.
allPages := h.findAllPagesByKindNotIn(KindPage)
for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} {
nodes := h.findPagesByKindIn(nodeType, allPages)
// Assign translations
for _, t1 := range nodes {
for _, t2 := range nodes {
if t1.isNewTranslation(t2) {
t1.translations = append(t1.translations, t2)
}
}
}
}
// Now we can sort the translations.
for _, p := range allPages {
if len(p.translations) > 0 {
pageBy(languagePageSort).Sort(p.translations)
}
}
return nil
}
// createMissingPages creates home page, taxonomies etc. that isnt't created as an
// effect of having a content file.
func (h *HugoSites) createMissingPages() error {
var newPages Pages
for _, s := range h.Sites {
if s.isEnabled(KindHome) {
// home pages
home := s.findPagesByKind(KindHome)
if len(home) > 1 {
panic("Too many homes")
}
if len(home) == 0 {
n := s.newHomePage()
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
}
// Will create content-less root sections.
newSections := s.assembleSections()
s.Pages = append(s.Pages, newSections...)
newPages = append(newPages, newSections...)
// taxonomy list and terms pages
taxonomies := s.Language.GetStringMapString("taxonomies")
if len(taxonomies) > 0 {
taxonomyPages := s.findPagesByKind(KindTaxonomy)
taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm)
for _, plural := range taxonomies {
if s.isEnabled(KindTaxonomyTerm) {
foundTaxonomyTermsPage := false
for _, p := range taxonomyTermsPages {
if p.sections[0] == plural {
foundTaxonomyTermsPage = true
break
}
}
if !foundTaxonomyTermsPage {
n := s.newTaxonomyTermsPage(plural)
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
}
if s.isEnabled(KindTaxonomy) {
for key := range s.Taxonomies[plural] {
foundTaxonomyPage := false
origKey := key
if s.Info.preserveTaxonomyNames {
key = s.PathSpec.MakePathSanitized(key)
}
for _, p := range taxonomyPages {
// Some people may have /authors/MaxMustermann etc. as paths.
// p.sections contains the raw values from the file system.
// See https://github.com/gohugoio/hugo/issues/4238
singularKey := s.PathSpec.MakePathSanitized(p.sections[1])
if p.sections[0] == plural && singularKey == key {
foundTaxonomyPage = true
break
}
}
if !foundTaxonomyPage {
n := s.newTaxonomyPage(plural, origKey)
s.Pages = append(s.Pages, n)
newPages = append(newPages, n)
}
}
}
}
}
}
if len(newPages) > 0 {
// This resorting is unfortunate, but it also needs to be sorted
// when sections are created.
first := h.Sites[0]
first.AllPages = append(first.AllPages, newPages...)
first.AllPages.Sort()
for _, s := range h.Sites {
s.Pages.Sort()
}
for i := 1; i < len(h.Sites); i++ {
h.Sites[i].AllPages = first.AllPages
}
}
return nil
}
func (h *HugoSites) removePageByFilename(filename string) {
for _, s := range h.Sites {
s.removePageFilename(filename)
}
}
func (h *HugoSites) setupTranslations() {
for _, s := range h.Sites {
for _, p := range s.rawAllPages {
if p.Kind == kindUnknown {
p.Kind = p.s.kindFromSections(p.sections)
}
if !p.s.isEnabled(p.Kind) {
continue
}
shouldBuild := p.shouldBuild()
s.updateBuildStats(p)
if shouldBuild {
if p.headless {
s.headlessPages = append(s.headlessPages, p)
} else {
s.Pages = append(s.Pages, p)
}
}
}
}
allPages := make(Pages, 0)
for _, s := range h.Sites {
allPages = append(allPages, s.Pages...)
}
allPages.Sort()
for _, s := range h.Sites {
s.AllPages = allPages
}
// Pull over the collections from the master site
for i := 1; i < len(h.Sites); i++ {
h.Sites[i].Data = h.Sites[0].Data
}
if len(h.Sites) > 1 {
allTranslations := pagesToTranslationsMap(allPages)
assignTranslationsToPages(allTranslations, allPages)
}
}
func (s *Site) preparePagesForRender(cfg *BuildCfg) {
pageChan := make(chan *Page)
wg := &sync.WaitGroup{}
numWorkers := getGoMaxProcs() * 4
for i := 0; i < numWorkers; i++ {
wg.Add(1)
go func(pages <-chan *Page, wg *sync.WaitGroup) {
defer wg.Done()
for p := range pages {
if err := p.prepareForRender(cfg); err != nil {
s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
}
}
}(pageChan, wg)
}
for _, p := range s.Pages {
pageChan <- p
}
for _, p := range s.headlessPages {
pageChan <- p
}
close(pageChan)
wg.Wait()
}
// Pages returns all pages for all sites.
func (h *HugoSites) Pages() Pages {
return h.Sites[0].AllPages
}
func handleShortcodes(p *Page, rawContentCopy []byte) ([]byte, error) {
if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {
p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())
err := p.shortcodeState.executeShortcodesForDelta(p)
if err != nil {
return rawContentCopy, err
}
rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, p.shortcodeState.renderedShortcodes)
if err != nil {
p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
}
}
return rawContentCopy, nil
}
func (s *Site) updateBuildStats(page *Page) {
if page.IsDraft() {
s.draftCount++
}
if page.IsFuture() {
s.futureCount++
}
if page.IsExpired() {
s.expiredCount++
}
}
func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages {
return h.Sites[0].findPagesByKindNotIn(kind, inPages)
}
func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages {
return h.Sites[0].findPagesByKindIn(kind, inPages)
}
func (h *HugoSites) findAllPagesByKind(kind string) Pages {
return h.findPagesByKindIn(kind, h.Sites[0].AllPages)
}
func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages {
return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages)
}
func (h *HugoSites) findPagesByShortcode(shortcode string) Pages {
var pages Pages
for _, s := range h.Sites {
pages = append(pages, s.findPagesByShortcode(shortcode)...)
}
return pages
}
// Used in partial reloading to determine if the change is in a bundle.
type contentChangeMap struct {
mu sync.RWMutex
branches []string
leafs []string
pathSpec *helpers.PathSpec
// Hugo supports symlinked content (both directories and files). This
// can lead to situations where the same file can be referenced from several
// locations in /content -- which is really cool, but also means we have to
// go an extra mile to handle changes.
// This map is only used in watch mode.
// It maps either file to files or the real dir to a set of content directories where it is in use.
symContent map[string]map[string]bool
symContentMu sync.Mutex
}
func (m *contentChangeMap) add(filename string, tp bundleDirType) {
m.mu.Lock()
dir := filepath.Dir(filename) + helpers.FilePathSeparator
switch tp {
case bundleBranch:
m.branches = append(m.branches, dir)
case bundleLeaf:
m.leafs = append(m.leafs, dir)
default:
panic("invalid bundle type")
}
m.mu.Unlock()
}
// Track the addition of bundle dirs.
func (m *contentChangeMap) handleBundles(b *bundleDirs) {
for _, bd := range b.bundles {
m.add(bd.fi.Path(), bd.tp)
}
}
// resolveAndRemove resolves the given filename to the root folder of a bundle, if relevant.
// It also removes the entry from the map. It will be re-added again by the partial
// build if it still is a bundle.
func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) {
m.mu.RLock()
defer m.mu.RUnlock()
// Bundles share resources, so we need to start from the virtual root.
relPath, _ := m.pathSpec.RelContentDir(filename)
dir, name := filepath.Split(relPath)
if !strings.HasSuffix(dir, helpers.FilePathSeparator) {
dir += helpers.FilePathSeparator
}
fileTp, _ := classifyBundledFile(name)
// This may be a member of a bundle. Start with branch bundles, the most specific.
if fileTp != bundleLeaf {
for i, b := range m.branches {
if b == dir {
m.branches = append(m.branches[:i], m.branches[i+1:]...)
return dir, b, bundleBranch
}
}
}
// And finally the leaf bundles, which can contain anything.
for i, l := range m.leafs {
if strings.HasPrefix(dir, l) {
m.leafs = append(m.leafs[:i], m.leafs[i+1:]...)
return dir, l, bundleLeaf
}
}
// Not part of any bundle
return dir, filename, bundleNot
}
func (m *contentChangeMap) addSymbolicLinkMapping(from, to string) {
m.symContentMu.Lock()
mm, found := m.symContent[from]
if !found {
mm = make(map[string]bool)
m.symContent[from] = mm
}
mm[to] = true
m.symContentMu.Unlock()
}
func (m *contentChangeMap) GetSymbolicLinkMappings(dir string) []string {
mm, found := m.symContent[dir]
if !found {
return nil
}
dirs := make([]string, len(mm))
i := 0
for dir := range mm {
dirs[i] = dir
i++
}
sort.Strings(dirs)
return dirs
}