hugo/hugolib/hugo_sites.go

// Copyright 2016-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
	"errors"
	"strings"
	"sync"

	"path/filepath"

	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"

	"github.com/gohugoio/hugo/i18n"
	"github.com/gohugoio/hugo/tpl"
	"github.com/gohugoio/hugo/tpl/tplimpl"
)

// HugoSites represents the sites to build. Each site represents a language.
type HugoSites struct {
	Sites []*Site

	runMode runmode

	multilingual *Multilingual

	// Multihost is set if multilingual and baseURL set on the language level.
	multihost bool

	*deps.Deps
}

func (h *HugoSites) IsMultihost() bool {
	return h != nil && h.multihost
}

// GetContentPage finds a Page with content given the absolute filename.
// Returns nil if none found.
func (h *HugoSites) GetContentPage(filename string) *Page {
	s := h.Sites[0]
	contendDir := filepath.Join(s.PathSpec.AbsPathify(s.Cfg.GetString("contentDir")))
	if !strings.HasPrefix(filename, contendDir) {
		return nil
	}

	rel := strings.TrimPrefix(filename, contendDir)
	rel = strings.TrimPrefix(rel, helpers.FilePathSeparator)

	pos := s.rawAllPages.findPagePosByFilePath(rel)

	if pos == -1 {
		return nil
	}
	return s.rawAllPages[pos]

}

// NewHugoSites creates a new collection of sites given the input sites, building
// a language configuration based on those.
func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) {

	if cfg.Language != nil {
		return nil, errors.New("Cannot provide Language in Cfg when sites are provided")
	}

	langConfig, err := newMultiLingualFromSites(cfg.Cfg, sites...)

	if err != nil {
		return nil, err
	}

	h := &HugoSites{
		multilingual: langConfig,
		Sites:        sites}

	for _, s := range sites {
		s.owner = h
	}

	// TODO(bep)
	cfg.Cfg.Set("multilingual", sites[0].multilingualEnabled())

	if err := applyDepsIfNeeded(cfg, sites...); err != nil {
		return nil, err
	}

	h.Deps = sites[0].Deps

	// The baseURL may be provided at the language level. If that is true,
	// then every language must have a baseURL. In this case we always render
	// to a language sub folder, which is then stripped from all the Permalink URLs etc.
	var baseURLFromLang bool

	for _, s := range sites {
		burl := s.Language.GetLocal("baseURL")
		if baseURLFromLang && burl == nil {
			return h, errors.New("baseURL must be set on all or none of the languages")
		}

		if burl != nil {
			baseURLFromLang = true
		}
	}

	if baseURLFromLang {
		for _, s := range sites {
			// TODO(bep) multihost check
			s.Info.defaultContentLanguageInSubdir = true
			s.Cfg.Set("defaultContentLanguageInSubdir", true)
		}
		h.multihost = true
	}

	return h, nil
}

func applyDepsIfNeeded(cfg deps.DepsCfg, sites ...*Site) error {
	if cfg.TemplateProvider == nil {
		cfg.TemplateProvider = tplimpl.DefaultTemplateProvider
	}

	if cfg.TranslationProvider == nil {
		cfg.TranslationProvider = i18n.NewTranslationProvider()
	}

	var (
		d   *deps.Deps
		err error
	)

	for _, s := range sites {
		if s.Deps != nil {
			continue
		}

		if d == nil {
			cfg.Language = s.Language
			cfg.WithTemplate = s.withSiteTemplates(cfg.WithTemplate)

			var err error
			d, err = deps.New(cfg)
			if err != nil {
				return err
			}

			d.OutputFormatsConfig = s.outputFormatsConfig
			s.Deps = d

			if err = d.LoadResources(); err != nil {
				return err
			}

		} else {
			d, err = d.ForLanguage(s.Language)
			if err != nil {
				return err
			}
			d.OutputFormatsConfig = s.outputFormatsConfig
			s.Deps = d
		}

	}

	return nil
}

// NewHugoSites creates HugoSites from the given config.
func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
	sites, err := createSitesFromConfig(cfg)
	if err != nil {
		return nil, err
	}
	return newHugoSites(cfg, sites...)
}

func (s *Site) withSiteTemplates(withTemplates ...func(templ tpl.TemplateHandler) error) func(templ tpl.TemplateHandler) error {
	return func(templ tpl.TemplateHandler) error {
		templ.LoadTemplates(s.PathSpec.GetLayoutDirPath(), "")
		if s.PathSpec.ThemeSet() {
			templ.LoadTemplates(s.PathSpec.GetThemeDir()+"/layouts", "theme")
		}

		for _, wt := range withTemplates {
			if wt == nil {
				continue
			}
			if err := wt(templ); err != nil {
				return err
			}
		}

		return nil
	}
}

func createSitesFromConfig(cfg deps.DepsCfg) ([]*Site, error) {

	var (
		sites []*Site
	)

	languages := getLanguages(cfg.Cfg)

	for _, lang := range languages {
		var s *Site
		var err error
		cfg.Language = lang
		s, err = newSite(cfg)

		if err != nil {
			return nil, err
		}

		sites = append(sites, s)
	}

	return sites, nil
}

// Reset resets the sites and template caches, making it ready for a full rebuild.
func (h *HugoSites) reset() {
	for i, s := range h.Sites {
		h.Sites[i] = s.reset()
	}
}

func (h *HugoSites) createSitesFromConfig() error {

	if err := loadLanguageSettings(h.Cfg); err != nil {
		return err
	}

	depsCfg := deps.DepsCfg{Fs: h.Fs, Cfg: h.Cfg}

	sites, err := createSitesFromConfig(depsCfg)

	if err != nil {
		return err
	}

	langConfig, err := newMultiLingualFromSites(depsCfg.Cfg, sites...)

	if err != nil {
		return err
	}

	h.Sites = sites

	for _, s := range sites {
		s.owner = h
	}

	if err := applyDepsIfNeeded(depsCfg, sites...); err != nil {
		return err
	}

	h.Deps = sites[0].Deps

	h.multilingual = langConfig

	return nil
}

func (h *HugoSites) toSiteInfos() []*SiteInfo {
	infos := make([]*SiteInfo, len(h.Sites))
	for i, s := range h.Sites {
		infos[i] = &s.Info
	}
	return infos
}

// BuildCfg holds build options used to, as an example, skip the render step.
type BuildCfg struct {
	// Whether we are in watch (server) mode
	Watching bool
	// Print build stats at the end of a build
	PrintStats bool
	// Reset site state before build. Use to force full rebuilds.
	ResetState bool
	// Re-creates the sites from configuration before a build.
	// This is needed if new languages are added.
	CreateSitesFromConfig bool
	// Skip rendering. Useful for testing.
	SkipRender bool
	// Use this to indicate what changed (for rebuilds).
	whatChanged *whatChanged
	// Recently visited URLs. This is used for partial re-rendering.
	RecentlyVisited map[string]bool
}

func (h *HugoSites) renderCrossSitesArtifacts() error {

	if !h.multilingual.enabled() || h.IsMultihost() {
		return nil
	}

	if h.Cfg.GetBool("disableSitemap") {
		return nil
	}

	sitemapEnabled := false
	for _, s := range h.Sites {
		if s.isEnabled(kindSitemap) {
			sitemapEnabled = true
			break
		}
	}

	if !sitemapEnabled {
		return nil
	}

	// TODO(bep) DRY
	sitemapDefault := parseSitemap(h.Cfg.GetStringMap("sitemap"))

	s := h.Sites[0]

	smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"}

	return s.renderAndWriteXML("sitemapindex",
		sitemapDefault.Filename, h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...)
}

func (h *HugoSites) assignMissingTranslations() error {
	// This looks heavy, but it should be a small number of nodes by now.
	allPages := h.findAllPagesByKindNotIn(KindPage)
	for _, nodeType := range []string{KindHome, KindSection, KindTaxonomy, KindTaxonomyTerm} {
		nodes := h.findPagesByKindIn(nodeType, allPages)

		// Assign translations
		for _, t1 := range nodes {
			for _, t2 := range nodes {
				if t1.isNewTranslation(t2) {
					t1.translations = append(t1.translations, t2)
				}
			}
		}
	}

	// Now we can sort the translations.
	for _, p := range allPages {
		if len(p.translations) > 0 {
			pageBy(languagePageSort).Sort(p.translations)
		}
	}
	return nil

}

// createMissingPages creates home page, taxonomies etc. that isnt't created as an
// effect of having a content file.
func (h *HugoSites) createMissingPages() error {
	var newPages Pages

	for _, s := range h.Sites {
		if s.isEnabled(KindHome) {
			// home pages
			home := s.findPagesByKind(KindHome)
			if len(home) > 1 {
				panic("Too many homes")
			}
			if len(home) == 0 {
				n := s.newHomePage()
				s.Pages = append(s.Pages, n)
				newPages = append(newPages, n)
			}
		}

		// Will create content-less root sections.
		newSections := s.assembleSections()
		s.Pages = append(s.Pages, newSections...)
		newPages = append(newPages, newSections...)

		// taxonomy list and terms pages
		taxonomies := s.Language.GetStringMapString("taxonomies")
		if len(taxonomies) > 0 {
			taxonomyPages := s.findPagesByKind(KindTaxonomy)
			taxonomyTermsPages := s.findPagesByKind(KindTaxonomyTerm)
			for _, plural := range taxonomies {
				if s.isEnabled(KindTaxonomyTerm) {
					foundTaxonomyTermsPage := false
					for _, p := range taxonomyTermsPages {
						if p.sections[0] == plural {
							foundTaxonomyTermsPage = true
							break
						}
					}

					if !foundTaxonomyTermsPage {
						n := s.newTaxonomyTermsPage(plural)
						s.Pages = append(s.Pages, n)
						newPages = append(newPages, n)
					}
				}

				if s.isEnabled(KindTaxonomy) {
					for key := range s.Taxonomies[plural] {
						foundTaxonomyPage := false
						origKey := key

						if s.Info.preserveTaxonomyNames {
							key = s.PathSpec.MakePathSanitized(key)
						}
						for _, p := range taxonomyPages {
							if p.sections[0] == plural && p.sections[1] == key {
								foundTaxonomyPage = true
								break
							}
						}

						if !foundTaxonomyPage {
							n := s.newTaxonomyPage(plural, origKey)
							s.Pages = append(s.Pages, n)
							newPages = append(newPages, n)
						}
					}
				}
			}
		}
	}

	if len(newPages) > 0 {
		// This resorting is unfortunate, but it also needs to be sorted
		// when sections are created.
		first := h.Sites[0]

		first.AllPages = append(first.AllPages, newPages...)

		first.AllPages.Sort()

		for _, s := range h.Sites {
			s.Pages.Sort()
		}

		for i := 1; i < len(h.Sites); i++ {
			h.Sites[i].AllPages = first.AllPages
		}
	}

	return nil
}

func (s *Site) assignSiteByLanguage(p *Page) {

	pageLang := p.Lang()

	if pageLang == "" {
		panic("Page language missing: " + p.Title)
	}

	for _, site := range s.owner.Sites {
		if strings.HasPrefix(site.Language.Lang, pageLang) {
			p.s = site
			p.Site = &site.Info
			return
		}
	}

}

func (h *HugoSites) setupTranslations() {

	master := h.Sites[0]

	for _, p := range master.rawAllPages {
		if p.Lang() == "" {
			panic("Page language missing: " + p.Title)
		}

		if p.Kind == kindUnknown {
			p.Kind = p.s.kindFromSections(p.sections)
		}

		if !p.s.isEnabled(p.Kind) {
			continue
		}

		shouldBuild := p.shouldBuild()

		for i, site := range h.Sites {
			// The site is assigned by language when read.
			if site == p.s {
				site.updateBuildStats(p)
				if shouldBuild {
					site.Pages = append(site.Pages, p)
				}
			}

			if !shouldBuild {
				continue
			}

			if i == 0 {
				site.AllPages = append(site.AllPages, p)
			}
		}

	}

	// Pull over the collections from the master site
	for i := 1; i < len(h.Sites); i++ {
		h.Sites[i].AllPages = h.Sites[0].AllPages
		h.Sites[i].Data = h.Sites[0].Data
	}

	if len(h.Sites) > 1 {
		pages := h.Sites[0].AllPages
		allTranslations := pagesToTranslationsMap(pages)
		assignTranslationsToPages(allTranslations, pages)
	}
}

func (s *Site) preparePagesForRender(cfg *BuildCfg) {

	pageChan := make(chan *Page)
	wg := &sync.WaitGroup{}
	numWorkers := getGoMaxProcs() * 4

	for i := 0; i < numWorkers; i++ {
		wg.Add(1)
		go func(pages <-chan *Page, wg *sync.WaitGroup) {
			defer wg.Done()
			for p := range pages {
				if !p.shouldRenderTo(s.rc.Format) {
					// No need to prepare
					continue
				}
				var shortcodeUpdate bool
				if p.shortcodeState != nil {
					shortcodeUpdate = p.shortcodeState.updateDelta()
				}

				if !shortcodeUpdate && !cfg.whatChanged.other && p.rendered {
					// No need to process it again.
					continue
				}

				// If we got this far it means that this is either a new Page pointer
				// or a template or similar has changed so wee need to do a rerendering
				// of the shortcodes etc.

				// Mark it as rendered
				p.rendered = true

				// If in watch mode or if we have multiple output formats,
				// we need to keep the original so we can
				// potentially repeat this process on rebuild.
				needsACopy := cfg.Watching || len(p.outputFormats) > 1
				var workContentCopy []byte
				if needsACopy {
					workContentCopy = make([]byte, len(p.workContent))
					copy(workContentCopy, p.workContent)
				} else {
					// Just reuse the same slice.
					workContentCopy = p.workContent
				}

				if p.Markup == "markdown" {
					tmpContent, tmpTableOfContents := helpers.ExtractTOC(workContentCopy)
					p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
					workContentCopy = tmpContent
				}

				var err error
				if workContentCopy, err = handleShortcodes(p, workContentCopy); err != nil {
					s.Log.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
				}

				if p.Markup != "html" {

					// Now we know enough to create a summary of the page and count some words
					summaryContent, err := p.setUserDefinedSummaryIfProvided(workContentCopy)

					if err != nil {
						s.Log.ERROR.Printf("Failed to set user defined summary for page %q: %s", p.Path(), err)
					} else if summaryContent != nil {
						workContentCopy = summaryContent.content
					}

					p.Content = helpers.BytesToHTML(workContentCopy)

					if summaryContent == nil {
						if err := p.setAutoSummary(); err != nil {
							s.Log.ERROR.Printf("Failed to set user auto summary for page %q: %s", p.pathOrTitle(), err)
						}
					}

				} else {
					p.Content = helpers.BytesToHTML(workContentCopy)
				}

				//analyze for raw stats
				p.analyzePage()

			}
		}(pageChan, wg)
	}

	for _, p := range s.Pages {
		pageChan <- p
	}

	close(pageChan)

	wg.Wait()

}

// Pages returns all pages for all sites.
func (h *HugoSites) Pages() Pages {
	return h.Sites[0].AllPages
}

func handleShortcodes(p *Page, rawContentCopy []byte) ([]byte, error) {
	if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {
		p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())
		err := p.shortcodeState.executeShortcodesForDelta(p)

		if err != nil {
			return rawContentCopy, err
		}

		rawContentCopy, err = replaceShortcodeTokens(rawContentCopy, shortcodePlaceholderPrefix, p.shortcodeState.renderedShortcodes)

		if err != nil {
			p.s.Log.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
		}
	}

	return rawContentCopy, nil
}

func (s *Site) updateBuildStats(page *Page) {
	if page.IsDraft() {
		s.draftCount++
	}

	if page.IsFuture() {
		s.futureCount++
	}

	if page.IsExpired() {
		s.expiredCount++
	}
}

func (h *HugoSites) findPagesByKindNotIn(kind string, inPages Pages) Pages {
	return h.Sites[0].findPagesByKindNotIn(kind, inPages)
}

func (h *HugoSites) findPagesByKindIn(kind string, inPages Pages) Pages {
	return h.Sites[0].findPagesByKindIn(kind, inPages)
}

func (h *HugoSites) findAllPagesByKind(kind string) Pages {
	return h.findPagesByKindIn(kind, h.Sites[0].AllPages)
}

func (h *HugoSites) findAllPagesByKindNotIn(kind string) Pages {
	return h.findPagesByKindNotIn(kind, h.Sites[0].AllPages)
}