mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-30 01:53:34 -05:00
f21e2f25c9
All config variables starts with low-case and uses camelCase. If there is abbreviation at the beginning of the name, the whole abbreviation will be written in low-case. If there is abbreviation at the end of the name, the whole abbreviation will be written in upper-case. For example, rssURI.
619 lines
14 KiB
Go
619 lines
14 KiB
Go
// Copyright 2016-present The Hugo Authors. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package hugolib
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/spf13/hugo/helpers"
|
|
|
|
"github.com/spf13/viper"
|
|
|
|
"github.com/fsnotify/fsnotify"
|
|
"github.com/spf13/hugo/source"
|
|
"github.com/spf13/hugo/tpl"
|
|
jww "github.com/spf13/jwalterweatherman"
|
|
)
|
|
|
|
// HugoSites represents the sites to build. Each site represents a language.
|
|
type HugoSites struct {
|
|
Sites []*Site
|
|
|
|
tmpl tpl.Template
|
|
runMode runmode
|
|
|
|
multilingual *Multilingual
|
|
|
|
// Maps internalID to a set of nodes.
|
|
nodeMap map[string]Nodes
|
|
nodeMapMu sync.RWMutex
|
|
}
|
|
|
|
// NewHugoSites creates a new collection of sites given the input sites, building
|
|
// a language configuration based on those.
|
|
func newHugoSites(sites ...*Site) (*HugoSites, error) {
|
|
langConfig, err := newMultiLingualFromSites(sites...)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
h := &HugoSites{multilingual: langConfig, Sites: sites, nodeMap: make(map[string]Nodes)}
|
|
|
|
for _, s := range sites {
|
|
s.owner = h
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// NewHugoSitesFromConfiguration creates HugoSites from the global Viper config.
|
|
func NewHugoSitesFromConfiguration() (*HugoSites, error) {
|
|
sites, err := createSitesFromConfig()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return newHugoSites(sites...)
|
|
}
|
|
|
|
func createSitesFromConfig() ([]*Site, error) {
|
|
var sites []*Site
|
|
multilingual := viper.GetStringMap("languages")
|
|
if len(multilingual) == 0 {
|
|
sites = append(sites, newSite(helpers.NewDefaultLanguage()))
|
|
}
|
|
|
|
if len(multilingual) > 0 {
|
|
var err error
|
|
|
|
languages, err := toSortedLanguages(multilingual)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Failed to parse multilingual config: %s", err)
|
|
}
|
|
|
|
for _, lang := range languages {
|
|
sites = append(sites, newSite(lang))
|
|
}
|
|
|
|
}
|
|
|
|
return sites, nil
|
|
}
|
|
|
|
func (h *HugoSites) addNode(nodeID string, node *Node) {
|
|
h.nodeMapMu.Lock()
|
|
|
|
if nodes, ok := h.nodeMap[nodeID]; ok {
|
|
h.nodeMap[nodeID] = append(nodes, node)
|
|
} else {
|
|
h.nodeMap[nodeID] = Nodes{node}
|
|
}
|
|
h.nodeMapMu.Unlock()
|
|
}
|
|
|
|
func (h *HugoSites) getNodes(nodeID string) Nodes {
|
|
if nodeID != "" {
|
|
h.nodeMapMu.RLock()
|
|
nodes, ok := h.nodeMap[nodeID]
|
|
h.nodeMapMu.RUnlock()
|
|
if ok {
|
|
return nodes
|
|
}
|
|
}
|
|
// Paginator pages will not have related nodes.
|
|
return Nodes{}
|
|
}
|
|
|
|
// Reset resets the sites, making it ready for a full rebuild.
|
|
func (h *HugoSites) reset() {
|
|
h.nodeMap = make(map[string]Nodes)
|
|
for i, s := range h.Sites {
|
|
h.Sites[i] = s.reset()
|
|
}
|
|
}
|
|
|
|
func (h *HugoSites) reCreateFromConfig() error {
|
|
h.nodeMap = make(map[string]Nodes)
|
|
|
|
sites, err := createSitesFromConfig()
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
langConfig, err := newMultiLingualFromSites(sites...)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
h.Sites = sites
|
|
|
|
for _, s := range sites {
|
|
s.owner = h
|
|
}
|
|
|
|
h.multilingual = langConfig
|
|
|
|
return nil
|
|
}
|
|
|
|
func (h *HugoSites) toSiteInfos() []*SiteInfo {
|
|
infos := make([]*SiteInfo, len(h.Sites))
|
|
for i, s := range h.Sites {
|
|
infos[i] = &s.Info
|
|
}
|
|
return infos
|
|
}
|
|
|
|
// BuildCfg holds build options used to, as an example, skip the render step.
|
|
type BuildCfg struct {
|
|
// Whether we are in watch (server) mode
|
|
Watching bool
|
|
// Print build stats at the end of a build
|
|
PrintStats bool
|
|
// Reset site state before build. Use to force full rebuilds.
|
|
ResetState bool
|
|
// Re-creates the sites from configuration before a build.
|
|
// This is needed if new languages are added.
|
|
CreateSitesFromConfig bool
|
|
// Skip rendering. Useful for testing.
|
|
SkipRender bool
|
|
// Use this to add templates to use for rendering.
|
|
// Useful for testing.
|
|
withTemplate func(templ tpl.Template) error
|
|
}
|
|
|
|
// Build builds all sites.
|
|
func (h *HugoSites) Build(config BuildCfg) error {
|
|
|
|
t0 := time.Now()
|
|
|
|
if config.ResetState {
|
|
h.reset()
|
|
}
|
|
|
|
if config.CreateSitesFromConfig {
|
|
if err := h.reCreateFromConfig(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
h.runMode.Watching = config.Watching
|
|
|
|
// We should probably refactor the Site and pull up most of the logic from there to here,
|
|
// but that seems like a daunting task.
|
|
// So for now, if there are more than one site (language),
|
|
// we pre-process the first one, then configure all the sites based on that.
|
|
firstSite := h.Sites[0]
|
|
|
|
if err := firstSite.preProcess(config); err != nil {
|
|
return err
|
|
}
|
|
|
|
h.setupTranslations(firstSite)
|
|
|
|
if len(h.Sites) > 1 {
|
|
// Initialize the rest
|
|
for _, site := range h.Sites[1:] {
|
|
site.initializeSiteInfo()
|
|
}
|
|
}
|
|
|
|
for _, s := range h.Sites {
|
|
if err := s.postProcess(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if err := h.preRender(config, whatChanged{source: true, other: true}); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !config.SkipRender {
|
|
for _, s := range h.Sites {
|
|
|
|
if err := s.render(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if config.PrintStats {
|
|
s.Stats()
|
|
}
|
|
}
|
|
|
|
if err := h.render(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if config.PrintStats {
|
|
jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds()))
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Rebuild rebuilds all sites.
|
|
func (h *HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
|
|
t0 := time.Now()
|
|
|
|
if config.CreateSitesFromConfig {
|
|
return errors.New("Rebuild does not support 'CreateSitesFromConfig'. Use Build.")
|
|
}
|
|
|
|
if config.ResetState {
|
|
return errors.New("Rebuild does not support 'ResetState'. Use Build.")
|
|
}
|
|
|
|
if !config.Watching {
|
|
return errors.New("Rebuild called when not in watch mode")
|
|
}
|
|
|
|
h.runMode.Watching = config.Watching
|
|
|
|
firstSite := h.Sites[0]
|
|
|
|
h.nodeMap = make(map[string]Nodes)
|
|
for _, s := range h.Sites {
|
|
s.resetBuildState()
|
|
}
|
|
|
|
helpers.InitLoggers()
|
|
|
|
changed, err := firstSite.reBuild(events)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Assign pages to sites per translation.
|
|
h.setupTranslations(firstSite)
|
|
|
|
if changed.source {
|
|
for _, s := range h.Sites {
|
|
if err := s.postProcess(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
if err := h.preRender(config, changed); err != nil {
|
|
return err
|
|
}
|
|
|
|
if !config.SkipRender {
|
|
for _, s := range h.Sites {
|
|
if err := s.render(); err != nil {
|
|
return err
|
|
}
|
|
if config.PrintStats {
|
|
s.Stats()
|
|
}
|
|
}
|
|
|
|
if err := h.render(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if config.PrintStats {
|
|
jww.FEEDBACK.Printf("total in %v ms\n", int(1000*time.Since(t0).Seconds()))
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
// Analyze prints a build report to Stdout.
|
|
// Useful for debugging.
|
|
func (h *HugoSites) Analyze() error {
|
|
if err := h.Build(BuildCfg{SkipRender: true}); err != nil {
|
|
return err
|
|
}
|
|
s := h.Sites[0]
|
|
return s.ShowPlan(os.Stdout)
|
|
}
|
|
|
|
// Render the cross-site artifacts.
|
|
func (h *HugoSites) render() error {
|
|
|
|
if !h.multilingual.enabled() {
|
|
return nil
|
|
}
|
|
|
|
if viper.GetBool("disableSitemap") {
|
|
return nil
|
|
}
|
|
|
|
// TODO(bep) DRY
|
|
sitemapDefault := parseSitemap(viper.GetStringMap("sitemap"))
|
|
|
|
s := h.Sites[0]
|
|
|
|
smLayouts := []string{"sitemapindex.xml", "_default/sitemapindex.xml", "_internal/_default/sitemapindex.xml"}
|
|
|
|
if err := s.renderAndWriteXML("sitemapindex", sitemapDefault.Filename,
|
|
h.toSiteInfos(), s.appendThemeTemplates(smLayouts)...); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (h *HugoSites) setupTranslations(master *Site) {
|
|
|
|
for _, p := range master.rawAllPages {
|
|
if p.Lang() == "" {
|
|
panic("Page language missing: " + p.Title)
|
|
}
|
|
|
|
shouldBuild := p.shouldBuild()
|
|
|
|
for i, site := range h.Sites {
|
|
if strings.HasPrefix(site.Language.Lang, p.Lang()) {
|
|
site.updateBuildStats(p)
|
|
if shouldBuild {
|
|
site.Pages = append(site.Pages, p)
|
|
p.Site = &site.Info
|
|
}
|
|
}
|
|
|
|
if !shouldBuild {
|
|
continue
|
|
}
|
|
|
|
if i == 0 {
|
|
site.AllPages = append(site.AllPages, p)
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
// Pull over the collections from the master site
|
|
for i := 1; i < len(h.Sites); i++ {
|
|
h.Sites[i].AllPages = h.Sites[0].AllPages
|
|
h.Sites[i].Data = h.Sites[0].Data
|
|
}
|
|
|
|
if len(h.Sites) > 1 {
|
|
pages := h.Sites[0].AllPages
|
|
allTranslations := pagesToTranslationsMap(h.multilingual, pages)
|
|
assignTranslationsToPages(allTranslations, pages)
|
|
}
|
|
}
|
|
|
|
// preRender performs build tasks that need to be done as late as possible.
|
|
// Shortcode handling is the main task in here.
|
|
// TODO(bep) We need to look at the whole handler-chain construct with he below in mind.
|
|
func (h *HugoSites) preRender(cfg BuildCfg, changed whatChanged) error {
|
|
|
|
for _, s := range h.Sites {
|
|
if err := s.setCurrentLanguageConfig(); err != nil {
|
|
return err
|
|
}
|
|
// Run "render prepare"
|
|
if err := s.renderHomePage(true); err != nil {
|
|
return err
|
|
}
|
|
if err := s.renderTaxonomiesLists(true); err != nil {
|
|
return err
|
|
}
|
|
if err := s.renderListsOfTaxonomyTerms(true); err != nil {
|
|
return err
|
|
}
|
|
if err := s.renderSectionLists(true); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
for _, s := range h.Sites {
|
|
if err := s.setCurrentLanguageConfig(); err != nil {
|
|
return err
|
|
}
|
|
s.preparePagesForRender(cfg, changed)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Site) preparePagesForRender(cfg BuildCfg, changed whatChanged) {
|
|
pageChan := make(chan *Page)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
for i := 0; i < getGoMaxProcs()*4; i++ {
|
|
wg.Add(1)
|
|
go func(pages <-chan *Page, wg *sync.WaitGroup) {
|
|
defer wg.Done()
|
|
for p := range pages {
|
|
|
|
if !changed.other && p.rendered {
|
|
// No need to process it again.
|
|
continue
|
|
}
|
|
|
|
// If we got this far it means that this is either a new Page pointer
|
|
// or a template or similar has changed so wee need to do a rerendering
|
|
// of the shortcodes etc.
|
|
|
|
// Mark it as rendered
|
|
p.rendered = true
|
|
|
|
// If in watch mode, we need to keep the original so we can
|
|
// repeat this process on rebuild.
|
|
if cfg.Watching {
|
|
p.rawContentCopy = make([]byte, len(p.rawContent))
|
|
copy(p.rawContentCopy, p.rawContent)
|
|
} else {
|
|
// Just reuse the same slice.
|
|
p.rawContentCopy = p.rawContent
|
|
}
|
|
|
|
if p.Markup == "markdown" {
|
|
tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContentCopy)
|
|
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
|
|
p.rawContentCopy = tmpContent
|
|
}
|
|
|
|
if err := handleShortcodes(p, s.owner.tmpl); err != nil {
|
|
jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
|
|
}
|
|
|
|
if p.Markup != "html" {
|
|
|
|
// Now we know enough to create a summary of the page and count some words
|
|
summaryContent, err := p.setUserDefinedSummaryIfProvided()
|
|
|
|
if err != nil {
|
|
jww.ERROR.Printf("Failed to set user defined summary for page %q: %s", p.Path(), err)
|
|
} else if summaryContent != nil {
|
|
p.rawContentCopy = summaryContent.content
|
|
}
|
|
|
|
p.Content = helpers.BytesToHTML(p.rawContentCopy)
|
|
|
|
if summaryContent == nil {
|
|
p.setAutoSummary()
|
|
}
|
|
|
|
} else {
|
|
p.Content = helpers.BytesToHTML(p.rawContentCopy)
|
|
}
|
|
|
|
// no need for this anymore
|
|
p.rawContentCopy = nil
|
|
|
|
//analyze for raw stats
|
|
p.analyzePage()
|
|
|
|
}
|
|
}(pageChan, wg)
|
|
}
|
|
|
|
for _, p := range s.Pages {
|
|
pageChan <- p
|
|
}
|
|
|
|
close(pageChan)
|
|
|
|
wg.Wait()
|
|
|
|
}
|
|
|
|
// Pages returns all pages for all sites.
|
|
func (h *HugoSites) Pages() Pages {
|
|
return h.Sites[0].AllPages
|
|
}
|
|
|
|
func handleShortcodes(p *Page, t tpl.Template) error {
|
|
if len(p.contentShortCodes) > 0 {
|
|
jww.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName())
|
|
shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
p.rawContentCopy, err = replaceShortcodeTokens(p.rawContentCopy, shortcodePlaceholderPrefix, shortcodes)
|
|
|
|
if err != nil {
|
|
jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *Site) updateBuildStats(page *Page) {
|
|
if page.IsDraft() {
|
|
s.draftCount++
|
|
}
|
|
|
|
if page.IsFuture() {
|
|
s.futureCount++
|
|
}
|
|
|
|
if page.IsExpired() {
|
|
s.expiredCount++
|
|
}
|
|
}
|
|
|
|
// Convenience func used in tests to build a single site/language excluding render phase.
|
|
func buildSiteSkipRender(s *Site, additionalTemplates ...string) error {
|
|
return doBuildSite(s, false, additionalTemplates...)
|
|
}
|
|
|
|
// Convenience func used in tests to build a single site/language including render phase.
|
|
func buildAndRenderSite(s *Site, additionalTemplates ...string) error {
|
|
return doBuildSite(s, true, additionalTemplates...)
|
|
}
|
|
|
|
// Convenience func used in tests to build a single site/language.
|
|
func doBuildSite(s *Site, render bool, additionalTemplates ...string) error {
|
|
sites, err := newHugoSites(s)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
addTemplates := func(templ tpl.Template) error {
|
|
for i := 0; i < len(additionalTemplates); i += 2 {
|
|
err := templ.AddTemplate(additionalTemplates[i], additionalTemplates[i+1])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
config := BuildCfg{SkipRender: !render, withTemplate: addTemplates}
|
|
return sites.Build(config)
|
|
}
|
|
|
|
// Convenience func used in tests.
|
|
func newHugoSitesFromSourceAndLanguages(input []source.ByteSource, languages helpers.Languages) (*HugoSites, error) {
|
|
if len(languages) == 0 {
|
|
panic("Must provide at least one language")
|
|
}
|
|
first := &Site{
|
|
Source: &source.InMemorySource{ByteSource: input},
|
|
Language: languages[0],
|
|
}
|
|
if len(languages) == 1 {
|
|
return newHugoSites(first)
|
|
}
|
|
|
|
sites := make([]*Site, len(languages))
|
|
sites[0] = first
|
|
for i := 1; i < len(languages); i++ {
|
|
sites[i] = &Site{Language: languages[i]}
|
|
}
|
|
|
|
return newHugoSites(sites...)
|
|
|
|
}
|
|
|
|
// Convenience func used in tests.
|
|
func newHugoSitesFromLanguages(languages helpers.Languages) (*HugoSites, error) {
|
|
return newHugoSitesFromSourceAndLanguages(nil, languages)
|
|
}
|
|
|
|
func newHugoSitesDefaultLanguage() (*HugoSites, error) {
|
|
return newHugoSitesFromSourceAndLanguages(nil, helpers.Languages{helpers.NewDefaultLanguage()})
|
|
}
|