Add segments config + --renderSegments flag

Named segments can be defined in `hugo.toml`.

* Eeach segment consists of zero or more `exclude` filters and zero or more `include` filters.
* Eeach filter consists of one or more field Glob matchers.
* Eeach filter in a section (`exclude` or `include`) is ORed together, each matcher in a filter is ANDed together.

The current list of fields that can be filtered are:

* path as defined in https://gohugo.io/methods/page/path/
* kind
* lang
* output (output format, e.g. html).

It is recommended to put coarse grained filters (e.g. for language and output format) in the excludes section, e.g.:

```toml
[segments.segment1]
  [[segments.segment1.excludes]]
    lang = "n*"
  [[segments.segment1.excludes]]
    no     = "en"
    output = "rss"
  [[segments.segment1.includes]]
    term = "{home,term,taxonomy}"
  [[segments.segment1.includes]]
    path = "{/docs,/docs/**}"
```

By default, Hugo will render all segments, but you can enable filters by setting the `renderSegments` option or `--renderSegments` flag, e.g:

```
hugo --renderSegments segment1,segment2
```

For segment `segment1` in the configuration above, this will:

* Skip rendering of all languages matching `n*`, e.g. `no`.
* Skip rendering of the output format `rss` for the `en` language.
* It will render all pages of kind `home`, `term` or `taxonomy`
* It will render the `/docs` section and all pages below.

Fixes #10106
This commit is contained in:
Bjørn Erik Pedersen 2024-03-04 10:16:56 +01:00
parent f1d755965f
commit 1f1c62e6c7
10 changed files with 501 additions and 3 deletions

View file

@ -521,6 +521,7 @@ func applyLocalFlagsBuildConfig(cmd *cobra.Command, r *rootCommand) {
cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory")
_ = cmd.Flags().SetAnnotation("cacheDir", cobra.BashCompSubdirsInDir, []string{})
cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory")
cmd.Flags().StringSliceP("renderSegments", "", []string{}, "named segments to render (configured in the segments config)")
_ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"})
}

View file

@ -39,6 +39,7 @@ import (
"github.com/gohugoio/hugo/config/services"
"github.com/gohugoio/hugo/deploy/deployconfig"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugolib/segments"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/media"
@ -139,6 +140,9 @@ type Config struct {
// a slice of page matcher and params to apply to those pages.
Cascade *config.ConfigNamespace[[]page.PageMatcherParamsConfig, map[page.PageMatcher]maps.Params] `mapstructure:"-"`
// The segments defines segments for the site. Used for partial/segmented builds.
Segments *config.ConfigNamespace[map[string]segments.SegmentConfig, segments.Segments] `mapstructure:"-"`
// Menu configuration.
// <docsmeta>{"refs": ["config:languages:menus"] }</docsmeta>
Menus *config.ConfigNamespace[map[string]navigation.MenuConfig, navigation.Menus] `mapstructure:"-"`
@ -366,6 +370,7 @@ func (c *Config) CompileConfig(logger loggers.Logger) error {
CreateTitle: helpers.GetTitleFunc(c.TitleCaseStyle),
IsUglyURLSection: isUglyURL,
IgnoreFile: ignoreFile,
SegmentFilter: c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...),
MainSections: c.MainSections,
Clock: clock,
transientErr: transientErr,
@ -402,6 +407,7 @@ type ConfigCompiled struct {
CreateTitle func(s string) string
IsUglyURLSection func(section string) bool
IgnoreFile func(filename string) bool
SegmentFilter segments.SegmentFilter
MainSections []string
Clock time.Time
@ -474,6 +480,10 @@ type RootConfig struct {
// A list of languages to disable.
DisableLanguages []string
// The named segments to render.
// This needs to match the name of the segment in the segments configuration.
RenderSegments []string
// Disable the injection of the Hugo generator tag on the home page.
DisableHugoGeneratorInject bool

View file

@ -25,11 +25,13 @@ import (
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/config/services"
"github.com/gohugoio/hugo/deploy/deployconfig"
"github.com/gohugoio/hugo/hugolib/segments"
"github.com/gohugoio/hugo/langs"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/minifiers"
"github.com/gohugoio/hugo/modules"
"github.com/gohugoio/hugo/navigation"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/related"
@ -120,6 +122,14 @@ var allDecoderSetups = map[string]decodeWeight{
return err
},
},
"segments": {
key: "segments",
decode: func(d decodeWeight, p decodeConfig) error {
var err error
p.c.Segments, err = segments.DecodeSegments(p.p.GetStringMap(d.key))
return err
},
},
"server": {
key: "server",
decode: func(d decodeWeight, p decodeConfig) error {

View file

@ -410,6 +410,10 @@ type BuildCfg struct {
// shouldRender returns whether this output format should be rendered or not.
func (cfg *BuildCfg) shouldRender(p *pageState) bool {
if p.skipRender() {
return false
}
if !p.renderOnce {
return true
}

View file

@ -28,16 +28,16 @@ import (
"github.com/bep/logg"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/hugolib/segments"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/publisher"
"github.com/gohugoio/hugo/source"
"github.com/gohugoio/hugo/tpl"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/para"
@ -318,9 +318,20 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error {
i := 0
for _, s := range h.Sites {
segmentFilter := s.conf.C.SegmentFilter
if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Lang: s.language.Lang}) {
l.Logf("skip language %q not matching segments set in --renderSegments", s.language.Lang)
continue
}
siteRenderContext.languageIdx = s.languagei
h.currentSite = s
for siteOutIdx, renderFormat := range s.renderFormats {
if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Output: renderFormat.Name, Lang: s.language.Lang}) {
l.Logf("skip output format %q for language %q not matching segments set in --renderSegments", renderFormat.Name, s.language.Lang)
continue
}
siteRenderContext.outIdx = siteOutIdx
siteRenderContext.sitesOutIdx = i
i++

View file

@ -22,6 +22,7 @@ import (
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/hugolib/segments"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/output"
@ -152,6 +153,19 @@ func (p *pageState) reusePageOutputContent() bool {
return p.pageOutputTemplateVariationsState.Load() == 1
}
func (p *pageState) skipRender() bool {
b := p.s.conf.C.SegmentFilter.ShouldExcludeFine(
segments.SegmentMatcherFields{
Path: p.Path(),
Kind: p.Kind(),
Lang: p.Lang(),
Output: p.pageOutput.f.Name,
},
)
return b
}
func (po *pageState) isRenderedAny() bool {
for _, o := range po.pageOutputs {
if o.isRendered() {

View file

@ -0,0 +1,257 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package segments
import (
"fmt"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/predicate"
"github.com/gohugoio/hugo/config"
hglob "github.com/gohugoio/hugo/hugofs/glob"
"github.com/mitchellh/mapstructure"
)
// Segments is a collection of named segments.
type Segments struct {
s map[string]excludeInclude
}
type excludeInclude struct {
exclude predicate.P[SegmentMatcherFields]
include predicate.P[SegmentMatcherFields]
}
// ShouldExcludeCoarse returns whether the given fields should be excluded.
// This is used for the coarser grained checks, e.g. language and output format.
// Note that ShouldExcludeCoarse(fields) == ShouldExcludeFine(fields) may
// not always be true, but ShouldExcludeCoarse(fields) == true == ShouldExcludeFine(fields)
// will always be truthful.
func (e excludeInclude) ShouldExcludeCoarse(fields SegmentMatcherFields) bool {
return e.exclude != nil && e.exclude(fields)
}
// ShouldExcludeFine returns whether the given fields should be excluded.
// This is used for the finer grained checks, e.g. on invididual pages.
func (e excludeInclude) ShouldExcludeFine(fields SegmentMatcherFields) bool {
if e.exclude != nil && e.exclude(fields) {
return true
}
return e.include != nil && !e.include(fields)
}
type SegmentFilter interface {
// ShouldExcludeCoarse returns whether the given fields should be excluded on a coarse level.
ShouldExcludeCoarse(SegmentMatcherFields) bool
// ShouldExcludeFine returns whether the given fields should be excluded on a fine level.
ShouldExcludeFine(SegmentMatcherFields) bool
}
type segmentFilter struct {
coarse predicate.P[SegmentMatcherFields]
fine predicate.P[SegmentMatcherFields]
}
func (f segmentFilter) ShouldExcludeCoarse(field SegmentMatcherFields) bool {
return f.coarse(field)
}
func (f segmentFilter) ShouldExcludeFine(fields SegmentMatcherFields) bool {
return f.fine(fields)
}
var (
matchAll = func(SegmentMatcherFields) bool { return true }
matchNothing = func(SegmentMatcherFields) bool { return false }
)
// Get returns a SegmentFilter for the given segments.
func (sms Segments) Get(onNotFound func(s string), ss ...string) SegmentFilter {
if ss == nil {
return segmentFilter{coarse: matchNothing, fine: matchNothing}
}
var sf segmentFilter
for _, s := range ss {
if seg, ok := sms.s[s]; ok {
if sf.coarse == nil {
sf.coarse = seg.ShouldExcludeCoarse
} else {
sf.coarse = sf.coarse.Or(seg.ShouldExcludeCoarse)
}
if sf.fine == nil {
sf.fine = seg.ShouldExcludeFine
} else {
sf.fine = sf.fine.Or(seg.ShouldExcludeFine)
}
} else if onNotFound != nil {
onNotFound(s)
}
}
if sf.coarse == nil {
sf.coarse = matchAll
}
if sf.fine == nil {
sf.fine = matchAll
}
return sf
}
type SegmentConfig struct {
Excludes []SegmentMatcherFields
Includes []SegmentMatcherFields
}
// SegmentMatcherFields is a matcher for a segment include or exclude.
// All of these are Glob patterns.
type SegmentMatcherFields struct {
Kind string
Path string
Lang string
Output string
}
func getGlob(s string) (glob.Glob, error) {
if s == "" {
return nil, nil
}
g, err := hglob.GetGlob(s)
if err != nil {
return nil, fmt.Errorf("failed to compile Glob %q: %w", s, err)
}
return g, nil
}
func compileSegments(f []SegmentMatcherFields) (predicate.P[SegmentMatcherFields], error) {
if f == nil {
return func(SegmentMatcherFields) bool { return false }, nil
}
var (
result predicate.P[SegmentMatcherFields]
section predicate.P[SegmentMatcherFields]
)
addToSection := func(matcherFields SegmentMatcherFields, f func(fields SegmentMatcherFields) string) error {
s1 := f(matcherFields)
g, err := getGlob(s1)
if err != nil {
return err
}
matcher := func(fields SegmentMatcherFields) bool {
s2 := f(fields)
if s2 == "" {
return false
}
return g.Match(s2)
}
if section == nil {
section = matcher
} else {
section = section.And(matcher)
}
return nil
}
for _, fields := range f {
if fields.Kind != "" {
if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Kind }); err != nil {
return result, err
}
}
if fields.Path != "" {
if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Path }); err != nil {
return result, err
}
}
if fields.Lang != "" {
if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Lang }); err != nil {
return result, err
}
}
if fields.Output != "" {
if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Output }); err != nil {
return result, err
}
}
if result == nil {
result = section
} else {
result = result.Or(section)
}
section = nil
}
return result, nil
}
func DecodeSegments(in map[string]any) (*config.ConfigNamespace[map[string]SegmentConfig, Segments], error) {
buildConfig := func(in any) (Segments, any, error) {
sms := Segments{
s: map[string]excludeInclude{},
}
m, err := maps.ToStringMapE(in)
if err != nil {
return sms, nil, err
}
if m == nil {
m = map[string]any{}
}
m = maps.CleanConfigStringMap(m)
var scfgm map[string]SegmentConfig
if err := mapstructure.Decode(m, &scfgm); err != nil {
return sms, nil, err
}
for k, v := range scfgm {
var (
include predicate.P[SegmentMatcherFields]
exclude predicate.P[SegmentMatcherFields]
err error
)
if v.Excludes != nil {
exclude, err = compileSegments(v.Excludes)
if err != nil {
return sms, nil, err
}
}
if v.Includes != nil {
include, err = compileSegments(v.Includes)
if err != nil {
return sms, nil, err
}
}
ei := excludeInclude{
exclude: exclude,
include: include,
}
sms.s[k] = ei
}
return sms, nil, nil
}
ns, err := config.DecodeNamespace[map[string]SegmentConfig](in, buildConfig)
if err != nil {
return nil, fmt.Errorf("failed to decode segments: %w", err)
}
return ns, nil
}

View file

@ -0,0 +1,76 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package segments_test
import (
"testing"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/hugolib"
)
func TestSegments(t *testing.T) {
files := `
-- hugo.toml --
baseURL = "https://example.org/"
renderSegments = ["docs"]
[languages]
[languages.en]
weight = 1
[languages.no]
weight = 2
[languages.nb]
weight = 3
[segments]
[segments.docs]
[[segments.docs.includes]]
kind = "{home,taxonomy,term}"
[[segments.docs.includes]]
path = "{/docs,/docs/**}"
[[segments.docs.excludes]]
path = "/blog/**"
[[segments.docs.excludes]]
lang = "n*"
output = "rss"
[[segments.docs.excludes]]
output = "json"
-- layouts/_default/single.html --
Single: {{ .Title }}|{{ .RelPermalink }}|
-- layouts/_default/list.html --
List: {{ .Title }}|{{ .RelPermalink }}|
-- content/docs/_index.md --
-- content/docs/section1/_index.md --
-- content/docs/section1/page1.md --
---
title: "Docs Page 1"
tags: ["tag1", "tag2"]
---
-- content/blog/_index.md --
-- content/blog/section1/page1.md --
---
title: "Blog Page 1"
tags: ["tag1", "tag2"]
---
`
b := hugolib.Test(t, files)
b.Assert(b.H.Configs.Base.RootConfig.RenderSegments, qt.DeepEquals, []string{"docs"})
b.AssertFileContent("public/docs/section1/page1/index.html", "Docs Page 1")
b.AssertFileExists("public/blog/section1/page1/index.html", false)
b.AssertFileExists("public/index.html", true)
b.AssertFileExists("public/index.xml", true)
b.AssertFileExists("public/no/index.html", true)
b.AssertFileExists("public/no/index.xml", false)
}

View file

@ -0,0 +1,115 @@
package segments
import (
"testing"
qt "github.com/frankban/quicktest"
)
func TestCompileSegments(t *testing.T) {
c := qt.New(t)
c.Run("excludes", func(c *qt.C) {
fields := []SegmentMatcherFields{
{
Lang: "n*",
Output: "rss",
},
}
match, err := compileSegments(fields)
c.Assert(err, qt.IsNil)
check := func() {
c.Assert(match, qt.IsNotNil)
c.Assert(match(SegmentMatcherFields{Lang: "no"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Lang: "no", Kind: "page"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss"}), qt.Equals, true)
c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "html"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Kind: "page"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss", Kind: "page"}), qt.Equals, true)
}
check()
fields = []SegmentMatcherFields{
{
Path: "/blog/**",
},
{
Lang: "n*",
Output: "rss",
},
}
match, err = compileSegments(fields)
c.Assert(err, qt.IsNil)
check()
c.Assert(match(SegmentMatcherFields{Path: "/blog/foo"}), qt.Equals, true)
})
c.Run("includes", func(c *qt.C) {
fields := []SegmentMatcherFields{
{
Path: "/docs/**",
},
{
Lang: "no",
Output: "rss",
},
}
match, err := compileSegments(fields)
c.Assert(err, qt.IsNil)
c.Assert(match, qt.IsNotNil)
c.Assert(match(SegmentMatcherFields{Lang: "no"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Kind: "page"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/blog/foo"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Lang: "en"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss"}), qt.Equals, true)
c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "html"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/docs/foo"}), qt.Equals, true)
})
c.Run("includes variant1", func(c *qt.C) {
c.Skip()
fields := []SegmentMatcherFields{
{
Kind: "home",
},
{
Path: "{/docs,/docs/**}",
},
}
match, err := compileSegments(fields)
c.Assert(err, qt.IsNil)
c.Assert(match, qt.IsNotNil)
c.Assert(match(SegmentMatcherFields{Path: "/blog/foo"}), qt.Equals, false)
c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/docs/foo"}), qt.Equals, true)
c.Assert(match(SegmentMatcherFields{Kind: "home", Path: "/"}), qt.Equals, true)
})
}
func BenchmarkSegmentsMatch(b *testing.B) {
fields := []SegmentMatcherFields{
{
Path: "/docs/**",
},
{
Lang: "no",
Output: "rss",
},
}
match, err := compileSegments(fields)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
match(SegmentMatcherFields{Lang: "no", Output: "rss"})
}
}

View file

@ -271,7 +271,7 @@ func (s *Site) renderAliases() error {
p := n.(*pageState)
// We cannot alias a page that's not rendered.
if p.m.noLink() {
if p.m.noLink() || p.skipRender() {
return false, nil
}