From 1f1c62e6c7fb5c420710fd15ac11995e6c546897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 4 Mar 2024 10:16:56 +0100 Subject: [PATCH] Add segments config + --renderSegments flag Named segments can be defined in `hugo.toml`. * Eeach segment consists of zero or more `exclude` filters and zero or more `include` filters. * Eeach filter consists of one or more field Glob matchers. * Eeach filter in a section (`exclude` or `include`) is ORed together, each matcher in a filter is ANDed together. The current list of fields that can be filtered are: * path as defined in https://gohugo.io/methods/page/path/ * kind * lang * output (output format, e.g. html). It is recommended to put coarse grained filters (e.g. for language and output format) in the excludes section, e.g.: ```toml [segments.segment1] [[segments.segment1.excludes]] lang = "n*" [[segments.segment1.excludes]] no = "en" output = "rss" [[segments.segment1.includes]] term = "{home,term,taxonomy}" [[segments.segment1.includes]] path = "{/docs,/docs/**}" ``` By default, Hugo will render all segments, but you can enable filters by setting the `renderSegments` option or `--renderSegments` flag, e.g: ``` hugo --renderSegments segment1,segment2 ``` For segment `segment1` in the configuration above, this will: * Skip rendering of all languages matching `n*`, e.g. `no`. * Skip rendering of the output format `rss` for the `en` language. * It will render all pages of kind `home`, `term` or `taxonomy` * It will render the `/docs` section and all pages below. Fixes #10106 --- commands/commandeer.go | 1 + config/allconfig/allconfig.go | 10 + config/allconfig/alldecoders.go | 10 + hugolib/hugo_sites.go | 4 + hugolib/hugo_sites_build.go | 15 +- hugolib/page.go | 14 + hugolib/segments/segments.go | 257 ++++++++++++++++++ hugolib/segments/segments_integration_test.go | 76 ++++++ hugolib/segments/segments_test.go | 115 ++++++++ hugolib/site_render.go | 2 +- 10 files changed, 501 insertions(+), 3 deletions(-) create mode 100644 hugolib/segments/segments.go create mode 100644 hugolib/segments/segments_integration_test.go create mode 100644 hugolib/segments/segments_test.go diff --git a/commands/commandeer.go b/commands/commandeer.go index 1e5928ff9..a0e4e3836 100644 --- a/commands/commandeer.go +++ b/commands/commandeer.go @@ -521,6 +521,7 @@ func applyLocalFlagsBuildConfig(cmd *cobra.Command, r *rootCommand) { cmd.Flags().StringP("cacheDir", "", "", "filesystem path to cache directory") _ = cmd.Flags().SetAnnotation("cacheDir", cobra.BashCompSubdirsInDir, []string{}) cmd.Flags().StringP("contentDir", "c", "", "filesystem path to content directory") + cmd.Flags().StringSliceP("renderSegments", "", []string{}, "named segments to render (configured in the segments config)") _ = cmd.Flags().SetAnnotation("theme", cobra.BashCompSubdirsInDir, []string{"themes"}) } diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index 3a7908d55..f0e72dabc 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -39,6 +39,7 @@ import ( "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/deploy/deployconfig" "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/hugolib/segments" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/media" @@ -139,6 +140,9 @@ type Config struct { // a slice of page matcher and params to apply to those pages. Cascade *config.ConfigNamespace[[]page.PageMatcherParamsConfig, map[page.PageMatcher]maps.Params] `mapstructure:"-"` + // The segments defines segments for the site. Used for partial/segmented builds. + Segments *config.ConfigNamespace[map[string]segments.SegmentConfig, segments.Segments] `mapstructure:"-"` + // Menu configuration. // {"refs": ["config:languages:menus"] } Menus *config.ConfigNamespace[map[string]navigation.MenuConfig, navigation.Menus] `mapstructure:"-"` @@ -366,6 +370,7 @@ func (c *Config) CompileConfig(logger loggers.Logger) error { CreateTitle: helpers.GetTitleFunc(c.TitleCaseStyle), IsUglyURLSection: isUglyURL, IgnoreFile: ignoreFile, + SegmentFilter: c.Segments.Config.Get(func(s string) { logger.Warnf("Render segment %q not found in configuration", s) }, c.RootConfig.RenderSegments...), MainSections: c.MainSections, Clock: clock, transientErr: transientErr, @@ -402,6 +407,7 @@ type ConfigCompiled struct { CreateTitle func(s string) string IsUglyURLSection func(section string) bool IgnoreFile func(filename string) bool + SegmentFilter segments.SegmentFilter MainSections []string Clock time.Time @@ -474,6 +480,10 @@ type RootConfig struct { // A list of languages to disable. DisableLanguages []string + // The named segments to render. + // This needs to match the name of the segment in the segments configuration. + RenderSegments []string + // Disable the injection of the Hugo generator tag on the home page. DisableHugoGeneratorInject bool diff --git a/config/allconfig/alldecoders.go b/config/allconfig/alldecoders.go index 5d31d5d35..7d968e4ad 100644 --- a/config/allconfig/alldecoders.go +++ b/config/allconfig/alldecoders.go @@ -25,11 +25,13 @@ import ( "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/deploy/deployconfig" + "github.com/gohugoio/hugo/hugolib/segments" "github.com/gohugoio/hugo/langs" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/minifiers" "github.com/gohugoio/hugo/modules" + "github.com/gohugoio/hugo/navigation" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/related" @@ -120,6 +122,14 @@ var allDecoderSetups = map[string]decodeWeight{ return err }, }, + "segments": { + key: "segments", + decode: func(d decodeWeight, p decodeConfig) error { + var err error + p.c.Segments, err = segments.DecodeSegments(p.p.GetStringMap(d.key)) + return err + }, + }, "server": { key: "server", decode: func(d decodeWeight, p decodeConfig) error { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 671785f44..cf939ba92 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -410,6 +410,10 @@ type BuildCfg struct { // shouldRender returns whether this output format should be rendered or not. func (cfg *BuildCfg) shouldRender(p *pageState) bool { + if p.skipRender() { + return false + } + if !p.renderOnce { return true } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 7b8a6ef23..c33f06bd5 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -28,16 +28,16 @@ import ( "github.com/bep/logg" "github.com/gohugoio/hugo/cache/dynacache" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugofs/files" "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/hugolib/segments" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/publisher" "github.com/gohugoio/hugo/source" "github.com/gohugoio/hugo/tpl" - "github.com/gohugoio/hugo/hugofs" - "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/para" @@ -318,9 +318,20 @@ func (h *HugoSites) render(l logg.LevelLogger, config *BuildCfg) error { i := 0 for _, s := range h.Sites { + segmentFilter := s.conf.C.SegmentFilter + if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Lang: s.language.Lang}) { + l.Logf("skip language %q not matching segments set in --renderSegments", s.language.Lang) + continue + } + siteRenderContext.languageIdx = s.languagei h.currentSite = s for siteOutIdx, renderFormat := range s.renderFormats { + if segmentFilter.ShouldExcludeCoarse(segments.SegmentMatcherFields{Output: renderFormat.Name, Lang: s.language.Lang}) { + l.Logf("skip output format %q for language %q not matching segments set in --renderSegments", renderFormat.Name, s.language.Lang) + continue + } + siteRenderContext.outIdx = siteOutIdx siteRenderContext.sitesOutIdx = i i++ diff --git a/hugolib/page.go b/hugolib/page.go index 0622a15fe..5d083fd5b 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -22,6 +22,7 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/hugolib/doctree" + "github.com/gohugoio/hugo/hugolib/segments" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/output" @@ -152,6 +153,19 @@ func (p *pageState) reusePageOutputContent() bool { return p.pageOutputTemplateVariationsState.Load() == 1 } +func (p *pageState) skipRender() bool { + b := p.s.conf.C.SegmentFilter.ShouldExcludeFine( + segments.SegmentMatcherFields{ + Path: p.Path(), + Kind: p.Kind(), + Lang: p.Lang(), + Output: p.pageOutput.f.Name, + }, + ) + + return b +} + func (po *pageState) isRenderedAny() bool { for _, o := range po.pageOutputs { if o.isRendered() { diff --git a/hugolib/segments/segments.go b/hugolib/segments/segments.go new file mode 100644 index 000000000..8f7c18121 --- /dev/null +++ b/hugolib/segments/segments.go @@ -0,0 +1,257 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segments + +import ( + "fmt" + + "github.com/gobwas/glob" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/predicate" + "github.com/gohugoio/hugo/config" + hglob "github.com/gohugoio/hugo/hugofs/glob" + "github.com/mitchellh/mapstructure" +) + +// Segments is a collection of named segments. +type Segments struct { + s map[string]excludeInclude +} + +type excludeInclude struct { + exclude predicate.P[SegmentMatcherFields] + include predicate.P[SegmentMatcherFields] +} + +// ShouldExcludeCoarse returns whether the given fields should be excluded. +// This is used for the coarser grained checks, e.g. language and output format. +// Note that ShouldExcludeCoarse(fields) == ShouldExcludeFine(fields) may +// not always be true, but ShouldExcludeCoarse(fields) == true == ShouldExcludeFine(fields) +// will always be truthful. +func (e excludeInclude) ShouldExcludeCoarse(fields SegmentMatcherFields) bool { + return e.exclude != nil && e.exclude(fields) +} + +// ShouldExcludeFine returns whether the given fields should be excluded. +// This is used for the finer grained checks, e.g. on invididual pages. +func (e excludeInclude) ShouldExcludeFine(fields SegmentMatcherFields) bool { + if e.exclude != nil && e.exclude(fields) { + return true + } + return e.include != nil && !e.include(fields) +} + +type SegmentFilter interface { + // ShouldExcludeCoarse returns whether the given fields should be excluded on a coarse level. + ShouldExcludeCoarse(SegmentMatcherFields) bool + + // ShouldExcludeFine returns whether the given fields should be excluded on a fine level. + ShouldExcludeFine(SegmentMatcherFields) bool +} + +type segmentFilter struct { + coarse predicate.P[SegmentMatcherFields] + fine predicate.P[SegmentMatcherFields] +} + +func (f segmentFilter) ShouldExcludeCoarse(field SegmentMatcherFields) bool { + return f.coarse(field) +} + +func (f segmentFilter) ShouldExcludeFine(fields SegmentMatcherFields) bool { + return f.fine(fields) +} + +var ( + matchAll = func(SegmentMatcherFields) bool { return true } + matchNothing = func(SegmentMatcherFields) bool { return false } +) + +// Get returns a SegmentFilter for the given segments. +func (sms Segments) Get(onNotFound func(s string), ss ...string) SegmentFilter { + if ss == nil { + return segmentFilter{coarse: matchNothing, fine: matchNothing} + } + var sf segmentFilter + for _, s := range ss { + if seg, ok := sms.s[s]; ok { + if sf.coarse == nil { + sf.coarse = seg.ShouldExcludeCoarse + } else { + sf.coarse = sf.coarse.Or(seg.ShouldExcludeCoarse) + } + if sf.fine == nil { + sf.fine = seg.ShouldExcludeFine + } else { + sf.fine = sf.fine.Or(seg.ShouldExcludeFine) + } + } else if onNotFound != nil { + onNotFound(s) + } + } + + if sf.coarse == nil { + sf.coarse = matchAll + } + if sf.fine == nil { + sf.fine = matchAll + } + + return sf +} + +type SegmentConfig struct { + Excludes []SegmentMatcherFields + Includes []SegmentMatcherFields +} + +// SegmentMatcherFields is a matcher for a segment include or exclude. +// All of these are Glob patterns. +type SegmentMatcherFields struct { + Kind string + Path string + Lang string + Output string +} + +func getGlob(s string) (glob.Glob, error) { + if s == "" { + return nil, nil + } + g, err := hglob.GetGlob(s) + if err != nil { + return nil, fmt.Errorf("failed to compile Glob %q: %w", s, err) + } + return g, nil +} + +func compileSegments(f []SegmentMatcherFields) (predicate.P[SegmentMatcherFields], error) { + if f == nil { + return func(SegmentMatcherFields) bool { return false }, nil + } + var ( + result predicate.P[SegmentMatcherFields] + section predicate.P[SegmentMatcherFields] + ) + + addToSection := func(matcherFields SegmentMatcherFields, f func(fields SegmentMatcherFields) string) error { + s1 := f(matcherFields) + g, err := getGlob(s1) + if err != nil { + return err + } + matcher := func(fields SegmentMatcherFields) bool { + s2 := f(fields) + if s2 == "" { + return false + } + return g.Match(s2) + } + if section == nil { + section = matcher + } else { + section = section.And(matcher) + } + return nil + } + + for _, fields := range f { + if fields.Kind != "" { + if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Kind }); err != nil { + return result, err + } + } + if fields.Path != "" { + if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Path }); err != nil { + return result, err + } + } + if fields.Lang != "" { + if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Lang }); err != nil { + return result, err + } + } + if fields.Output != "" { + if err := addToSection(fields, func(fields SegmentMatcherFields) string { return fields.Output }); err != nil { + return result, err + } + } + + if result == nil { + result = section + } else { + result = result.Or(section) + } + section = nil + + } + + return result, nil +} + +func DecodeSegments(in map[string]any) (*config.ConfigNamespace[map[string]SegmentConfig, Segments], error) { + buildConfig := func(in any) (Segments, any, error) { + sms := Segments{ + s: map[string]excludeInclude{}, + } + m, err := maps.ToStringMapE(in) + if err != nil { + return sms, nil, err + } + if m == nil { + m = map[string]any{} + } + m = maps.CleanConfigStringMap(m) + + var scfgm map[string]SegmentConfig + if err := mapstructure.Decode(m, &scfgm); err != nil { + return sms, nil, err + } + + for k, v := range scfgm { + var ( + include predicate.P[SegmentMatcherFields] + exclude predicate.P[SegmentMatcherFields] + err error + ) + if v.Excludes != nil { + exclude, err = compileSegments(v.Excludes) + if err != nil { + return sms, nil, err + } + } + if v.Includes != nil { + include, err = compileSegments(v.Includes) + if err != nil { + return sms, nil, err + } + } + + ei := excludeInclude{ + exclude: exclude, + include: include, + } + sms.s[k] = ei + + } + + return sms, nil, nil + } + + ns, err := config.DecodeNamespace[map[string]SegmentConfig](in, buildConfig) + if err != nil { + return nil, fmt.Errorf("failed to decode segments: %w", err) + } + return ns, nil +} diff --git a/hugolib/segments/segments_integration_test.go b/hugolib/segments/segments_integration_test.go new file mode 100644 index 000000000..465a7abe0 --- /dev/null +++ b/hugolib/segments/segments_integration_test.go @@ -0,0 +1,76 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package segments_test + +import ( + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/hugolib" +) + +func TestSegments(t *testing.T) { + files := ` +-- hugo.toml -- +baseURL = "https://example.org/" +renderSegments = ["docs"] +[languages] +[languages.en] +weight = 1 +[languages.no] +weight = 2 +[languages.nb] +weight = 3 +[segments] +[segments.docs] +[[segments.docs.includes]] +kind = "{home,taxonomy,term}" +[[segments.docs.includes]] +path = "{/docs,/docs/**}" +[[segments.docs.excludes]] +path = "/blog/**" +[[segments.docs.excludes]] +lang = "n*" +output = "rss" +[[segments.docs.excludes]] +output = "json" +-- layouts/_default/single.html -- +Single: {{ .Title }}|{{ .RelPermalink }}| +-- layouts/_default/list.html -- +List: {{ .Title }}|{{ .RelPermalink }}| +-- content/docs/_index.md -- +-- content/docs/section1/_index.md -- +-- content/docs/section1/page1.md -- +--- +title: "Docs Page 1" +tags: ["tag1", "tag2"] +--- +-- content/blog/_index.md -- +-- content/blog/section1/page1.md -- +--- +title: "Blog Page 1" +tags: ["tag1", "tag2"] +--- +` + + b := hugolib.Test(t, files) + b.Assert(b.H.Configs.Base.RootConfig.RenderSegments, qt.DeepEquals, []string{"docs"}) + + b.AssertFileContent("public/docs/section1/page1/index.html", "Docs Page 1") + b.AssertFileExists("public/blog/section1/page1/index.html", false) + b.AssertFileExists("public/index.html", true) + b.AssertFileExists("public/index.xml", true) + b.AssertFileExists("public/no/index.html", true) + b.AssertFileExists("public/no/index.xml", false) +} diff --git a/hugolib/segments/segments_test.go b/hugolib/segments/segments_test.go new file mode 100644 index 000000000..1a2dfb97b --- /dev/null +++ b/hugolib/segments/segments_test.go @@ -0,0 +1,115 @@ +package segments + +import ( + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestCompileSegments(t *testing.T) { + c := qt.New(t) + + c.Run("excludes", func(c *qt.C) { + fields := []SegmentMatcherFields{ + { + Lang: "n*", + Output: "rss", + }, + } + + match, err := compileSegments(fields) + c.Assert(err, qt.IsNil) + + check := func() { + c.Assert(match, qt.IsNotNil) + c.Assert(match(SegmentMatcherFields{Lang: "no"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Lang: "no", Kind: "page"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss"}), qt.Equals, true) + c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "html"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Kind: "page"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss", Kind: "page"}), qt.Equals, true) + } + + check() + + fields = []SegmentMatcherFields{ + { + Path: "/blog/**", + }, + { + Lang: "n*", + Output: "rss", + }, + } + + match, err = compileSegments(fields) + c.Assert(err, qt.IsNil) + check() + c.Assert(match(SegmentMatcherFields{Path: "/blog/foo"}), qt.Equals, true) + }) + + c.Run("includes", func(c *qt.C) { + fields := []SegmentMatcherFields{ + { + Path: "/docs/**", + }, + { + Lang: "no", + Output: "rss", + }, + } + + match, err := compileSegments(fields) + c.Assert(err, qt.IsNil) + c.Assert(match, qt.IsNotNil) + c.Assert(match(SegmentMatcherFields{Lang: "no"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Kind: "page"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/blog/foo"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Lang: "en"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "rss"}), qt.Equals, true) + c.Assert(match(SegmentMatcherFields{Lang: "no", Output: "html"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/docs/foo"}), qt.Equals, true) + }) + + c.Run("includes variant1", func(c *qt.C) { + c.Skip() + + fields := []SegmentMatcherFields{ + { + Kind: "home", + }, + { + Path: "{/docs,/docs/**}", + }, + } + + match, err := compileSegments(fields) + c.Assert(err, qt.IsNil) + c.Assert(match, qt.IsNotNil) + c.Assert(match(SegmentMatcherFields{Path: "/blog/foo"}), qt.Equals, false) + c.Assert(match(SegmentMatcherFields{Kind: "page", Path: "/docs/foo"}), qt.Equals, true) + c.Assert(match(SegmentMatcherFields{Kind: "home", Path: "/"}), qt.Equals, true) + }) +} + +func BenchmarkSegmentsMatch(b *testing.B) { + fields := []SegmentMatcherFields{ + { + Path: "/docs/**", + }, + { + Lang: "no", + Output: "rss", + }, + } + + match, err := compileSegments(fields) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + match(SegmentMatcherFields{Lang: "no", Output: "rss"}) + } +} diff --git a/hugolib/site_render.go b/hugolib/site_render.go index 86be897fa..1cd509fea 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -271,7 +271,7 @@ func (s *Site) renderAliases() error { p := n.(*pageState) // We cannot alias a page that's not rendered. - if p.m.noLink() { + if p.m.noLink() || p.skipRender() { return false, nil }