From 90da7664bf1f3a0ca2e18144b5deacf532c6e3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 11 Feb 2023 16:20:24 +0100 Subject: [PATCH] Add page fragments support to Related The main topic of this commit is that you can now index fragments (content heading identifiers) when calling `.Related`. You can do this by: * Configure one or more indices with type `fragments` * The name of those index configurations maps to an (optional) front matter slice with fragment references. This allows you to link page<->fragment and page<->page. * This also will index all the fragments (heading identifiers) of the pages. It's also possible to use type `fragments` indices in shortcode, e.g.: ``` {{ $related := site.RegularPages.Related .Page }} ``` But, and this is important, you need to include the shortcode using the `{{<` delimiter. Not doing so will create infinite loops and timeouts. This commit also: * Adds two new methods to Page: Fragments (can also be used to build ToC) and HeadingsFiltered (this is only used in Related Content with index type `fragments` and `enableFilter` set to true. * Consolidates all `.Related*` methods into one, which takes either a `Page` or an options map as its only argument. * Add `context.Context` to all of the content related Page API. Turns out it wasn't strictly needed for this particular feature, but it will soon become usefil, e.g. in #9339. Closes #10711 Updates #9339 Updates #10725 --- .gitignore | 2 + common/collections/slice.go | 20 ++ common/collections/slice_test.go | 15 + compare/compare.go | 16 + docs/content/en/content-management/related.md | 96 ++++-- go.mod | 1 + go.sum | 1 + hugolib/content_factory.go | 3 +- hugolib/content_map_page.go | 2 +- hugolib/embedded_shortcodes_test.go | 3 +- hugolib/hugo_sites.go | 14 +- hugolib/hugo_sites_build.go | 2 +- hugolib/hugo_sites_build_errors_test.go | 2 +- hugolib/image_test.go | 153 ---------- hugolib/language_content_dir_test.go | 3 +- hugolib/page.go | 48 ++- hugolib/page__content.go | 14 +- hugolib/page__menus.go | 7 +- hugolib/page__new.go | 3 +- hugolib/page__per_output.go | 223 ++++++++------ hugolib/page__position.go | 6 +- hugolib/page_test.go | 53 ++-- hugolib/shortcode.go | 101 ++++--- hugolib/shortcode_page.go | 41 ++- hugolib/shortcode_test.go | 28 +- hugolib/site.go | 22 +- hugolib/site_render.go | 3 +- hugolib/site_test.go | 7 +- hugolib/testhelpers_test.go | 3 +- lazy/init.go | 30 +- lazy/init_test.go | 37 +-- markup/asciidocext/convert.go | 26 +- markup/asciidocext/convert_test.go | 82 +----- markup/converter/converter.go | 2 +- markup/goldmark/convert.go | 10 +- markup/goldmark/toc.go | 10 +- markup/tableofcontents/tableofcontents.go | 110 ++++++- .../tableofcontents/tableofcontents_test.go | 85 +++++- related/integration_test.go | 121 ++++++++ related/inverted_index.go | 277 +++++++++++++----- related/inverted_index_test.go | 41 +-- resources/errorResource.go | 3 +- resources/image_test.go | 3 +- resources/page/page.go | 41 ++- resources/page/page_lazy_contentprovider.go | 81 ++--- resources/page/page_marshaljson.autogen.go | 34 --- resources/page/page_nop.go | 35 ++- resources/page/pagegroup.go | 10 +- resources/page/pagegroup_test.go | 15 +- resources/page/pages.go | 15 - resources/page/pages_related.go | 79 +++-- resources/page/pages_related_test.go | 25 +- resources/page/pages_sort.go | 5 +- resources/page/pages_sort_test.go | 9 +- resources/page/pagination_test.go | 23 +- resources/page/testhelpers_test.go | 34 ++- resources/postpub/postpub.go | 5 +- resources/resource.go | 3 +- resources/resource/resourcetypes.go | 6 +- .../integrity/integrity_test.go | 3 +- .../minifier/minify_test.go | 3 +- resources/transform.go | 5 +- resources/transform_test.go | 21 +- .../texttemplate/hugo_template.go | 1 + tpl/transform/transform.go | 5 +- tpl/transform/transform_test.go | 5 +- 66 files changed, 1363 insertions(+), 829 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..00b5b2e80 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ + +*.test \ No newline at end of file diff --git a/common/collections/slice.go b/common/collections/slice.go index 51cb6ec1f..bf5c7b52b 100644 --- a/common/collections/slice.go +++ b/common/collections/slice.go @@ -15,6 +15,7 @@ package collections import ( "reflect" + "sort" ) // Slicer defines a very generic way to create a typed slice. This is used @@ -74,3 +75,22 @@ func StringSliceToInterfaceSlice(ss []string) []any { return result } + +type SortedStringSlice []string + +// Contains returns true if s is in ss. +func (ss SortedStringSlice) Contains(s string) bool { + i := sort.SearchStrings(ss, s) + return i < len(ss) && ss[i] == s +} + +// Count returns the number of times s is in ss. +func (ss SortedStringSlice) Count(s string) int { + var count int + i := sort.SearchStrings(ss, s) + for i < len(ss) && ss[i] == s { + count++ + i++ + } + return count +} diff --git a/common/collections/slice_test.go b/common/collections/slice_test.go index 8e6553994..5788b9161 100644 --- a/common/collections/slice_test.go +++ b/common/collections/slice_test.go @@ -122,3 +122,18 @@ func TestSlice(t *testing.T) { c.Assert(test.expected, qt.DeepEquals, result, errMsg) } } + +func TestSortedStringSlice(t *testing.T) { + t.Parallel() + c := qt.New(t) + + var s SortedStringSlice = []string{"a", "b", "b", "b", "c", "d"} + + c.Assert(s.Contains("a"), qt.IsTrue) + c.Assert(s.Contains("b"), qt.IsTrue) + c.Assert(s.Contains("z"), qt.IsFalse) + c.Assert(s.Count("b"), qt.Equals, 3) + c.Assert(s.Count("z"), qt.Equals, 0) + c.Assert(s.Count("a"), qt.Equals, 1) + +} diff --git a/compare/compare.go b/compare/compare.go index de97690c7..67bb1c125 100644 --- a/compare/compare.go +++ b/compare/compare.go @@ -36,3 +36,19 @@ type ProbablyEqer interface { type Comparer interface { Compare(other any) int } + +// Eq returns whether v1 is equal to v2. +// It will use the Eqer interface if implemented, which +// defines equals when two value are interchangeable +// in the Hugo templates. +func Eq(v1, v2 any) bool { + if v1 == nil || v2 == nil { + return v1 == v2 + } + + if eqer, ok := v1.(Eqer); ok { + return eqer.Eq(v2) + } + + return v1 == v2 +} diff --git a/docs/content/en/content-management/related.md b/docs/content/en/content-management/related.md index 2d2077c81..bd3a5d466 100644 --- a/docs/content/en/content-management/related.md +++ b/docs/content/en/content-management/related.md @@ -31,40 +31,82 @@ To list up to 5 related pages (which share the same _date_ or _keyword_ paramete {{ end }} {{< /code >}} -### Methods +The `Related` method takes one argument which may be a `Page` or a options map. The options map have these options: -Here is the list of "Related" methods available on a page collection such `.RegularPages`. +indices +: The indices to search in. -#### .Related PAGE +document +: The document to search for related content for. -Returns a collection of pages related the given one. +namedSlices +: The keywords to search for. + +fragments +: Fragments holds a a list of special keywords that is used for indices configured as type "fragments". This will match the fragment identifiers of the documents. + +A fictional example using all of the above options: ```go-html-template -{{ $related := site.RegularPages.Related . }} -``` - -#### .RelatedIndices PAGE INDICE1 [INDICE2 ...] - -Returns a collection of pages related to a given one restricted to a list of indices. - -```go-html-template -{{ $related := site.RegularPages.RelatedIndices . "tags" "date" }} -``` - -#### .RelatedTo KEYVALS [KEYVALS2 ...] - -Returns a collection of pages related together by a set of indices and their match. - -In order to build those set and pass them as argument, one must use the `keyVals` function where the first argument would be the `indice` and the consecutive ones its potential `matches`. - -```go-html-template -{{ $related := site.RegularPages.RelatedTo ( keyVals "tags" "hugo" "rocks") ( keyVals "date" .Date ) }} +{{ $page := . }} +{{ $opts := + "indices" (slice "tags" "keywords") + "document" $page + "namedSlices" (slice (keyVals "tags" "hugo" "rocks") (keyVals "date" $page.Date)) + "fragments" (slice "heading-1" "heading-2") +}} ``` {{% note %}} -Read [this blog article](https://regisphilibert.com/blog/2018/04/hugo-optmized-relashionships-with-related-content/) for a great explanation of more advanced usage of this feature. +We improved and simplified this feature in Hugo 0.111.0. Before this we had 3 different methods: `Related`, `RelatedTo` and `RelatedIndicies`. Now we have only one method: `Related`. The old methods are still available but deprecated. Also see [this blog article](https://regisphilibert.com/blog/2018/04/hugo-optmized-relashionships-with-related-content/) for a great explanation of more advanced usage of this feature. {{% /note %}} +## Index Content Headings in Related Content + +{{< new-in "0.111.0" >}} + +Hugo can index the headings in your content and use this to find related content. You can enable this by adding a index of type `fragments` to your `related` configuration: + + +```toml +[related] +threshold = 20 +includeNewer = true +toLower = false +[[related.indices]] +name = "fragmentrefs" +type = "fragments" +applyFilter = false +weight = 80 +``` + +* The `name` maps to a optional front matter slice attribute that can be used to link from the page level down to the fragment/heading level. +* If `applyFilter`is enabled, the `.HeadingsFiltered` on each page in the result will reflect the filtered headings. This is useful if you want to show the headings in the related content listing: + +```go-html-template +{{ $related := .Site.RegularPages.Related . | first 5 }} +{{ with $related }} +

See Also

+ +{{ end }} +``` + ## Configure Related Content Hugo provides a sensible default configuration of Related Content, but you can fine-tune this in your configuration, on the global or language level if needed. @@ -109,6 +151,12 @@ toLower name : The index name. This value maps directly to a page param. Hugo supports string values (`author` in the example) and lists (`tags`, `keywords` etc.) and time and date objects. +type +: {{< new-in "0.111.0" >}}. One of `basic`(default) or `fragments`. + +applyFilter +: {{< new-in "0.111.0" >}}. Apply a `type` specific filter to the result of a search. This is currently only used for the `fragments` type. + weight : An integer weight that indicates _how important_ this parameter is relative to the other parameters. It can be 0, which has the effect of turning this index off, or even negative. Test with different values to see what fits your content best. diff --git a/go.mod b/go.mod index d53a7a06e..5ae9efd26 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( github.com/yuin/goldmark v1.5.4 go.uber.org/atomic v1.10.0 gocloud.dev v0.28.0 + golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 golang.org/x/image v0.0.0-20211028202545-6944b10bf410 golang.org/x/net v0.4.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 45d1f4ebe..48c2ba125 100644 --- a/go.sum +++ b/go.sum @@ -2002,6 +2002,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/hugolib/content_factory.go b/hugolib/content_factory.go index 017a0bc97..e22f46513 100644 --- a/hugolib/content_factory.go +++ b/hugolib/content_factory.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "io" "path/filepath" @@ -83,7 +84,7 @@ func (f ContentFactory) ApplyArchetypeTemplate(w io.Writer, p page.Page, archety return fmt.Errorf("failed to parse archetype template: %s: %w", err, err) } - result, err := executeToString(ps.s.Tmpl(), templ, d) + result, err := executeToString(context.TODO(), ps.s.Tmpl(), templ, d) if err != nil { return fmt.Errorf("failed to execute archetype template: %s: %w", err, err) } diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index d8f28286c..70c5d6a27 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -171,7 +171,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB return nil, err } - ps.init.Add(func() (any, error) { + ps.init.Add(func(context.Context) (any, error) { pp, err := newPagePaths(s, ps, metaProvider) if err != nil { return nil, err diff --git a/hugolib/embedded_shortcodes_test.go b/hugolib/embedded_shortcodes_test.go index 1707bcfa7..1e06494bf 100644 --- a/hugolib/embedded_shortcodes_test.go +++ b/hugolib/embedded_shortcodes_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "encoding/json" "fmt" "html/template" @@ -70,7 +71,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) { c.Assert(len(s.RegularPages()), qt.Equals, 1) - content, err := s.RegularPages()[0].Content() + content, err := s.RegularPages()[0].Content(context.Background()) c.Assert(err, qt.IsNil) output := cast.ToString(content) diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 569c27be5..cdc5d97fb 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -194,7 +194,7 @@ func (h *hugoSitesInit) Reset() { } func (h *HugoSites) Data() map[string]any { - if _, err := h.init.data.Do(); err != nil { + if _, err := h.init.data.Do(context.Background()); err != nil { h.SendError(fmt.Errorf("failed to load data: %w", err)) return nil } @@ -202,7 +202,7 @@ func (h *HugoSites) Data() map[string]any { } func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) { - if _, err := h.init.gitInfo.Do(); err != nil { + if _, err := h.init.gitInfo.Do(context.Background()); err != nil { return source.GitInfo{}, err } @@ -214,7 +214,7 @@ func (h *HugoSites) gitInfoForPage(p page.Page) (source.GitInfo, error) { } func (h *HugoSites) codeownersForPage(p page.Page) ([]string, error) { - if _, err := h.init.gitInfo.Do(); err != nil { + if _, err := h.init.gitInfo.Do(context.Background()); err != nil { return nil, err } @@ -363,7 +363,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { donec: make(chan bool), } - h.init.data.Add(func() (any, error) { + h.init.data.Add(func(context.Context) (any, error) { err := h.loadData(h.PathSpec.BaseFs.Data.Dirs) if err != nil { return nil, fmt.Errorf("failed to load data: %w", err) @@ -371,7 +371,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) - h.init.layouts.Add(func() (any, error) { + h.init.layouts.Add(func(context.Context) (any, error) { for _, s := range h.Sites { if err := s.Tmpl().(tpl.TemplateManager).MarkReady(); err != nil { return nil, err @@ -380,7 +380,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) - h.init.translations.Add(func() (any, error) { + h.init.translations.Add(func(context.Context) (any, error) { if len(h.Sites) > 1 { allTranslations := pagesToTranslationsMap(h.Sites) assignTranslationsToPages(allTranslations, h.Sites) @@ -389,7 +389,7 @@ func newHugoSites(cfg deps.DepsCfg, sites ...*Site) (*HugoSites, error) { return nil, nil }) - h.init.gitInfo.Add(func() (any, error) { + h.init.gitInfo.Add(func(context.Context) (any, error) { err := h.loadGitInfo() if err != nil { return nil, fmt.Errorf("failed to load Git info: %w", err) diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index 5eee564aa..66abf4f16 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -268,7 +268,7 @@ func (h *HugoSites) assemble(bcfg *BuildCfg) error { } func (h *HugoSites) render(config *BuildCfg) error { - if _, err := h.init.layouts.Do(); err != nil { + if _, err := h.init.layouts.Do(context.Background()); err != nil { return err } diff --git a/hugolib/hugo_sites_build_errors_test.go b/hugolib/hugo_sites_build_errors_test.go index ffbfe1c17..f42b44461 100644 --- a/hugolib/hugo_sites_build_errors_test.go +++ b/hugolib/hugo_sites_build_errors_test.go @@ -396,7 +396,7 @@ line 4 } -func TestErrorNestedShortocde(t *testing.T) { +func TestErrorNestedShortcode(t *testing.T) { t.Parallel() files := ` diff --git a/hugolib/image_test.go b/hugolib/image_test.go index ac18b9423..db1707c22 100644 --- a/hugolib/image_test.go +++ b/hugolib/image_test.go @@ -14,162 +14,9 @@ package hugolib import ( - "io" - "os" - "path/filepath" - "runtime" - "strings" "testing" - - "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/htesting" - - qt "github.com/frankban/quicktest" - "github.com/gohugoio/hugo/hugofs" ) -// We have many tests for the different resize operations etc. in the resource package, -// this is an integration test. -func TestImageOps(t *testing.T) { - c := qt.New(t) - // Make this a real as possible. - workDir, clean, err := htesting.CreateTempDir(hugofs.Os, "image-resize") - c.Assert(err, qt.IsNil) - defer clean() - - newBuilder := func(timeout any) *sitesBuilder { - v := config.NewWithTestDefaults() - v.Set("workingDir", workDir) - v.Set("baseURL", "https://example.org") - v.Set("timeout", timeout) - - b := newTestSitesBuilder(t).WithWorkingDir(workDir) - b.Fs = hugofs.NewDefault(v) - b.WithViper(v) - b.WithContent("mybundle/index.md", ` ---- -title: "My bundle" ---- - -{{< imgproc >}} - -`) - - b.WithTemplatesAdded( - "shortcodes/imgproc.html", ` -{{ $img := resources.Get "images/sunset.jpg" }} -{{ $r := $img.Resize "129x239" }} -IMG SHORTCODE: {{ $r.RelPermalink }}/{{ $r.Width }} -`, - "index.html", ` -{{ $p := .Site.GetPage "mybundle" }} -{{ $img1 := resources.Get "images/sunset.jpg" }} -{{ $img2 := $p.Resources.GetMatch "sunset.jpg" }} -{{ $img3 := resources.GetMatch "images/*.jpg" }} -{{ $r := $img1.Resize "123x234" }} -{{ $r2 := $r.Resize "12x23" }} -{{ $b := $img2.Resize "345x678" }} -{{ $b2 := $b.Resize "34x67" }} -{{ $c := $img3.Resize "456x789" }} -{{ $fingerprinted := $img1.Resize "350x" | fingerprint }} - -{{ $images := slice $r $r2 $b $b2 $c $fingerprinted }} - -{{ range $i, $r := $images }} -{{ printf "Resized%d:" (add $i 1) }} {{ $r.Name }}|{{ $r.Width }}|{{ $r.Height }}|{{ $r.MediaType }}|{{ $r.RelPermalink }}| -{{ end }} - -{{ $blurryGrayscale1 := $r | images.Filter images.Grayscale (images.GaussianBlur 8) }} -BG1: {{ $blurryGrayscale1.RelPermalink }}/{{ $blurryGrayscale1.Width }} -{{ $blurryGrayscale2 := $r.Filter images.Grayscale (images.GaussianBlur 8) }} -BG2: {{ $blurryGrayscale2.RelPermalink }}/{{ $blurryGrayscale2.Width }} -{{ $blurryGrayscale2_2 := $r.Filter images.Grayscale (images.GaussianBlur 8) }} -BG2_2: {{ $blurryGrayscale2_2.RelPermalink }}/{{ $blurryGrayscale2_2.Width }} - -{{ $filters := slice images.Grayscale (images.GaussianBlur 9) }} -{{ $blurryGrayscale3 := $r | images.Filter $filters }} -BG3: {{ $blurryGrayscale3.RelPermalink }}/{{ $blurryGrayscale3.Width }} - -{{ $blurryGrayscale4 := $r.Filter $filters }} -BG4: {{ $blurryGrayscale4.RelPermalink }}/{{ $blurryGrayscale4.Width }} - -{{ $p.Content }} - -`) - - return b - } - - imageDir := filepath.Join(workDir, "assets", "images") - bundleDir := filepath.Join(workDir, "content", "mybundle") - - c.Assert(os.MkdirAll(imageDir, 0777), qt.IsNil) - c.Assert(os.MkdirAll(bundleDir, 0777), qt.IsNil) - src, err := os.Open("testdata/sunset.jpg") - c.Assert(err, qt.IsNil) - out, err := os.Create(filepath.Join(imageDir, "sunset.jpg")) - c.Assert(err, qt.IsNil) - _, err = io.Copy(out, src) - c.Assert(err, qt.IsNil) - out.Close() - - src.Seek(0, 0) - - out, err = os.Create(filepath.Join(bundleDir, "sunset.jpg")) - c.Assert(err, qt.IsNil) - _, err = io.Copy(out, src) - c.Assert(err, qt.IsNil) - out.Close() - src.Close() - - // First build it with a very short timeout to trigger errors. - b := newBuilder("10ns") - - imgExpect := ` -Resized1: images/sunset.jpg|123|234|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_123x234_resize_q75_box.jpg| -Resized2: images/sunset.jpg|12|23|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ada4bb1a57f77a63306e3bd67286248e.jpg| -Resized3: sunset.jpg|345|678|image/jpeg|/mybundle/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_345x678_resize_q75_box.jpg| -Resized4: sunset.jpg|34|67|image/jpeg|/mybundle/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_44d8c928664d7c5a67377c6ec58425ce.jpg| -Resized5: images/sunset.jpg|456|789|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_456x789_resize_q75_box.jpg| -Resized6: images/sunset.jpg|350|219|image/jpeg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_350x0_resize_q75_box.a86fe88d894e5db613f6aa8a80538fefc25b20fa24ba0d782c057adcef616f56.jpg| -BG1: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_2ae8bb993431ec1aec40fe59927b46b4.jpg/123 -BG2: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_2ae8bb993431ec1aec40fe59927b46b4.jpg/123 -BG3: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg/123 -BG4: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg/123 -IMG SHORTCODE: /images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_129x239_resize_q75_box.jpg/129 -` - - assertImages := func() { - b.Helper() - b.AssertFileContent("public/index.html", imgExpect) - b.AssertImage(350, 219, "public/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_350x0_resize_q75_box.a86fe88d894e5db613f6aa8a80538fefc25b20fa24ba0d782c057adcef616f56.jpg") - b.AssertImage(129, 239, "public/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_129x239_resize_q75_box.jpg") - } - - err = b.BuildE(BuildCfg{}) - if runtime.GOOS != "windows" && !strings.Contains(runtime.GOARCH, "arm") && !htesting.IsGitHubAction() { - // TODO(bep) - c.Assert(err, qt.Not(qt.IsNil)) - } - - b = newBuilder(29000) - b.Build(BuildCfg{}) - - assertImages() - - // Truncate one image. - imgInCache := filepath.Join(workDir, "resources/_gen/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_ed7740a90b82802261c2fbdb98bc8082.jpg") - f, err := os.Create(imgInCache) - c.Assert(err, qt.IsNil) - f.Close() - - // Build it again to make sure we read images from file cache. - b = newBuilder("30s") - b.Build(BuildCfg{}) - - assertImages() -} - func TestImageResizeMultilingual(t *testing.T) { b := newTestSitesBuilder(t).WithConfigFile("toml", ` baseURL="https://example.org" diff --git a/hugolib/language_content_dir_test.go b/hugolib/language_content_dir_test.go index 57cdab67b..23809f4df 100644 --- a/hugolib/language_content_dir_test.go +++ b/hugolib/language_content_dir_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "os" "path/filepath" @@ -245,7 +246,7 @@ Content. c.Assert(svP2.Language().Lang, qt.Equals, "sv") c.Assert(nnP2.Language().Lang, qt.Equals, "nn") - content, _ := nnP2.Content() + content, _ := nnP2.Content(context.Background()) contentStr := cast.ToString(content) c.Assert(contentStr, qt.Contains, "SVP3-REF: https://example.org/sv/sect/p-sv-3/") c.Assert(contentStr, qt.Contains, "SVP3-RELREF: /sv/sect/p-sv-3/") diff --git a/hugolib/page.go b/hugolib/page.go index 97f1ed351..40972d7c5 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -15,6 +15,7 @@ package hugolib import ( "bytes" + "context" "fmt" "path" "path/filepath" @@ -24,8 +25,10 @@ import ( "go.uber.org/atomic" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/related" "github.com/gohugoio/hugo/markup/converter" + "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/gohugoio/hugo/tpl" @@ -148,6 +151,43 @@ func (p *pageState) GetIdentity() identity.Identity { return identity.NewPathIdentity(files.ComponentFolderContent, filepath.FromSlash(p.Pathc())) } +func (p *pageState) Fragments(ctx context.Context) *tableofcontents.Fragments { + p.s.initInit(ctx, p.cp.initToC, p) + if p.pageOutput.cp.tableOfContents == nil { + return tableofcontents.Empty + } + return p.pageOutput.cp.tableOfContents +} + +func (p *pageState) HeadingsFiltered(context.Context) tableofcontents.Headings { + return nil +} + +type pageHeadingsFiltered struct { + *pageState + headings tableofcontents.Headings +} + +func (p *pageHeadingsFiltered) HeadingsFiltered(context.Context) tableofcontents.Headings { + return p.headings +} + +func (p *pageHeadingsFiltered) page() page.Page { + return p.pageState +} + +// For internal use by the related content feature. +func (p *pageState) ApplyFilterToHeadings(ctx context.Context, fn func(*tableofcontents.Heading) bool) related.Document { + if p.pageOutput.cp.tableOfContents == nil { + return p + } + headings := p.pageOutput.cp.tableOfContents.Headings.FilterBy(fn) + return &pageHeadingsFiltered{ + pageState: p, + headings: headings, + } +} + func (p *pageState) GitInfo() source.GitInfo { return p.gitInfo } @@ -351,7 +391,7 @@ func (p *pageState) String() string { // IsTranslated returns whether this content file is translated to // other language(s). func (p *pageState) IsTranslated() bool { - p.s.h.init.translations.Do() + p.s.h.init.translations.Do(context.Background()) return len(p.translations) > 0 } @@ -375,13 +415,13 @@ func (p *pageState) TranslationKey() string { // AllTranslations returns all translations, including the current Page. func (p *pageState) AllTranslations() page.Pages { - p.s.h.init.translations.Do() + p.s.h.init.translations.Do(context.Background()) return p.allTranslations } // Translations returns the translations excluding the current Page. func (p *pageState) Translations() page.Pages { - p.s.h.init.translations.Do() + p.s.h.init.translations.Do(context.Background()) return p.translations } @@ -461,7 +501,7 @@ func (p *pageState) initOutputFormat(isRenderingSite bool, idx int) error { // Must be run after the site section tree etc. is built and ready. func (p *pageState) initPage() error { - if _, err := p.init.Do(); err != nil { + if _, err := p.init.Do(context.Background()); err != nil { return err } return nil diff --git a/hugolib/page__content.go b/hugolib/page__content.go index a721d1fce..89c38bd84 100644 --- a/hugolib/page__content.go +++ b/hugolib/page__content.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "github.com/gohugoio/hugo/output" @@ -37,9 +38,9 @@ type pageContent struct { } // returns the content to be processed by Goldmark or similar. -func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMap, renderedShortcodes map[string]string) []byte { +func (p pageContent) contentToRender(ctx context.Context, parsed pageparser.Result, pm *pageContentMap, renderedShortcodes map[string]shortcodeRenderer) ([]byte, bool, error) { source := parsed.Input() - + var hasVariants bool c := make([]byte, 0, len(source)+(len(source)/10)) for _, it := range pm.items { @@ -57,7 +58,12 @@ func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMa panic(fmt.Sprintf("rendered shortcode %q not found", v.placeholder)) } - c = append(c, []byte(renderedShortcode)...) + b, more, err := renderedShortcode.renderShortcode(ctx) + if err != nil { + return nil, false, fmt.Errorf("failed to render shortcode: %w", err) + } + hasVariants = hasVariants || more + c = append(c, []byte(b)...) } else { // Insert the placeholder so we can insert the content after @@ -69,7 +75,7 @@ func (p pageContent) contentToRender(parsed pageparser.Result, pm *pageContentMa } } - return c + return c, hasVariants, nil } func (p pageContent) selfLayoutForOutput(f output.Format) string { diff --git a/hugolib/page__menus.go b/hugolib/page__menus.go index 49d392c2f..5bed2bc03 100644 --- a/hugolib/page__menus.go +++ b/hugolib/page__menus.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "sync" "github.com/gohugoio/hugo/navigation" @@ -29,13 +30,13 @@ type pageMenus struct { } func (p *pageMenus) HasMenuCurrent(menuID string, me *navigation.MenuEntry) bool { - p.p.s.init.menus.Do() + p.p.s.init.menus.Do(context.Background()) p.init() return p.q.HasMenuCurrent(menuID, me) } func (p *pageMenus) IsMenuCurrent(menuID string, inme *navigation.MenuEntry) bool { - p.p.s.init.menus.Do() + p.p.s.init.menus.Do(context.Background()) p.init() return p.q.IsMenuCurrent(menuID, inme) } @@ -43,7 +44,7 @@ func (p *pageMenus) IsMenuCurrent(menuID string, inme *navigation.MenuEntry) boo func (p *pageMenus) Menus() navigation.PageMenus { // There is a reverse dependency here. initMenus will, once, build the // site menus and update any relevant page. - p.p.s.init.menus.Do() + p.p.s.init.menus.Do(context.Background()) return p.menus() } diff --git a/hugolib/page__new.go b/hugolib/page__new.go index e52b9476b..3787cd2bd 100644 --- a/hugolib/page__new.go +++ b/hugolib/page__new.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "html/template" "strings" @@ -121,7 +122,7 @@ func newPageFromMeta( return nil, err } - ps.init.Add(func() (any, error) { + ps.init.Add(func(context.Context) (any, error) { pp, err := newPagePaths(metaProvider.s, ps, metaProvider) if err != nil { return nil, err diff --git a/hugolib/page__per_output.go b/hugolib/page__per_output.go index 97e9cc465..827a6b792 100644 --- a/hugolib/page__per_output.go +++ b/hugolib/page__per_output.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "html/template" - "runtime/debug" "strings" "sync" "unicode/utf8" @@ -34,6 +33,7 @@ import ( "github.com/gohugoio/hugo/markup/converter/hooks" "github.com/gohugoio/hugo/markup/highlight/chromalexers" + "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/gohugoio/hugo/markup/converter" @@ -87,43 +87,35 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err renderHooks: &renderHooks{}, } - initContent := func() (err error) { - p.s.h.IncrContentRender() - + initToC := func(ctx context.Context) (err error) { if p.cmap == nil { // Nothing to do. return nil } - defer func() { - // See https://github.com/gohugoio/hugo/issues/6210 - if r := recover(); r != nil { - err = fmt.Errorf("%s", r) - p.s.Log.Errorf("[BUG] Got panic:\n%s\n%s", r, string(debug.Stack())) - } - }() if err := po.cp.initRenderHooks(); err != nil { return err } - var hasShortcodeVariants bool - f := po.f - cp.contentPlaceholders, hasShortcodeVariants, err = p.shortcodeState.renderShortcodesForPage(p, f) + cp.contentPlaceholders, err = p.shortcodeState.prepareShortcodesForPage(ctx, p, f) if err != nil { return err } - if hasShortcodeVariants { + var hasVariants bool + cp.workContent, hasVariants, err = p.contentToRender(ctx, p.source.parsed, p.cmap, cp.contentPlaceholders) + if err != nil { + return err + } + if hasVariants { p.pageOutputTemplateVariationsState.Store(2) } - cp.workContent = p.contentToRender(p.source.parsed, p.cmap, cp.contentPlaceholders) - isHTML := cp.p.m.markup == "html" if !isHTML { - r, err := po.contentRenderer.RenderContent(cp.workContent, true) + r, err := po.contentRenderer.RenderContent(ctx, cp.workContent, true) if err != nil { return err } @@ -132,8 +124,9 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err if tocProvider, ok := r.(converter.TableOfContentsProvider); ok { cfg := p.s.ContentSpec.Converters.GetMarkupConfig() - cp.tableOfContents = template.HTML( - tocProvider.TableOfContents().ToHTML( + cp.tableOfContents = tocProvider.TableOfContents() + cp.tableOfContentsHTML = template.HTML( + cp.tableOfContents.ToHTML( cfg.TableOfContents.StartLevel, cfg.TableOfContents.EndLevel, cfg.TableOfContents.Ordered, @@ -141,26 +134,60 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err ) } else { tmpContent, tmpTableOfContents := helpers.ExtractTOC(cp.workContent) - cp.tableOfContents = helpers.BytesToHTML(tmpTableOfContents) + cp.tableOfContentsHTML = helpers.BytesToHTML(tmpTableOfContents) + cp.tableOfContents = tableofcontents.Empty cp.workContent = tmpContent } } - if cp.placeholdersEnabled { - // ToC was accessed via .Page.TableOfContents in the shortcode, - // at a time when the ToC wasn't ready. - cp.contentPlaceholders[tocShortcodePlaceholder] = string(cp.tableOfContents) + return nil + + } + + initContent := func(ctx context.Context) (err error) { + + p.s.h.IncrContentRender() + + if p.cmap == nil { + // Nothing to do. + return nil } if p.cmap.hasNonMarkdownShortcode || cp.placeholdersEnabled { // There are one or more replacement tokens to be replaced. - cp.workContent, err = replaceShortcodeTokens(cp.workContent, cp.contentPlaceholders) + var hasShortcodeVariants bool + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + if token == tocShortcodePlaceholder { + // The Page's TableOfContents was accessed in a shortcode. + if cp.tableOfContentsHTML == "" { + cp.p.s.initInit(ctx, cp.initToC, cp.p) + } + return []byte(cp.tableOfContentsHTML), nil + } + renderer, found := cp.contentPlaceholders[token] + if found { + repl, more, err := renderer.renderShortcode(ctx) + if err != nil { + return nil, err + } + hasShortcodeVariants = hasShortcodeVariants || more + return repl, nil + } + // This should never happen. + return nil, fmt.Errorf("unknown shortcode token %q", token) + } + + cp.workContent, err = expandShortcodeTokens(ctx, cp.workContent, tokenHandler) if err != nil { return err } + if hasShortcodeVariants { + p.pageOutputTemplateVariationsState.Store(2) + } } if cp.p.source.hasSummaryDivider { + isHTML := cp.p.m.markup == "html" if isHTML { src := p.source.parsed.Input() @@ -183,7 +210,7 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err } } } else if cp.p.m.summary != "" { - b, err := po.contentRenderer.RenderContent([]byte(cp.p.m.summary), false) + b, err := po.contentRenderer.RenderContent(ctx, []byte(cp.p.m.summary), false) if err != nil { return err } @@ -196,12 +223,16 @@ func newPageContentOutput(p *pageState, po *pageOutput) (*pageContentOutput, err return nil } - // There may be recursive loops in shortcodes and render hooks. - cp.initMain = parent.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (any, error) { - return nil, initContent() + cp.initToC = parent.Branch(func(ctx context.Context) (any, error) { + return nil, initToC(ctx) }) - cp.initPlain = cp.initMain.Branch(func() (any, error) { + // There may be recursive loops in shortcodes and render hooks. + cp.initMain = cp.initToC.BranchWithTimeout(p.s.siteCfg.timeout, func(ctx context.Context) (any, error) { + return nil, initContent(ctx) + }) + + cp.initPlain = cp.initMain.Branch(func(context.Context) (any, error) { cp.plain = tpl.StripHTML(string(cp.content)) cp.plainWords = strings.Fields(cp.plain) cp.setWordCounts(p.m.isCJKLanguage) @@ -228,6 +259,7 @@ type pageContentOutput struct { p *pageState // Lazy load dependencies + initToC *lazy.Init initMain *lazy.Init initPlain *lazy.Init @@ -243,12 +275,13 @@ type pageContentOutput struct { // Temporary storage of placeholders mapped to their content. // These are shortcodes etc. Some of these will need to be replaced // after any markup is rendered, so they share a common prefix. - contentPlaceholders map[string]string + contentPlaceholders map[string]shortcodeRenderer // Content sections - content template.HTML - summary template.HTML - tableOfContents template.HTML + content template.HTML + summary template.HTML + tableOfContents *tableofcontents.Fragments + tableOfContentsHTML template.HTML truncated bool @@ -263,76 +296,76 @@ func (p *pageContentOutput) trackDependency(id identity.Provider) { if p.dependencyTracker != nil { p.dependencyTracker.Add(id) } + } func (p *pageContentOutput) Reset() { if p.dependencyTracker != nil { p.dependencyTracker.Reset() } + p.initToC.Reset() p.initMain.Reset() p.initPlain.Reset() p.renderHooks = &renderHooks{} } -func (p *pageContentOutput) Content() (any, error) { - if p.p.s.initInit(p.initMain, p.p) { - return p.content, nil - } - return nil, nil +func (p *pageContentOutput) Content(ctx context.Context) (any, error) { + p.p.s.initInit(ctx, p.initMain, p.p) + return p.content, nil } -func (p *pageContentOutput) FuzzyWordCount() int { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) FuzzyWordCount(ctx context.Context) int { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.fuzzyWordCount } -func (p *pageContentOutput) Len() int { - p.p.s.initInit(p.initMain, p.p) +func (p *pageContentOutput) Len(ctx context.Context) int { + p.p.s.initInit(ctx, p.initMain, p.p) return len(p.content) } -func (p *pageContentOutput) Plain() string { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) Plain(ctx context.Context) string { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.plain } -func (p *pageContentOutput) PlainWords() []string { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) PlainWords(ctx context.Context) []string { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.plainWords } -func (p *pageContentOutput) ReadingTime() int { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) ReadingTime(ctx context.Context) int { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.readingTime } -func (p *pageContentOutput) Summary() template.HTML { - p.p.s.initInit(p.initMain, p.p) +func (p *pageContentOutput) Summary(ctx context.Context) template.HTML { + p.p.s.initInit(ctx, p.initMain, p.p) if !p.p.source.hasSummaryDivider { - p.p.s.initInit(p.initPlain, p.p) + p.p.s.initInit(ctx, p.initPlain, p.p) } return p.summary } -func (p *pageContentOutput) TableOfContents() template.HTML { - p.p.s.initInit(p.initMain, p.p) - return p.tableOfContents +func (p *pageContentOutput) TableOfContents(ctx context.Context) template.HTML { + p.p.s.initInit(ctx, p.initMain, p.p) + return p.tableOfContentsHTML } -func (p *pageContentOutput) Truncated() bool { +func (p *pageContentOutput) Truncated(ctx context.Context) bool { if p.p.truncated { return true } - p.p.s.initInit(p.initPlain, p.p) + p.p.s.initInit(ctx, p.initPlain, p.p) return p.truncated } -func (p *pageContentOutput) WordCount() int { - p.p.s.initInit(p.initPlain, p.p) +func (p *pageContentOutput) WordCount(ctx context.Context) int { + p.p.s.initInit(ctx, p.initPlain, p.p) return p.wordCount } -func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) { +func (p *pageContentOutput) RenderString(ctx context.Context, args ...any) (template.HTML, error) { if len(args) < 1 || len(args) > 2 { return "", errors.New("want 1 or 2 arguments") } @@ -405,42 +438,62 @@ func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) { return "", err } - placeholders, hasShortcodeVariants, err := s.renderShortcodesForPage(p.p, p.f) + placeholders, err := s.prepareShortcodesForPage(ctx, p.p, p.f) if err != nil { return "", err } - if hasShortcodeVariants { + contentToRender, hasVariants, err := p.p.contentToRender(ctx, parsed, pm, placeholders) + if err != nil { + return "", err + } + if hasVariants { p.p.pageOutputTemplateVariationsState.Store(2) } - - b, err := p.renderContentWithConverter(conv, p.p.contentToRender(parsed, pm, placeholders), false) + b, err := p.renderContentWithConverter(ctx, conv, contentToRender, false) if err != nil { return "", p.p.wrapError(err) } rendered = b.Bytes() - if p.placeholdersEnabled { - // ToC was accessed via .Page.TableOfContents in the shortcode, - // at a time when the ToC wasn't ready. - if _, err := p.p.Content(); err != nil { - return "", err - } - placeholders[tocShortcodePlaceholder] = string(p.tableOfContents) - } - if pm.hasNonMarkdownShortcode || p.placeholdersEnabled { - rendered, err = replaceShortcodeTokens(rendered, placeholders) + var hasShortcodeVariants bool + + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + if token == tocShortcodePlaceholder { + // The Page's TableOfContents was accessed in a shortcode. + if p.tableOfContentsHTML == "" { + p.p.s.initInit(ctx, p.initToC, p.p) + } + return []byte(p.tableOfContentsHTML), nil + } + renderer, found := placeholders[token] + if found { + repl, more, err := renderer.renderShortcode(ctx) + if err != nil { + return nil, err + } + hasShortcodeVariants = hasShortcodeVariants || more + return repl, nil + } + // This should not happen. + return nil, fmt.Errorf("unknown shortcode token %q", token) + } + + rendered, err = expandShortcodeTokens(ctx, rendered, tokenHandler) if err != nil { return "", err } + if hasShortcodeVariants { + p.p.pageOutputTemplateVariationsState.Store(2) + } } // We need a consolidated view in $page.HasShortcode p.p.shortcodeState.transferNames(s) } else { - c, err := p.renderContentWithConverter(conv, []byte(contentToRender), false) + c, err := p.renderContentWithConverter(ctx, conv, []byte(contentToRender), false) if err != nil { return "", p.p.wrapError(err) } @@ -457,12 +510,12 @@ func (p *pageContentOutput) RenderString(args ...any) (template.HTML, error) { return template.HTML(string(rendered)), nil } -func (p *pageContentOutput) RenderWithTemplateInfo(info tpl.Info, layout ...string) (template.HTML, error) { +func (p *pageContentOutput) RenderWithTemplateInfo(ctx context.Context, info tpl.Info, layout ...string) (template.HTML, error) { p.p.addDependency(info) - return p.Render(layout...) + return p.Render(ctx, layout...) } -func (p *pageContentOutput) Render(layout ...string) (template.HTML, error) { +func (p *pageContentOutput) Render(ctx context.Context, layout ...string) (template.HTML, error) { templ, found, err := p.p.resolveTemplate(layout...) if err != nil { return "", p.p.wrapError(err) @@ -475,7 +528,7 @@ func (p *pageContentOutput) Render(layout ...string) (template.HTML, error) { p.p.addDependency(templ.(tpl.Info)) // Make sure to send the *pageState and not the *pageContentOutput to the template. - res, err := executeToString(p.p.s.Tmpl(), templ, p.p) + res, err := executeToString(ctx, p.p.s.Tmpl(), templ, p.p) if err != nil { return "", p.p.wrapError(fmt.Errorf("failed to execute template %s: %w", templ.Name(), err)) } @@ -629,15 +682,15 @@ func (p *pageContentOutput) setAutoSummary() error { return nil } -func (cp *pageContentOutput) RenderContent(content []byte, renderTOC bool) (converter.Result, error) { +func (cp *pageContentOutput) RenderContent(ctx context.Context, content []byte, renderTOC bool) (converter.Result, error) { if err := cp.initRenderHooks(); err != nil { return nil, err } c := cp.p.getContentConverter() - return cp.renderContentWithConverter(c, content, renderTOC) + return cp.renderContentWithConverter(ctx, c, content, renderTOC) } -func (cp *pageContentOutput) renderContentWithConverter(c converter.Converter, content []byte, renderTOC bool) (converter.Result, error) { +func (cp *pageContentOutput) renderContentWithConverter(ctx context.Context, c converter.Converter, content []byte, renderTOC bool) (converter.Result, error) { r, err := c.Convert( converter.RenderContext{ Src: content, @@ -711,10 +764,10 @@ func (t targetPathsHolder) targetPaths() page.TargetPaths { return t.paths } -func executeToString(h tpl.TemplateHandler, templ tpl.Template, data any) (string, error) { +func executeToString(ctx context.Context, h tpl.TemplateHandler, templ tpl.Template, data any) (string, error) { b := bp.GetBuffer() defer bp.PutBuffer(b) - if err := h.Execute(templ, b, data); err != nil { + if err := h.ExecuteWithContext(ctx, templ, b, data); err != nil { return "", err } return b.String(), nil diff --git a/hugolib/page__position.go b/hugolib/page__position.go index a087872cc..d977a7052 100644 --- a/hugolib/page__position.go +++ b/hugolib/page__position.go @@ -14,6 +14,8 @@ package hugolib import ( + "context" + "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/resources/page" ) @@ -33,12 +35,12 @@ type nextPrev struct { } func (n *nextPrev) next() page.Page { - n.init.Do() + n.init.Do(context.Background()) return n.nextPage } func (n *nextPrev) prev() page.Page { - n.init.Do() + n.init.Do(context.Background()) return n.prevPage } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 939d06d41..49617f17e 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "html/template" "os" @@ -311,13 +312,13 @@ func normalizeContent(c string) string { func checkPageTOC(t *testing.T, page page.Page, toc string) { t.Helper() - if page.TableOfContents() != template.HTML(toc) { - t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(), toc) + if page.TableOfContents(context.Background()) != template.HTML(toc) { + t.Fatalf("Page TableOfContents is:\n%q.\nExpected %q", page.TableOfContents(context.Background()), toc) } } func checkPageSummary(t *testing.T, page page.Page, summary string, msg ...any) { - a := normalizeContent(string(page.Summary())) + a := normalizeContent(string(page.Summary(context.Background()))) b := normalizeContent(summary) if a != b { t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg) @@ -443,9 +444,9 @@ func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { p := s.RegularPages()[0] - if p.Summary() != template.HTML( + if p.Summary(context.Background()) != template.HTML( "

The best static site generator.1

") { - t.Fatalf("Got summary:\n%q", p.Summary()) + t.Fatalf("Got summary:\n%q", p.Summary(context.Background())) } cnt := content(p) @@ -719,7 +720,7 @@ func TestSummaryWithHTMLTagsOnNextLine(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { c := qt.New(t) p := pages[0] - s := string(p.Summary()) + s := string(p.Summary(context.Background())) c.Assert(s, qt.Contains, "Happy new year everyone!") c.Assert(s, qt.Not(qt.Contains), "User interface") } @@ -1122,8 +1123,8 @@ func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { t.Parallel() assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 8 { - t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 8, p.WordCount()) + if p.WordCount(context.Background()) != 8 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 8, p.WordCount(context.Background())) } } @@ -1136,8 +1137,8 @@ func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 15 { - t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 15, p.WordCount()) + if p.WordCount(context.Background()) != 15 { + t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 15, p.WordCount(context.Background())) } } testAllMarkdownEnginesForPages(t, assertFunc, settings, simplePageWithAllCJKRunes) @@ -1149,13 +1150,13 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 74 { - t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 74, p.WordCount()) + if p.WordCount(context.Background()) != 74 { + t.Fatalf("[%s] incorrect word count, expected %v, got %v", ext, 74, p.WordCount(context.Background())) } - if p.Summary() != simplePageWithMainEnglishWithCJKRunesSummary { - t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(), - simplePageWithMainEnglishWithCJKRunesSummary, p.Summary()) + if p.Summary(context.Background()) != simplePageWithMainEnglishWithCJKRunesSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(context.Background()), + simplePageWithMainEnglishWithCJKRunesSummary, p.Summary(context.Background())) } } @@ -1170,13 +1171,13 @@ func TestWordCountWithIsCJKLanguageFalse(t *testing.T) { assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 75 { - t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.Plain(), 74, p.WordCount()) + if p.WordCount(context.Background()) != 75 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.Plain(context.Background()), 74, p.WordCount(context.Background())) } - if p.Summary() != simplePageWithIsCJKLanguageFalseSummary { - t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(), - simplePageWithIsCJKLanguageFalseSummary, p.Summary()) + if p.Summary(context.Background()) != simplePageWithIsCJKLanguageFalseSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.Plain(context.Background()), + simplePageWithIsCJKLanguageFalseSummary, p.Summary(context.Background())) } } @@ -1187,16 +1188,16 @@ func TestWordCount(t *testing.T) { t.Parallel() assertFunc := func(t *testing.T, ext string, pages page.Pages) { p := pages[0] - if p.WordCount() != 483 { - t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount()) + if p.WordCount(context.Background()) != 483 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount(context.Background())) } - if p.FuzzyWordCount() != 500 { - t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.FuzzyWordCount()) + if p.FuzzyWordCount(context.Background()) != 500 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.FuzzyWordCount(context.Background())) } - if p.ReadingTime() != 3 { - t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime()) + if p.ReadingTime(context.Background()) != 3 { + t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime(context.Background())) } } diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 2951a1436..a82caff43 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -15,6 +15,7 @@ package hugolib import ( "bytes" + "context" "fmt" "html/template" "path" @@ -302,13 +303,44 @@ const ( innerCleanupExpand = "$1" ) -func renderShortcode( +func prepareShortcode( + ctx context.Context, level int, s *Site, tplVariants tpl.TemplateVariants, sc *shortcode, parent *ShortcodeWithPage, - p *pageState) (string, bool, error) { + p *pageState) (shortcodeRenderer, error) { + + toParseErr := func(err error) error { + return p.parseError(fmt.Errorf("failed to render shortcode %q: %w", sc.name, err), p.source.parsed.Input(), sc.pos) + } + + // Allow the caller to delay the rendering of the shortcode if needed. + var fn shortcodeRenderFunc = func(ctx context.Context) ([]byte, bool, error) { + r, err := doRenderShortcode(ctx, level, s, tplVariants, sc, parent, p) + if err != nil { + return nil, false, toParseErr(err) + } + b, hasVariants, err := r.renderShortcode(ctx) + if err != nil { + return nil, false, toParseErr(err) + } + return b, hasVariants, nil + } + + return fn, nil + +} + +func doRenderShortcode( + ctx context.Context, + level int, + s *Site, + tplVariants tpl.TemplateVariants, + sc *shortcode, + parent *ShortcodeWithPage, + p *pageState) (shortcodeRenderer, error) { var tmpl tpl.Template // Tracks whether this shortcode or any of its children has template variations @@ -319,7 +351,7 @@ func renderShortcode( if sc.isInline { if !p.s.ExecHelper.Sec().EnableInlineShortcodes { - return "", false, nil + return zeroShortcode, nil } templName := path.Join("_inline_shortcode", p.File().Path(), sc.name) if sc.isClosing { @@ -332,7 +364,7 @@ func renderShortcode( pos := fe.Position() pos.LineNumber += p.posOffset(sc.pos).LineNumber fe = fe.UpdatePosition(pos) - return "", false, p.wrapError(fe) + return zeroShortcode, p.wrapError(fe) } } else { @@ -340,7 +372,7 @@ func renderShortcode( var found bool tmpl, found = s.TextTmpl().Lookup(templName) if !found { - return "", false, fmt.Errorf("no earlier definition of shortcode %q found", sc.name) + return zeroShortcode, fmt.Errorf("no earlier definition of shortcode %q found", sc.name) } } } else { @@ -348,7 +380,7 @@ func renderShortcode( tmpl, found, more = s.Tmpl().LookupVariant(sc.name, tplVariants) if !found { s.Log.Errorf("Unable to locate template for shortcode %q in page %q", sc.name, p.File().Path()) - return "", false, nil + return zeroShortcode, nil } hasVariants = hasVariants || more } @@ -365,16 +397,20 @@ func renderShortcode( case string: inner += innerData case *shortcode: - s, more, err := renderShortcode(level+1, s, tplVariants, innerData, data, p) + s, err := prepareShortcode(ctx, level+1, s, tplVariants, innerData, data, p) if err != nil { - return "", false, err + return zeroShortcode, err } + ss, more, err := s.renderShortcodeString(ctx) hasVariants = hasVariants || more - inner += s + if err != nil { + return zeroShortcode, err + } + inner += ss default: s.Log.Errorf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ", sc.name, p.File().Path(), reflect.TypeOf(innerData)) - return "", false, nil + return zeroShortcode, nil } } @@ -382,9 +418,9 @@ func renderShortcode( // shortcode. if sc.doMarkup && (level > 0 || sc.configVersion() == 1) { var err error - b, err := p.pageOutput.contentRenderer.RenderContent([]byte(inner), false) + b, err := p.pageOutput.contentRenderer.RenderContent(ctx, []byte(inner), false) if err != nil { - return "", false, err + return zeroShortcode, err } newInner := b.Bytes() @@ -418,14 +454,14 @@ func renderShortcode( } - result, err := renderShortcodeWithPage(s.Tmpl(), tmpl, data) + result, err := renderShortcodeWithPage(ctx, s.Tmpl(), tmpl, data) if err != nil && sc.isInline { fe := herrors.NewFileErrorFromName(err, p.File().Filename()) pos := fe.Position() pos.LineNumber += p.posOffset(sc.pos).LineNumber fe = fe.UpdatePosition(pos) - return "", false, fe + return zeroShortcode, fe } if len(sc.inner) == 0 && len(sc.indentation) > 0 { @@ -444,7 +480,7 @@ func renderShortcode( bp.PutBuffer(b) } - return result, hasVariants, err + return prerenderedShortcode{s: result, hasVariants: hasVariants}, err } func (s *shortcodeHandler) hasShortcodes() bool { @@ -473,28 +509,24 @@ func (s *shortcodeHandler) hasName(name string) bool { return ok } -func (s *shortcodeHandler) renderShortcodesForPage(p *pageState, f output.Format) (map[string]string, bool, error) { - rendered := make(map[string]string) +func (s *shortcodeHandler) prepareShortcodesForPage(ctx context.Context, p *pageState, f output.Format) (map[string]shortcodeRenderer, error) { + rendered := make(map[string]shortcodeRenderer) tplVariants := tpl.TemplateVariants{ Language: p.Language().Lang, OutputFormat: f, } - var hasVariants bool - for _, v := range s.shortcodes { - s, more, err := renderShortcode(0, s.s, tplVariants, v, nil, p) + s, err := prepareShortcode(ctx, 0, s.s, tplVariants, v, nil, p) if err != nil { - err = p.parseError(fmt.Errorf("failed to render shortcode %q: %w", v.name, err), p.source.parsed.Input(), v.pos) - return nil, false, err + return nil, err } - hasVariants = hasVariants || more rendered[v.placeholder] = s } - return rendered, hasVariants, nil + return rendered, nil } func (s *shortcodeHandler) parseError(err error, input []byte, pos int) error { @@ -668,11 +700,11 @@ Loop: // Replace prefixed shortcode tokens with the real content. // Note: This function will rewrite the input slice. -func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]byte, error) { - if len(replacements) == 0 { - return source, nil - } - +func expandShortcodeTokens( + ctx context.Context, + source []byte, + tokenHandler func(ctx context.Context, token string) ([]byte, error), +) ([]byte, error) { start := 0 pre := []byte(shortcodePlaceholderPrefix) @@ -691,8 +723,11 @@ func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]by } end := j + postIdx + 4 - - newVal := []byte(replacements[string(source[j:end])]) + key := string(source[j:end]) + newVal, err := tokenHandler(ctx, key) + if err != nil { + return nil, err + } // Issue #1148: Check for wrapping p-tags

if j >= 3 && bytes.Equal(source[j-3:j], pStart) { @@ -712,11 +747,11 @@ func replaceShortcodeTokens(source []byte, replacements map[string]string) ([]by return source, nil } -func renderShortcodeWithPage(h tpl.TemplateHandler, tmpl tpl.Template, data *ShortcodeWithPage) (string, error) { +func renderShortcodeWithPage(ctx context.Context, h tpl.TemplateHandler, tmpl tpl.Template, data *ShortcodeWithPage) (string, error) { buffer := bp.GetBuffer() defer bp.PutBuffer(buffer) - err := h.Execute(tmpl, buffer, data) + err := h.ExecuteWithContext(ctx, tmpl, buffer, data) if err != nil { return "", fmt.Errorf("failed to process shortcode: %w", err) } diff --git a/hugolib/shortcode_page.go b/hugolib/shortcode_page.go index 5a56e434f..3bc061bc0 100644 --- a/hugolib/shortcode_page.go +++ b/hugolib/shortcode_page.go @@ -14,13 +14,48 @@ package hugolib import ( + "context" "html/template" "github.com/gohugoio/hugo/resources/page" ) +// A placeholder for the TableOfContents markup. This is what we pass to the Goldmark etc. renderers. var tocShortcodePlaceholder = createShortcodePlaceholder("TOC", 0) +// shortcodeRenderer is typically used to delay rendering of inner shortcodes +// marked with placeholders in the content. +type shortcodeRenderer interface { + renderShortcode(context.Context) ([]byte, bool, error) + renderShortcodeString(context.Context) (string, bool, error) +} + +type shortcodeRenderFunc func(context.Context) ([]byte, bool, error) + +func (f shortcodeRenderFunc) renderShortcode(ctx context.Context) ([]byte, bool, error) { + return f(ctx) +} + +func (f shortcodeRenderFunc) renderShortcodeString(ctx context.Context) (string, bool, error) { + b, has, err := f(ctx) + return string(b), has, err +} + +type prerenderedShortcode struct { + s string + hasVariants bool +} + +func (p prerenderedShortcode) renderShortcode(context.Context) ([]byte, bool, error) { + return []byte(p.s), p.hasVariants, nil +} + +func (p prerenderedShortcode) renderShortcodeString(context.Context) (string, bool, error) { + return p.s, p.hasVariants, nil +} + +var zeroShortcode = prerenderedShortcode{} + // This is sent to the shortcodes. They cannot access the content // they're a part of. It would cause an infinite regress. // @@ -50,7 +85,11 @@ func (p *pageForShortcode) page() page.Page { return p.PageWithoutContent.(page.Page) } -func (p *pageForShortcode) TableOfContents() template.HTML { +func (p *pageForShortcode) String() string { + return p.p.String() +} + +func (p *pageForShortcode) TableOfContents(context.Context) template.HTML { p.p.enablePlaceholders() return p.toc } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index b5f27d621..2f285d0da 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "path/filepath" "reflect" @@ -247,7 +248,7 @@ CSV: {{< myShort >}} func BenchmarkReplaceShortcodeTokens(b *testing.B) { type input struct { in []byte - replacements map[string]string + tokenHandler func(ctx context.Context, token string) ([]byte, error) expect []byte } @@ -263,22 +264,30 @@ func BenchmarkReplaceShortcodeTokens(b *testing.B) { {strings.Repeat("A ", 3000) + " HAHAHUGOSHORTCODE-1HBHB." + strings.Repeat("BC ", 1000) + " HAHAHUGOSHORTCODE-1HBHB.", map[string]string{"HAHAHUGOSHORTCODE-1HBHB": "Hello World"}, []byte(strings.Repeat("A ", 3000) + " Hello World." + strings.Repeat("BC ", 1000) + " Hello World.")}, } - in := make([]input, b.N*len(data)) cnt := 0 + in := make([]input, b.N*len(data)) for i := 0; i < b.N; i++ { for _, this := range data { - in[cnt] = input{[]byte(this.input), this.replacements, this.expect} + replacements := make(map[string]shortcodeRenderer) + for k, v := range this.replacements { + replacements[k] = prerenderedShortcode{s: v} + } + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + return []byte(this.replacements[token]), nil + } + in[cnt] = input{[]byte(this.input), tokenHandler, this.expect} cnt++ } } b.ResetTimer() cnt = 0 + ctx := context.Background() for i := 0; i < b.N; i++ { for j := range data { currIn := in[cnt] cnt++ - results, err := replaceShortcodeTokens(currIn.in, currIn.replacements) + results, err := expandShortcodeTokens(ctx, currIn.in, currIn.tokenHandler) if err != nil { b.Fatalf("[%d] failed: %s", i, err) continue @@ -383,7 +392,16 @@ func TestReplaceShortcodeTokens(t *testing.T) { }, } { - results, err := replaceShortcodeTokens([]byte(this.input), this.replacements) + replacements := make(map[string]shortcodeRenderer) + for k, v := range this.replacements { + replacements[k] = prerenderedShortcode{s: v} + } + tokenHandler := func(ctx context.Context, token string) ([]byte, error) { + return []byte(this.replacements[token]), nil + } + + ctx := context.Background() + results, err := expandShortcodeTokens(ctx, []byte(this.input), tokenHandler) if b, ok := this.expect.(bool); ok && !b { if err == nil { diff --git a/hugolib/site.go b/hugolib/site.go index 0ca7a81b4..e90fa41ff 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "fmt" "html/template" "io" @@ -173,7 +174,7 @@ type Site struct { } func (s *Site) Taxonomies() page.TaxonomyList { - s.init.taxonomies.Do() + s.init.taxonomies.Do(context.Background()) return s.taxonomies } @@ -214,8 +215,9 @@ func (init *siteInit) Reset() { init.taxonomies.Reset() } -func (s *Site) initInit(init *lazy.Init, pctx pageContext) bool { - _, err := init.Do() +func (s *Site) initInit(ctx context.Context, init *lazy.Init, pctx pageContext) bool { + _, err := init.Do(ctx) + if err != nil { s.h.FatalError(pctx.wrapError(err)) } @@ -227,7 +229,7 @@ func (s *Site) prepareInits() { var init lazy.Init - s.init.prevNext = init.Branch(func() (any, error) { + s.init.prevNext = init.Branch(func(context.Context) (any, error) { regularPages := s.RegularPages() for i, p := range regularPages { np, ok := p.(nextPrevProvider) @@ -254,7 +256,7 @@ func (s *Site) prepareInits() { return nil, nil }) - s.init.prevNextInSection = init.Branch(func() (any, error) { + s.init.prevNextInSection = init.Branch(func(context.Context) (any, error) { var sections page.Pages s.home.treeRef.m.collectSectionsRecursiveIncludingSelf(pageMapQuery{Prefix: s.home.treeRef.key}, func(n *contentNode) { sections = append(sections, n.p) @@ -311,12 +313,12 @@ func (s *Site) prepareInits() { return nil, nil }) - s.init.menus = init.Branch(func() (any, error) { + s.init.menus = init.Branch(func(context.Context) (any, error) { s.assembleMenus() return nil, nil }) - s.init.taxonomies = init.Branch(func() (any, error) { + s.init.taxonomies = init.Branch(func(context.Context) (any, error) { err := s.pageMap.assembleTaxonomies() return nil, err }) @@ -327,7 +329,7 @@ type siteRenderingContext struct { } func (s *Site) Menus() navigation.Menus { - s.init.menus.Do() + s.init.menus.Do(context.Background()) return s.menus } @@ -1821,7 +1823,9 @@ func (s *Site) renderForTemplate(name, outputFormat string, d any, w io.Writer, return nil } - if err = s.Tmpl().Execute(templ, w, d); err != nil { + ctx := context.Background() + + if err = s.Tmpl().ExecuteWithContext(ctx, templ, w, d); err != nil { return fmt.Errorf("render of %q failed: %w", name, err) } return diff --git a/hugolib/site_render.go b/hugolib/site_render.go index b572c443e..51d638dde 100644 --- a/hugolib/site_render.go +++ b/hugolib/site_render.go @@ -19,9 +19,8 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/tpl" - "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/tpl" "errors" diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 8dac8fc92..a2ee56994 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -14,6 +14,7 @@ package hugolib import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -630,7 +631,7 @@ func TestOrderedPages(t *testing.T) { t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "Three", rbypubdate[0].Title()) } - bylength := s.RegularPages().ByLength() + bylength := s.RegularPages().ByLength(context.Background()) if bylength[0].Title() != "One" { t.Errorf("Pages in unexpected order. First should be '%s', got '%s'", "One", bylength[0].Title()) } @@ -662,7 +663,7 @@ func TestGroupedPages(t *testing.T) { writeSourcesToSource(t, "content", fs, groupedSources...) s := buildSingleSite(t, deps.DepsCfg{Fs: fs, Cfg: cfg}, BuildCfg{}) - rbysection, err := s.RegularPages().GroupBy("Section", "desc") + rbysection, err := s.RegularPages().GroupBy(context.Background(), "Section", "desc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } @@ -683,7 +684,7 @@ func TestGroupedPages(t *testing.T) { t.Errorf("PageGroup has unexpected number of pages. Third group should have '%d' pages, got '%d' pages", 2, len(rbysection[2].Pages)) } - bytype, err := s.RegularPages().GroupBy("Type", "asc") + bytype, err := s.RegularPages().GroupBy(context.Background(), "Type", "asc") if err != nil { t.Fatalf("Unable to make PageGroup array: %s", err) } diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index ca74e9340..89255c695 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -2,6 +2,7 @@ package hugolib import ( "bytes" + "context" "fmt" "image/jpeg" "io" @@ -1005,7 +1006,7 @@ func getPage(in page.Page, ref string) page.Page { } func content(c resource.ContentProvider) string { - cc, err := c.Content() + cc, err := c.Content(context.Background()) if err != nil { panic(err) } diff --git a/lazy/init.go b/lazy/init.go index b998d0305..4de2a83f7 100644 --- a/lazy/init.go +++ b/lazy/init.go @@ -29,7 +29,7 @@ func New() *Init { // Init holds a graph of lazily initialized dependencies. type Init struct { - // Used in tests + // Used mainly for testing. initCount uint64 mu sync.Mutex @@ -40,11 +40,11 @@ type Init struct { init onceMore out any err error - f func() (any, error) + f func(context.Context) (any, error) } // Add adds a func as a new child dependency. -func (ini *Init) Add(initFn func() (any, error)) *Init { +func (ini *Init) Add(initFn func(context.Context) (any, error)) *Init { if ini == nil { ini = New() } @@ -59,14 +59,14 @@ func (ini *Init) InitCount() int { // AddWithTimeout is same as Add, but with a timeout that aborts initialization. func (ini *Init) AddWithTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) *Init { - return ini.Add(func() (any, error) { - return ini.withTimeout(timeout, f) + return ini.Add(func(ctx context.Context) (any, error) { + return ini.withTimeout(ctx, timeout, f) }) } // Branch creates a new dependency branch based on an existing and adds // the given dependency as a child. -func (ini *Init) Branch(initFn func() (any, error)) *Init { +func (ini *Init) Branch(initFn func(context.Context) (any, error)) *Init { if ini == nil { ini = New() } @@ -75,13 +75,13 @@ func (ini *Init) Branch(initFn func() (any, error)) *Init { // BranchdWithTimeout is same as Branch, but with a timeout. func (ini *Init) BranchWithTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) *Init { - return ini.Branch(func() (any, error) { - return ini.withTimeout(timeout, f) + return ini.Branch(func(ctx context.Context) (any, error) { + return ini.withTimeout(ctx, timeout, f) }) } // Do initializes the entire dependency graph. -func (ini *Init) Do() (any, error) { +func (ini *Init) Do(ctx context.Context) (any, error) { if ini == nil { panic("init is nil") } @@ -92,7 +92,7 @@ func (ini *Init) Do() (any, error) { if prev != nil { // A branch. Initialize the ancestors. if prev.shouldInitialize() { - _, err := prev.Do() + _, err := prev.Do(ctx) if err != nil { ini.err = err return @@ -105,12 +105,12 @@ func (ini *Init) Do() (any, error) { } if ini.f != nil { - ini.out, ini.err = ini.f() + ini.out, ini.err = ini.f(ctx) } for _, child := range ini.children { if child.shouldInitialize() { - _, err := child.Do() + _, err := child.Do(ctx) if err != nil { ini.err = err return @@ -154,7 +154,7 @@ func (ini *Init) Reset() { } } -func (ini *Init) add(branch bool, initFn func() (any, error)) *Init { +func (ini *Init) add(branch bool, initFn func(context.Context) (any, error)) *Init { ini.mu.Lock() defer ini.mu.Unlock() @@ -179,8 +179,8 @@ func (ini *Init) checkDone() { } } -func (ini *Init) withTimeout(timeout time.Duration, f func(ctx context.Context) (any, error)) (any, error) { - ctx, cancel := context.WithTimeout(context.Background(), timeout) +func (ini *Init) withTimeout(ctx context.Context, timeout time.Duration, f func(ctx context.Context) (any, error)) (any, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() c := make(chan verr, 1) diff --git a/lazy/init_test.go b/lazy/init_test.go index 4d871b937..499ea2cce 100644 --- a/lazy/init_test.go +++ b/lazy/init_test.go @@ -48,16 +48,16 @@ func TestInit(t *testing.T) { var result string - f1 := func(name string) func() (any, error) { - return func() (any, error) { + f1 := func(name string) func(context.Context) (any, error) { + return func(context.Context) (any, error) { result += name + "|" doWork() return name, nil } } - f2 := func() func() (any, error) { - return func() (any, error) { + f2 := func() func(context.Context) (any, error) { + return func(context.Context) (any, error) { doWork() return nil, nil } @@ -75,6 +75,8 @@ func TestInit(t *testing.T) { var wg sync.WaitGroup + ctx := context.Background() + // Add some concurrency and randomness to verify thread safety and // init order. for i := 0; i < 100; i++ { @@ -83,20 +85,20 @@ func TestInit(t *testing.T) { defer wg.Done() var err error if rnd.Intn(10) < 5 { - _, err = root.Do() + _, err = root.Do(ctx) c.Assert(err, qt.IsNil) } // Add a new branch on the fly. if rnd.Intn(10) > 5 { branch := branch1_2.Branch(f2()) - _, err = branch.Do() + _, err = branch.Do(ctx) c.Assert(err, qt.IsNil) } else { - _, err = branch1_2_1.Do() + _, err = branch1_2_1.Do(ctx) c.Assert(err, qt.IsNil) } - _, err = branch1_2.Do() + _, err = branch1_2.Do(ctx) c.Assert(err, qt.IsNil) }(i) @@ -114,7 +116,7 @@ func TestInitAddWithTimeout(t *testing.T) { return nil, nil }) - _, err := init.Do() + _, err := init.Do(context.Background()) c.Assert(err, qt.IsNil) } @@ -133,7 +135,7 @@ func TestInitAddWithTimeoutTimeout(t *testing.T) { return nil, nil }) - _, err := init.Do() + _, err := init.Do(context.Background()) c.Assert(err, qt.Not(qt.IsNil)) @@ -149,7 +151,7 @@ func TestInitAddWithTimeoutError(t *testing.T) { return nil, errors.New("failed") }) - _, err := init.Do() + _, err := init.Do(context.Background()) c.Assert(err, qt.Not(qt.IsNil)) } @@ -178,8 +180,8 @@ func TestInitBranchOrder(t *testing.T) { base := New() - work := func(size int, f func()) func() (any, error) { - return func() (any, error) { + work := func(size int, f func()) func(context.Context) (any, error) { + return func(context.Context) (any, error) { doWorkOfSize(size) if f != nil { f() @@ -205,13 +207,14 @@ func TestInitBranchOrder(t *testing.T) { } var wg sync.WaitGroup + ctx := context.Background() for _, v := range inits { v := v wg.Add(1) go func() { defer wg.Done() - _, err := v.Do() + _, err := v.Do(ctx) c.Assert(err, qt.IsNil) }() } @@ -225,17 +228,17 @@ func TestInitBranchOrder(t *testing.T) { func TestResetError(t *testing.T) { c := qt.New(t) r := false - i := New().Add(func() (any, error) { + i := New().Add(func(context.Context) (any, error) { if r { return nil, nil } return nil, errors.New("r is false") }) - _, err := i.Do() + _, err := i.Do(context.Background()) c.Assert(err, qt.IsNotNil) i.Reset() r = true - _, err = i.Do() + _, err = i.Do(context.Background()) c.Assert(err, qt.IsNil) } diff --git a/markup/asciidocext/convert.go b/markup/asciidocext/convert.go index 4c83e0e95..c9524778f 100644 --- a/markup/asciidocext/convert.go +++ b/markup/asciidocext/convert.go @@ -53,10 +53,10 @@ func (p provider) New(cfg converter.ProviderConfig) (converter.Provider, error) type asciidocResult struct { converter.Result - toc tableofcontents.Root + toc *tableofcontents.Fragments } -func (r asciidocResult) TableOfContents() tableofcontents.Root { +func (r asciidocResult) TableOfContents() *tableofcontents.Fragments { return r.toc } @@ -205,16 +205,16 @@ func hasAsciiDoc() bool { // extractTOC extracts the toc from the given src html. // It returns the html without the TOC, and the TOC data -func (a *asciidocConverter) extractTOC(src []byte) ([]byte, tableofcontents.Root, error) { +func (a *asciidocConverter) extractTOC(src []byte) ([]byte, *tableofcontents.Fragments, error) { var buf bytes.Buffer buf.Write(src) node, err := html.Parse(&buf) if err != nil { - return nil, tableofcontents.Root{}, err + return nil, nil, err } var ( f func(*html.Node) bool - toc tableofcontents.Root + toc *tableofcontents.Fragments toVisit []*html.Node ) f = func(n *html.Node) bool { @@ -242,12 +242,12 @@ func (a *asciidocConverter) extractTOC(src []byte) ([]byte, tableofcontents.Root } f(node) if err != nil { - return nil, tableofcontents.Root{}, err + return nil, nil, err } buf.Reset() err = html.Render(&buf, node) if err != nil { - return nil, tableofcontents.Root{}, err + return nil, nil, err } // ltrim and rtrim which are added by html.Render res := buf.Bytes()[25:] @@ -256,9 +256,9 @@ func (a *asciidocConverter) extractTOC(src []byte) ([]byte, tableofcontents.Root } // parseTOC returns a TOC root from the given toc Node -func parseTOC(doc *html.Node) tableofcontents.Root { +func parseTOC(doc *html.Node) *tableofcontents.Fragments { var ( - toc tableofcontents.Root + toc tableofcontents.Builder f func(*html.Node, int, int) ) f = func(n *html.Node, row, level int) { @@ -276,9 +276,9 @@ func parseTOC(doc *html.Node) tableofcontents.Root { continue } href := attr(c, "href")[1:] - toc.AddAt(tableofcontents.Heading{ - Text: nodeContent(c), - ID: href, + toc.AddAt(&tableofcontents.Heading{ + Title: nodeContent(c), + ID: href, }, row, level) } f(n.FirstChild, row, level) @@ -289,7 +289,7 @@ func parseTOC(doc *html.Node) tableofcontents.Root { } } f(doc.FirstChild, -1, 0) - return toc + return toc.Build() } func attr(node *html.Node, key string) string { diff --git a/markup/asciidocext/convert_test.go b/markup/asciidocext/convert_test.go index 3a350c5ce..47208c066 100644 --- a/markup/asciidocext/convert_test.go +++ b/markup/asciidocext/convert_test.go @@ -21,13 +21,13 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/markup_config" - "github.com/gohugoio/hugo/markup/tableofcontents" qt "github.com/frankban/quicktest" ) @@ -343,49 +343,8 @@ testContent c.Assert(err, qt.IsNil) toc, ok := r.(converter.TableOfContentsProvider) c.Assert(ok, qt.Equals, true) - expected := tableofcontents.Root{ - Headings: tableofcontents.Headings{ - { - ID: "", - Text: "", - Headings: tableofcontents.Headings{ - { - ID: "_introduction", - Text: "Introduction", - Headings: nil, - }, - { - ID: "_section_1", - Text: "Section 1", - Headings: tableofcontents.Headings{ - { - ID: "_section_1_1", - Text: "Section 1.1", - Headings: tableofcontents.Headings{ - { - ID: "_section_1_1_1", - Text: "Section 1.1.1", - Headings: nil, - }, - }, - }, - { - ID: "_section_1_2", - Text: "Section 1.2", - Headings: nil, - }, - }, - }, - { - ID: "_section_2", - Text: "Section 2", - Headings: nil, - }, - }, - }, - }, - } - c.Assert(toc.TableOfContents(), qt.DeepEquals, expected) + + c.Assert(toc.TableOfContents().Identifiers, qt.DeepEquals, collections.SortedStringSlice{"_introduction", "_section_1", "_section_1_1", "_section_1_1_1", "_section_1_2", "_section_2"}) c.Assert(string(r.Bytes()), qt.Not(qt.Contains), "

") } @@ -404,22 +363,7 @@ func TestTableOfContentsWithCode(t *testing.T) { c.Assert(err, qt.IsNil) toc, ok := r.(converter.TableOfContentsProvider) c.Assert(ok, qt.Equals, true) - expected := tableofcontents.Root{ - Headings: tableofcontents.Headings{ - { - ID: "", - Text: "", - Headings: tableofcontents.Headings{ - { - ID: "_some_code_in_the_title", - Text: "Some code in the title", - Headings: nil, - }, - }, - }, - }, - } - c.Assert(toc.TableOfContents(), qt.DeepEquals, expected) + c.Assert(toc.TableOfContents().HeadingsMap["_some_code_in_the_title"].Title, qt.Equals, "Some code in the title") c.Assert(string(r.Bytes()), qt.Not(qt.Contains), "
") } @@ -443,21 +387,7 @@ func TestTableOfContentsPreserveTOC(t *testing.T) { c.Assert(err, qt.IsNil) toc, ok := r.(converter.TableOfContentsProvider) c.Assert(ok, qt.Equals, true) - expected := tableofcontents.Root{ - Headings: tableofcontents.Headings{ - { - ID: "", - Text: "", - Headings: tableofcontents.Headings{ - { - ID: "some-title", - Text: "Some title", - Headings: nil, - }, - }, - }, - }, - } - c.Assert(toc.TableOfContents(), qt.DeepEquals, expected) + + c.Assert(toc.TableOfContents().Identifiers, qt.DeepEquals, collections.SortedStringSlice{"some-title"}) c.Assert(string(r.Bytes()), qt.Contains, "
") } diff --git a/markup/converter/converter.go b/markup/converter/converter.go index c760381f4..7e5b56b07 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -101,7 +101,7 @@ type DocumentInfo interface { // TableOfContentsProvider provides the content as a ToC structure. type TableOfContentsProvider interface { - TableOfContents() tableofcontents.Root + TableOfContents() *tableofcontents.Fragments } // AnchorNameSanitizer tells how a converter sanitizes anchor names. diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index a179cd233..6c1c7ad0a 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -160,11 +160,11 @@ var _ identity.IdentitiesProvider = (*converterResult)(nil) type converterResult struct { converter.Result - toc tableofcontents.Root + toc *tableofcontents.Fragments ids identity.Identities } -func (c converterResult) TableOfContents() tableofcontents.Root { +func (c converterResult) TableOfContents() *tableofcontents.Fragments { return c.toc } @@ -228,9 +228,9 @@ type parserContext struct { parser.Context } -func (p *parserContext) TableOfContents() tableofcontents.Root { +func (p *parserContext) TableOfContents() *tableofcontents.Fragments { if v := p.Get(tocResultKey); v != nil { - return v.(tableofcontents.Root) + return v.(*tableofcontents.Fragments) } - return tableofcontents.Root{} + return nil } diff --git a/markup/goldmark/toc.go b/markup/goldmark/toc.go index 396c1d071..ac5040e85 100644 --- a/markup/goldmark/toc.go +++ b/markup/goldmark/toc.go @@ -41,8 +41,8 @@ func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parse } var ( - toc tableofcontents.Root - tocHeading tableofcontents.Heading + toc tableofcontents.Builder + tocHeading = &tableofcontents.Heading{} level int row = -1 inHeading bool @@ -53,10 +53,10 @@ func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parse s := ast.WalkStatus(ast.WalkContinue) if n.Kind() == ast.KindHeading { if inHeading && !entering { - tocHeading.Text = headingText.String() + tocHeading.Title = headingText.String() headingText.Reset() toc.AddAt(tocHeading, row, level-1) - tocHeading = tableofcontents.Heading{} + tocHeading = &tableofcontents.Heading{} inHeading = false return s, nil } @@ -106,7 +106,7 @@ func (t *tocTransformer) Transform(n *ast.Document, reader text.Reader, pc parse return s, nil }) - pc.Set(tocResultKey, toc) + pc.Set(tocResultKey, toc.Build()) } type tocExtension struct { diff --git a/markup/tableofcontents/tableofcontents.go b/markup/tableofcontents/tableofcontents.go index 2e7f47d20..bd0aaa801 100644 --- a/markup/tableofcontents/tableofcontents.go +++ b/markup/tableofcontents/tableofcontents.go @@ -14,35 +14,104 @@ package tableofcontents import ( + "sort" "strings" + + "github.com/gohugoio/hugo/common/collections" ) +// Empty is an empty ToC. +var Empty = &Fragments{ + Headings: Headings{}, + HeadingsMap: map[string]*Heading{}, +} + +// Builder is used to build the ToC data structure. +type Builder struct { + toc *Fragments +} + +// Add adds the heading to the ToC. +func (b *Builder) AddAt(h *Heading, row, level int) { + if b.toc == nil { + b.toc = &Fragments{} + } + b.toc.addAt(h, row, level) +} + +// Build returns the ToC. +func (b Builder) Build() *Fragments { + if b.toc == nil { + return Empty + } + b.toc.HeadingsMap = make(map[string]*Heading) + b.toc.walk(func(h *Heading) { + if h.ID != "" { + b.toc.HeadingsMap[h.ID] = h + b.toc.Identifiers = append(b.toc.Identifiers, h.ID) + } + }) + sort.Strings(b.toc.Identifiers) + return b.toc +} + // Headings holds the top level headings. -type Headings []Heading +type Headings []*Heading + +// FilterBy returns a new Headings slice with all headings that matches the given predicate. +// For internal use only. +func (h Headings) FilterBy(fn func(*Heading) bool) Headings { + var out Headings + + for _, h := range h { + h.walk(func(h *Heading) { + if fn(h) { + out = append(out, h) + } + }) + } + return out +} // Heading holds the data about a heading and its children. type Heading struct { - ID string - Text string + ID string + Title string Headings Headings } // IsZero is true when no ID or Text is set. func (h Heading) IsZero() bool { - return h.ID == "" && h.Text == "" + return h.ID == "" && h.Title == "" } -// Root implements AddAt, which can be used to build the -// data structure for the ToC. -type Root struct { +func (h *Heading) walk(fn func(*Heading)) { + fn(h) + for _, h := range h.Headings { + h.walk(fn) + } +} + +// Fragments holds the table of contents for a page. +type Fragments struct { + // Headings holds the top level headings. Headings Headings + + // Identifiers holds all the identifiers in the ToC as a sorted slice. + // Note that collections.SortedStringSlice has both a Contains and Count method + // that can be used to identify missing and duplicate IDs. + Identifiers collections.SortedStringSlice + + // HeadingsMap holds all the headings in the ToC as a map. + // Note that with duplicate IDs, the last one will win. + HeadingsMap map[string]*Heading } -// AddAt adds the heading into the given location. -func (toc *Root) AddAt(h Heading, row, level int) { +// addAt adds the heading into the given location. +func (toc *Fragments) addAt(h *Heading, row, level int) { for i := len(toc.Headings); i <= row; i++ { - toc.Headings = append(toc.Headings, Heading{}) + toc.Headings = append(toc.Headings, &Heading{}) } if level == 0 { @@ -50,19 +119,22 @@ func (toc *Root) AddAt(h Heading, row, level int) { return } - heading := &toc.Headings[row] + heading := toc.Headings[row] for i := 1; i < level; i++ { if len(heading.Headings) == 0 { - heading.Headings = append(heading.Headings, Heading{}) + heading.Headings = append(heading.Headings, &Heading{}) } - heading = &heading.Headings[len(heading.Headings)-1] + heading = heading.Headings[len(heading.Headings)-1] } heading.Headings = append(heading.Headings, h) } // ToHTML renders the ToC as HTML. -func (toc Root) ToHTML(startLevel, stopLevel int, ordered bool) string { +func (toc *Fragments) ToHTML(startLevel, stopLevel int, ordered bool) string { + if toc == nil { + return "" + } b := &tocBuilder{ s: strings.Builder{}, h: toc.Headings, @@ -74,6 +146,12 @@ func (toc Root) ToHTML(startLevel, stopLevel int, ordered bool) string { return b.s.String() } +func (toc Fragments) walk(fn func(*Heading)) { + for _, h := range toc.Headings { + h.walk(fn) + } +} + type tocBuilder struct { s strings.Builder h Headings @@ -133,11 +211,11 @@ func (b *tocBuilder) writeHeadings(level, indent int, h Headings) { } } -func (b *tocBuilder) writeHeading(level, indent int, h Heading) { +func (b *tocBuilder) writeHeading(level, indent int, h *Heading) { b.indent(indent) b.s.WriteString("
  • ") if !h.IsZero() { - b.s.WriteString("" + h.Text + "") + b.s.WriteString("" + h.Title + "") } b.writeHeadings(level, indent, h.Headings) b.s.WriteString("
    • \n") diff --git a/markup/tableofcontents/tableofcontents_test.go b/markup/tableofcontents/tableofcontents_test.go index daeb9f991..adbda4b00 100644 --- a/markup/tableofcontents/tableofcontents_test.go +++ b/markup/tableofcontents/tableofcontents_test.go @@ -17,18 +17,33 @@ import ( "testing" qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/common/collections" ) +var newTestTocBuilder = func() Builder { + var b Builder + b.AddAt(&Heading{Title: "Heading 1", ID: "h1-1"}, 0, 0) + b.AddAt(&Heading{Title: "1-H2-1", ID: "1-h2-1"}, 0, 1) + b.AddAt(&Heading{Title: "1-H2-2", ID: "1-h2-2"}, 0, 1) + b.AddAt(&Heading{Title: "1-H3-1", ID: "1-h2-2"}, 0, 2) + b.AddAt(&Heading{Title: "Heading 2", ID: "h1-2"}, 1, 0) + return b +} + +var newTestToc = func() *Fragments { + return newTestTocBuilder().Build() +} + func TestToc(t *testing.T) { c := qt.New(t) - toc := &Root{} + toc := &Fragments{} - toc.AddAt(Heading{Text: "Heading 1", ID: "h1-1"}, 0, 0) - toc.AddAt(Heading{Text: "1-H2-1", ID: "1-h2-1"}, 0, 1) - toc.AddAt(Heading{Text: "1-H2-2", ID: "1-h2-2"}, 0, 1) - toc.AddAt(Heading{Text: "1-H3-1", ID: "1-h2-2"}, 0, 2) - toc.AddAt(Heading{Text: "Heading 2", ID: "h1-2"}, 1, 0) + toc.addAt(&Heading{Title: "Heading 1", ID: "h1-1"}, 0, 0) + toc.addAt(&Heading{Title: "1-H2-1", ID: "1-h2-1"}, 0, 1) + toc.addAt(&Heading{Title: "1-H2-2", ID: "1-h2-2"}, 0, 1) + toc.addAt(&Heading{Title: "1-H3-1", ID: "1-h2-2"}, 0, 2) + toc.addAt(&Heading{Title: "Heading 2", ID: "h1-2"}, 1, 0) got := toc.ToHTML(1, -1, false) c.Assert(got, qt.Equals, `