Speed up GetTerms

```text
name                               old time/op    new time/op    delta
TaxonomiesGetTerms/pages_100-10      5.25ms  5%    5.13ms  4%     ~     (p=0.486 n=4+4)
TaxonomiesGetTerms/pages_1000-10     30.1ms  1%    26.8ms  1%  -11.13%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_10000-10     1.33s 24%     0.29s  2%  -78.42%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_20000-10     5.50s 12%     0.83s 28%  -84.88%  (p=0.029 n=4+4)

name                               old alloc/op   new alloc/op   delta
TaxonomiesGetTerms/pages_100-10      4.08MB  0%    4.06MB  0%   -0.59%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_1000-10     25.1MB  0%    24.9MB  0%   -0.87%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_10000-10     238MB  2%     233MB  0%   -1.94%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_20000-10     469MB  0%     465MB  0%   -1.00%  (p=0.029 n=4+4)

name                               old allocs/op  new allocs/op  delta
TaxonomiesGetTerms/pages_100-10       49.5k  0%     48.9k  0%   -1.17%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_1000-10       304k  0%      298k  0%   -1.97%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_10000-10     3.02M  7%     2.81M  0%   -7.09%  (p=0.029 n=4+4)
TaxonomiesGetTerms/pages_20000-10     5.77M  1%     5.59M  0%   -3.19%  (p=0.029 n=4+4)

```

Note that the numbers above represents a full site build, but GetTerms is a big part of the site in question.

Fixes #12610
This commit is contained in:
Bjørn Erik Pedersen 2024-06-19 12:52:10 +02:00
parent b46d101d5a
commit 478a9107a6
2 changed files with 66 additions and 9 deletions

View file

@ -98,6 +98,7 @@ type pageMap struct {
cachePages1 *dynacache.Partition[string, page.Pages] cachePages1 *dynacache.Partition[string, page.Pages]
cachePages2 *dynacache.Partition[string, page.Pages] cachePages2 *dynacache.Partition[string, page.Pages]
cacheResources *dynacache.Partition[string, resource.Resources] cacheResources *dynacache.Partition[string, resource.Resources]
cacheGetTerms *dynacache.Partition[string, map[string]page.Pages]
cacheContentRendered *dynacache.Partition[string, *resources.StaleValue[contentSummary]] cacheContentRendered *dynacache.Partition[string, *resources.StaleValue[contentSummary]]
cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]] cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]]
contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]] contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]]
@ -448,16 +449,13 @@ func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages {
func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages { func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages {
prefix := paths.AddLeadingSlash(taxonomy) prefix := paths.AddLeadingSlash(taxonomy)
v, err := m.cachePages1.GetOrCreate(prefix+path, func(string) (page.Pages, error) { termPages, err := m.cacheGetTerms.GetOrCreate(prefix, func(string) (map[string]page.Pages, error) {
var pas page.Pages mm := make(map[string]page.Pages)
err := m.treeTaxonomyEntries.WalkPrefix( err := m.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeNone, doctree.LockTypeNone,
paths.AddTrailingSlash(prefix), paths.AddTrailingSlash(prefix),
func(s string, n *weightedContentNode) (bool, error) { func(s string, n *weightedContentNode) (bool, error) {
if strings.HasSuffix(s, path) { mm[n.n.Path()] = append(mm[n.n.Path()], n.term)
pas = append(pas, n.term)
}
return false, nil return false, nil
}, },
) )
@ -465,15 +463,18 @@ func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages {
return nil, err return nil, err
} }
page.SortByDefault(pas) // Sort the terms.
for _, v := range mm {
page.SortByDefault(v)
}
return pas, nil return mm, nil
}) })
if err != nil { if err != nil {
panic(err) panic(err)
} }
return v return termPages[path]
} }
func (m *pageMap) forEachResourceInPage( func (m *pageMap) forEachResourceInPage(
@ -898,6 +899,7 @@ func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *
pageTrees: pageTrees.Shape(0, i), pageTrees: pageTrees.Shape(0, i),
cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}), cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
cacheGetTerms: dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%d", i), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}),
cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}), cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}),
cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}), cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),

View file

@ -970,3 +970,58 @@ title: p1
b.AssertFileExists("public/ja/s1/index.html", false) // failing test b.AssertFileExists("public/ja/s1/index.html", false) // failing test
b.AssertFileExists("public/ja/s1/category/index.html", true) b.AssertFileExists("public/ja/s1/category/index.html", true)
} }
func BenchmarkTaxonomiesGetTerms(b *testing.B) {
createBuilders := func(b *testing.B, numPages int) []*IntegrationTestBuilder {
files := `
-- hugo.toml --
baseURL = "https://example.com"
disableKinds = ["RSS", "sitemap", "section"]
[taxononomies]
tag = "tags"
-- layouts/_default/list.html --
List.
-- layouts/_default/single.html --
GetTerms.tags: {{ range .GetTerms "tags" }}{{ .Title }}|{{ end }}
-- content/_index.md --
`
tagsVariants := []string{
"tags: ['a']",
"tags: ['a', 'b']",
"tags: ['a', 'b', 'c']",
"tags: ['a', 'b', 'c', 'd']",
"tags: ['a', 'b', 'd', 'e']",
"tags: ['a', 'b', 'c', 'd', 'e']",
"tags: ['a', 'd']",
"tags: ['a', 'f']",
}
for i := 1; i < numPages; i++ {
tags := tagsVariants[i%len(tagsVariants)]
files += fmt.Sprintf("\n-- content/posts/p%d.md --\n---\n%s\n---", i+1, tags)
}
cfg := IntegrationTestConfig{
T: b,
TxtarString: files,
}
builders := make([]*IntegrationTestBuilder, b.N)
for i := range builders {
builders[i] = NewIntegrationTestBuilder(cfg)
}
b.ResetTimer()
return builders
}
for _, numPages := range []int{100, 1000, 10000, 20000} {
b.Run(fmt.Sprintf("pages_%d", numPages), func(b *testing.B) {
builders := createBuilders(b, numPages)
for i := 0; i < b.N; i++ {
builders[i].Build()
}
})
}
}