hugo/hugolib/content_map_page.go
Bjørn Erik Pedersen 6cd0784e44 Implement defer
Closes #8086
Closes #12589
2024-06-23 11:25:47 +02:00

2144 lines
56 KiB
Go

// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"context"
"fmt"
"io"
"path"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/bep/logg"
"github.com/gohugoio/hugo/cache/dynacache"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/common/predicate"
"github.com/gohugoio/hugo/common/rungroup"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/hugolib/doctree"
"github.com/gohugoio/hugo/hugolib/pagesfromdata"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/resources"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/resources/kinds"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/gohugoio/hugo/resources/resource"
)
var pagePredicates = struct {
KindPage predicate.P[*pageState]
KindSection predicate.P[*pageState]
KindHome predicate.P[*pageState]
KindTerm predicate.P[*pageState]
ShouldListLocal predicate.P[*pageState]
ShouldListGlobal predicate.P[*pageState]
ShouldListAny predicate.P[*pageState]
ShouldLink predicate.P[page.Page]
}{
KindPage: func(p *pageState) bool {
return p.Kind() == kinds.KindPage
},
KindSection: func(p *pageState) bool {
return p.Kind() == kinds.KindSection
},
KindHome: func(p *pageState) bool {
return p.Kind() == kinds.KindHome
},
KindTerm: func(p *pageState) bool {
return p.Kind() == kinds.KindTerm
},
ShouldListLocal: func(p *pageState) bool {
return p.m.shouldList(false)
},
ShouldListGlobal: func(p *pageState) bool {
return p.m.shouldList(true)
},
ShouldListAny: func(p *pageState) bool {
return p.m.shouldListAny()
},
ShouldLink: func(p page.Page) bool {
return !p.(*pageState).m.noLink()
},
}
type pageMap struct {
i int
s *Site
// Main storage for all pages.
*pageTrees
// Used for simple page lookups by name, e.g. "mypage.md" or "mypage".
pageReverseIndex *contentTreeReverseIndex
cachePages1 *dynacache.Partition[string, page.Pages]
cachePages2 *dynacache.Partition[string, page.Pages]
cacheResources *dynacache.Partition[string, resource.Resources]
cacheGetTerms *dynacache.Partition[string, map[string]page.Pages]
cacheContentRendered *dynacache.Partition[string, *resources.StaleValue[contentSummary]]
cacheContentPlain *dynacache.Partition[string, *resources.StaleValue[contentPlainPlainWords]]
contentTableOfContents *dynacache.Partition[string, *resources.StaleValue[contentTableOfContents]]
contentDataFileSeenItems *maps.Cache[string, map[uint64]bool]
cfg contentMapConfig
}
// pageTrees holds pages and resources in a tree structure for all sites/languages.
// Each site gets its own tree set via the Shape method.
type pageTrees struct {
// This tree contains all Pages.
// This include regular pages, sections, taxonomies and so on.
// Note that all of these trees share the same key structure,
// so you can take a leaf Page key and do a prefix search
// with key + "/" to get all of its resources.
treePages *doctree.NodeShiftTree[contentNodeI]
// This tree contains Resources bundled in pages.
treeResources *doctree.NodeShiftTree[contentNodeI]
// All pages and resources.
treePagesResources doctree.WalkableTrees[contentNodeI]
// This tree contains all taxonomy entries, e.g "/tags/blue/page1"
treeTaxonomyEntries *doctree.TreeShiftTree[*weightedContentNode]
// Stores the state for _content.gotmpl files.
// Mostly releveant for rebuilds.
treePagesFromTemplateAdapters *doctree.TreeShiftTree[*pagesfromdata.PagesFromTemplate]
// A slice of the resource trees.
resourceTrees doctree.MutableTrees
}
// collectAndMarkStaleIdentities collects all identities from in all trees matching the given key.
// We currently re-read all page/resources for all languages that share the same path,
// so we mark all entries as stale (which will trigger cache invalidation), then
// return the first.
func (t *pageTrees) collectAndMarkStaleIdentities(p *paths.Path) []identity.Identity {
key := p.Base()
var ids []identity.Identity
// We need only one identity sample per dimension.
nCount := 0
cb := func(n contentNodeI) bool {
if n == nil {
return false
}
n.MarkStale()
if nCount > 0 {
return true
}
nCount++
n.ForEeachIdentity(func(id identity.Identity) bool {
ids = append(ids, id)
return false
})
return false
}
tree := t.treePages
nCount = 0
tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(),
cb,
)
tree = t.treeResources
nCount = 0
tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(),
cb,
)
if p.Component() == files.ComponentFolderContent {
// It may also be a bundled content resource.
key := p.ForBundleType(paths.PathTypeContentResource).Base()
tree = t.treeResources
nCount = 0
tree.ForEeachInDimension(key, doctree.DimensionLanguage.Index(),
cb,
)
}
return ids
}
// collectIdentitiesSurrounding collects all identities surrounding the given key.
func (t *pageTrees) collectIdentitiesSurrounding(key string, maxSamplesPerTree int) []identity.Identity {
ids := t.collectIdentitiesSurroundingIn(key, maxSamplesPerTree, t.treePages)
ids = append(ids, t.collectIdentitiesSurroundingIn(key, maxSamplesPerTree, t.treeResources)...)
return ids
}
func (t *pageTrees) collectIdentitiesSurroundingIn(key string, maxSamples int, tree *doctree.NodeShiftTree[contentNodeI]) []identity.Identity {
var ids []identity.Identity
section, ok := tree.LongestPrefixAll(path.Dir(key))
if ok {
count := 0
prefix := section + "/"
level := strings.Count(prefix, "/")
tree.WalkPrefixRaw(prefix, func(s string, n contentNodeI) bool {
if level != strings.Count(s, "/") {
return false
}
n.ForEeachIdentity(func(id identity.Identity) bool {
ids = append(ids, id)
return false
})
count++
return count > maxSamples
})
}
return ids
}
func (t *pageTrees) DeletePageAndResourcesBelow(ss ...string) {
commit1 := t.resourceTrees.Lock(true)
defer commit1()
commit2 := t.treePages.Lock(true)
defer commit2()
for _, s := range ss {
t.resourceTrees.DeletePrefix(paths.AddTrailingSlash(s))
t.treePages.Delete(s)
}
}
// Shape shapes all trees in t to the given dimension.
func (t pageTrees) Shape(d, v int) *pageTrees {
t.treePages = t.treePages.Shape(d, v)
t.treeResources = t.treeResources.Shape(d, v)
t.treeTaxonomyEntries = t.treeTaxonomyEntries.Shape(d, v)
t.treePagesFromTemplateAdapters = t.treePagesFromTemplateAdapters.Shape(d, v)
t.createMutableTrees()
return &t
}
func (t *pageTrees) createMutableTrees() {
t.treePagesResources = doctree.WalkableTrees[contentNodeI]{
t.treePages,
t.treeResources,
}
t.resourceTrees = doctree.MutableTrees{
t.treeResources,
}
}
var (
_ resource.Identifier = pageMapQueryPagesInSection{}
_ resource.Identifier = pageMapQueryPagesBelowPath{}
)
type pageMapQueryPagesInSection struct {
pageMapQueryPagesBelowPath
Recursive bool
IncludeSelf bool
}
func (q pageMapQueryPagesInSection) Key() string {
return "gagesInSection" + "/" + q.pageMapQueryPagesBelowPath.Key() + "/" + strconv.FormatBool(q.Recursive) + "/" + strconv.FormatBool(q.IncludeSelf)
}
// This needs to be hashable.
type pageMapQueryPagesBelowPath struct {
Path string
// Additional identifier for this query.
// Used as part of the cache key.
KeyPart string
// Page inclusion filter.
// May be nil.
Include predicate.P[*pageState]
}
func (q pageMapQueryPagesBelowPath) Key() string {
return q.Path + "/" + q.KeyPart
}
// Apply fn to all pages in m matching the given predicate.
// fn may return true to stop the walk.
func (m *pageMap) forEachPage(include predicate.P[*pageState], fn func(p *pageState) (bool, error)) error {
if include == nil {
include = func(p *pageState) bool {
return true
}
}
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: m.treePages,
LockType: doctree.LockTypeRead,
Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if p, ok := n.(*pageState); ok && include(p) {
if terminate, err := fn(p); terminate || err != nil {
return terminate, err
}
}
return false, nil
},
}
return w.Walk(context.Background())
}
func (m *pageMap) forEeachPageIncludingBundledPages(include predicate.P[*pageState], fn func(p *pageState) (bool, error)) error {
if include == nil {
include = func(p *pageState) bool {
return true
}
}
if err := m.forEachPage(include, fn); err != nil {
return err
}
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: m.treeResources,
LockType: doctree.LockTypeRead,
Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if rs, ok := n.(*resourceSource); ok {
if p, ok := rs.r.(*pageState); ok && include(p) {
if terminate, err := fn(p); terminate || err != nil {
return terminate, err
}
}
}
return false, nil
},
}
return w.Walk(context.Background())
}
func (m *pageMap) getOrCreatePagesFromCache(
cache *dynacache.Partition[string, page.Pages],
key string, create func(string) (page.Pages, error),
) (page.Pages, error) {
if cache == nil {
cache = m.cachePages1
}
return cache.GetOrCreate(key, create)
}
func (m *pageMap) getPagesInSection(q pageMapQueryPagesInSection) page.Pages {
cacheKey := q.Key()
pages, err := m.getOrCreatePagesFromCache(nil, cacheKey, func(string) (page.Pages, error) {
prefix := paths.AddTrailingSlash(q.Path)
var (
pas page.Pages
otherBranch string
)
include := q.Include
if include == nil {
include = pagePredicates.ShouldListLocal
}
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: m.treePages,
Prefix: prefix,
}
w.Handle = func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if q.Recursive {
if p, ok := n.(*pageState); ok && include(p) {
pas = append(pas, p)
}
return false, nil
}
if p, ok := n.(*pageState); ok && include(p) {
pas = append(pas, p)
}
if n.isContentNodeBranch() {
currentBranch := key + "/"
if otherBranch == "" || otherBranch != currentBranch {
w.SkipPrefix(currentBranch)
}
otherBranch = currentBranch
}
return false, nil
}
err := w.Walk(context.Background())
if err == nil {
if q.IncludeSelf {
if n := m.treePages.Get(q.Path); n != nil {
if p, ok := n.(*pageState); ok && include(p) {
pas = append(pas, p)
}
}
}
page.SortByDefault(pas)
}
return pas, err
})
if err != nil {
panic(err)
}
return pages
}
func (m *pageMap) getPagesWithTerm(q pageMapQueryPagesBelowPath) page.Pages {
key := q.Key()
v, err := m.cachePages1.GetOrCreate(key, func(string) (page.Pages, error) {
var pas page.Pages
include := q.Include
if include == nil {
include = pagePredicates.ShouldListLocal
}
err := m.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeNone,
paths.AddTrailingSlash(q.Path),
func(s string, n *weightedContentNode) (bool, error) {
p := n.n.(*pageState)
if !include(p) {
return false, nil
}
pas = append(pas, pageWithWeight0{n.weight, p})
return false, nil
},
)
if err != nil {
return nil, err
}
page.SortByDefault(pas)
return pas, nil
})
if err != nil {
panic(err)
}
return v
}
func (m *pageMap) getTermsForPageInTaxonomy(path, taxonomy string) page.Pages {
prefix := paths.AddLeadingSlash(taxonomy)
termPages, err := m.cacheGetTerms.GetOrCreate(prefix, func(string) (map[string]page.Pages, error) {
mm := make(map[string]page.Pages)
err := m.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeNone,
paths.AddTrailingSlash(prefix),
func(s string, n *weightedContentNode) (bool, error) {
mm[n.n.Path()] = append(mm[n.n.Path()], n.term)
return false, nil
},
)
if err != nil {
return nil, err
}
// Sort the terms.
for _, v := range mm {
page.SortByDefault(v)
}
return mm, nil
})
if err != nil {
panic(err)
}
return termPages[path]
}
func (m *pageMap) forEachResourceInPage(
ps *pageState,
lockType doctree.LockType,
exact bool,
handle func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error),
) error {
keyPage := ps.Path()
if keyPage == "/" {
keyPage = ""
}
prefix := paths.AddTrailingSlash(ps.Path())
isBranch := ps.IsNode()
rw := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: m.treeResources,
Prefix: prefix,
LockType: lockType,
Exact: exact,
}
rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if isBranch {
// A resourceKey always represents a filename with extension.
// A page key points to the logical path of a page, which when sourced from the filesystem
// may represent a directory (bundles) or a single content file (e.g. p1.md).
// So, to avoid any overlapping ambiguity, we start looking from the owning directory.
ownerKey, _ := m.treePages.LongestPrefixAll(path.Dir(resourceKey))
if ownerKey != keyPage {
// Stop walking downwards, someone else owns this resource.
rw.SkipPrefix(ownerKey + "/")
return false, nil
}
}
return handle(resourceKey, n, match)
}
return rw.Walk(context.Background())
}
func (m *pageMap) getResourcesForPage(ps *pageState) (resource.Resources, error) {
var res resource.Resources
m.forEachResourceInPage(ps, doctree.LockTypeNone, false, func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
rs := n.(*resourceSource)
if rs.r != nil {
res = append(res, rs.r)
}
return false, nil
})
return res, nil
}
func (m *pageMap) getOrCreateResourcesForPage(ps *pageState) resource.Resources {
keyPage := ps.Path()
if keyPage == "/" {
keyPage = ""
}
key := keyPage + "/get-resources-for-page"
v, err := m.cacheResources.GetOrCreate(key, func(string) (resource.Resources, error) {
res, err := m.getResourcesForPage(ps)
if err != nil {
return nil, err
}
if translationKey := ps.m.pageConfig.TranslationKey; translationKey != "" {
// This this should not be a very common case.
// Merge in resources from the other languages.
translatedPages, _ := m.s.h.translationKeyPages.Get(translationKey)
for _, tp := range translatedPages {
if tp == ps {
continue
}
tps := tp.(*pageState)
// Make sure we query from the correct language root.
res2, err := tps.s.pageMap.getResourcesForPage(tps)
if err != nil {
return nil, err
}
// Add if Name not already in res.
for _, r := range res2 {
var found bool
for _, r2 := range res {
if r2.(resource.NameNormalizedProvider).NameNormalized() == r.(resource.NameNormalizedProvider).NameNormalized() {
found = true
break
}
}
if !found {
res = append(res, r)
}
}
}
}
lessFunc := func(i, j int) bool {
ri, rj := res[i], res[j]
if ri.ResourceType() < rj.ResourceType() {
return true
}
p1, ok1 := ri.(page.Page)
p2, ok2 := rj.(page.Page)
if ok1 != ok2 {
// Pull pages behind other resources.
return ok2
}
if ok1 {
return page.DefaultPageSort(p1, p2)
}
// Make sure not to use RelPermalink or any of the other methods that
// trigger lazy publishing.
return ri.Name() < rj.Name()
}
sort.SliceStable(res, lessFunc)
if len(ps.m.pageConfig.ResourcesMeta) > 0 {
for i, r := range res {
res[i] = resources.CloneWithMetadataFromMapIfNeeded(ps.m.pageConfig.ResourcesMeta, r)
}
sort.SliceStable(res, lessFunc)
}
return res, nil
})
if err != nil {
panic(err)
}
return v
}
type weightedContentNode struct {
n contentNodeI
weight int
term *pageWithOrdinal
}
type buildStateReseter interface {
resetBuildState()
}
type contentNodeI interface {
identity.IdentityProvider
identity.ForEeachIdentityProvider
Path() string
isContentNodeBranch() bool
buildStateReseter
resource.StaleMarker
}
var _ contentNodeI = (*contentNodeIs)(nil)
type contentNodeIs []contentNodeI
func (n contentNodeIs) Path() string {
return n[0].Path()
}
func (n contentNodeIs) isContentNodeBranch() bool {
return n[0].isContentNodeBranch()
}
func (n contentNodeIs) GetIdentity() identity.Identity {
return n[0].GetIdentity()
}
func (n contentNodeIs) ForEeachIdentity(f func(identity.Identity) bool) bool {
for _, nn := range n {
if nn != nil {
if nn.ForEeachIdentity(f) {
return true
}
}
}
return false
}
func (n contentNodeIs) resetBuildState() {
for _, nn := range n {
if nn != nil {
nn.resetBuildState()
}
}
}
func (n contentNodeIs) MarkStale() {
for _, nn := range n {
resource.MarkStale(nn)
}
}
type contentNodeShifter struct {
numLanguages int
}
func (s *contentNodeShifter) Delete(n contentNodeI, dimension doctree.Dimension) (contentNodeI, bool, bool) {
lidx := dimension[0]
switch v := n.(type) {
case contentNodeIs:
deleted := v[lidx]
resource.MarkStale(deleted)
wasDeleted := deleted != nil
v[lidx] = nil
isEmpty := true
for _, vv := range v {
if vv != nil {
isEmpty = false
break
}
}
return deleted, wasDeleted, isEmpty
case resourceSources:
deleted := v[lidx]
resource.MarkStale(deleted)
wasDeleted := deleted != nil
v[lidx] = nil
isEmpty := true
for _, vv := range v {
if vv != nil {
isEmpty = false
break
}
}
return deleted, wasDeleted, isEmpty
case *resourceSource:
if lidx != v.LangIndex() {
return nil, false, false
}
resource.MarkStale(v)
return v, true, true
case *pageState:
if lidx != v.s.languagei {
return nil, false, false
}
resource.MarkStale(v)
return v, true, true
default:
panic(fmt.Sprintf("unknown type %T", n))
}
}
func (s *contentNodeShifter) Shift(n contentNodeI, dimension doctree.Dimension, exact bool) (contentNodeI, bool, doctree.DimensionFlag) {
lidx := dimension[0]
// How accurate is the match.
accuracy := doctree.DimensionLanguage
switch v := n.(type) {
case contentNodeIs:
if len(v) == 0 {
panic("empty contentNodeIs")
}
vv := v[lidx]
if vv != nil {
return vv, true, accuracy
}
return nil, false, 0
case resourceSources:
vv := v[lidx]
if vv != nil {
return vv, true, doctree.DimensionLanguage
}
if exact {
return nil, false, 0
}
// For non content resources, pick the first match.
for _, vv := range v {
if vv != nil {
if vv.isPage() {
return nil, false, 0
}
return vv, true, 0
}
}
case *resourceSource:
if v.LangIndex() == lidx {
return v, true, doctree.DimensionLanguage
}
if !v.isPage() && !exact {
return v, true, 0
}
case *pageState:
if v.s.languagei == lidx {
return n, true, doctree.DimensionLanguage
}
default:
panic(fmt.Sprintf("unknown type %T", n))
}
return nil, false, 0
}
func (s *contentNodeShifter) ForEeachInDimension(n contentNodeI, d int, f func(contentNodeI) bool) {
if d != doctree.DimensionLanguage.Index() {
panic("only language dimension supported")
}
switch vv := n.(type) {
case contentNodeIs:
for _, v := range vv {
if v != nil {
if f(v) {
return
}
}
}
default:
f(vv)
}
}
func (s *contentNodeShifter) InsertInto(old, new contentNodeI, dimension doctree.Dimension) (contentNodeI, contentNodeI, bool) {
langi := dimension[doctree.DimensionLanguage.Index()]
switch vv := old.(type) {
case *pageState:
newp, ok := new.(*pageState)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
if vv.s.languagei == newp.s.languagei && newp.s.languagei == langi {
return new, vv, true
}
is := make(contentNodeIs, s.numLanguages)
is[vv.s.languagei] = old
is[langi] = new
return is, old, false
case contentNodeIs:
oldv := vv[langi]
vv[langi] = new
return vv, oldv, oldv != nil
case resourceSources:
oldv := vv[langi]
vv[langi] = new.(*resourceSource)
return vv, oldv, oldv != nil
case *resourceSource:
newp, ok := new.(*resourceSource)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
if vv.LangIndex() == newp.LangIndex() && newp.LangIndex() == langi {
return new, vv, true
}
rs := make(resourceSources, s.numLanguages)
rs[vv.LangIndex()] = vv
rs[langi] = newp
return rs, vv, false
default:
panic(fmt.Sprintf("unknown type %T", old))
}
}
func (s *contentNodeShifter) Insert(old, new contentNodeI) (contentNodeI, contentNodeI, bool) {
switch vv := old.(type) {
case *pageState:
newp, ok := new.(*pageState)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
if vv.s.languagei == newp.s.languagei {
if newp != old {
resource.MarkStale(old)
}
return new, vv, true
}
is := make(contentNodeIs, s.numLanguages)
is[newp.s.languagei] = new
is[vv.s.languagei] = old
return is, old, false
case contentNodeIs:
newp, ok := new.(*pageState)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
oldp := vv[newp.s.languagei]
if oldp != newp {
resource.MarkStale(oldp)
}
vv[newp.s.languagei] = new
return vv, oldp, oldp != nil
case *resourceSource:
newp, ok := new.(*resourceSource)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
if vv.LangIndex() == newp.LangIndex() {
if vv != newp {
resource.MarkStale(vv)
}
return new, vv, true
}
rs := make(resourceSources, s.numLanguages)
rs[newp.LangIndex()] = newp
rs[vv.LangIndex()] = vv
return rs, vv, false
case resourceSources:
newp, ok := new.(*resourceSource)
if !ok {
panic(fmt.Sprintf("unknown type %T", new))
}
oldp := vv[newp.LangIndex()]
if oldp != newp {
resource.MarkStale(oldp)
}
vv[newp.LangIndex()] = newp
return vv, oldp, oldp != nil
default:
panic(fmt.Sprintf("unknown type %T", old))
}
}
func newPageMap(i int, s *Site, mcache *dynacache.Cache, pageTrees *pageTrees) *pageMap {
var m *pageMap
var taxonomiesConfig taxonomiesConfig = s.conf.Taxonomies
m = &pageMap{
pageTrees: pageTrees.Shape(0, i),
cachePages1: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag1/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
cachePages2: dynacache.GetOrCreatePartition[string, page.Pages](mcache, fmt.Sprintf("/pag2/%d", i), dynacache.OptionsPartition{Weight: 10, ClearWhen: dynacache.ClearOnRebuild}),
cacheGetTerms: dynacache.GetOrCreatePartition[string, map[string]page.Pages](mcache, fmt.Sprintf("/gett/%d", i), dynacache.OptionsPartition{Weight: 5, ClearWhen: dynacache.ClearOnRebuild}),
cacheResources: dynacache.GetOrCreatePartition[string, resource.Resources](mcache, fmt.Sprintf("/ress/%d", i), dynacache.OptionsPartition{Weight: 60, ClearWhen: dynacache.ClearOnRebuild}),
cacheContentRendered: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentSummary]](mcache, fmt.Sprintf("/cont/ren/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
cacheContentPlain: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentPlainPlainWords]](mcache, fmt.Sprintf("/cont/pla/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
contentTableOfContents: dynacache.GetOrCreatePartition[string, *resources.StaleValue[contentTableOfContents]](mcache, fmt.Sprintf("/cont/toc/%d", i), dynacache.OptionsPartition{Weight: 70, ClearWhen: dynacache.ClearOnChange}),
contentDataFileSeenItems: maps.NewCache[string, map[uint64]bool](),
cfg: contentMapConfig{
lang: s.Lang(),
taxonomyConfig: taxonomiesConfig.Values(),
taxonomyDisabled: !s.conf.IsKindEnabled(kinds.KindTaxonomy),
taxonomyTermDisabled: !s.conf.IsKindEnabled(kinds.KindTerm),
pageDisabled: !s.conf.IsKindEnabled(kinds.KindPage),
},
i: i,
s: s,
}
m.pageReverseIndex = &contentTreeReverseIndex{
initFn: func(rm map[any]contentNodeI) {
add := func(k string, n contentNodeI) {
existing, found := rm[k]
if found && existing != ambiguousContentNode {
rm[k] = ambiguousContentNode
} else if !found {
rm[k] = n
}
}
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: m.treePages,
LockType: doctree.LockTypeRead,
Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
p := n.(*pageState)
if p.PathInfo() != nil {
add(p.PathInfo().BaseNameNoIdentifier(), p)
}
return false, nil
},
}
if err := w.Walk(context.Background()); err != nil {
panic(err)
}
},
contentTreeReverseIndexMap: &contentTreeReverseIndexMap{},
}
return m
}
type contentTreeReverseIndex struct {
initFn func(rm map[any]contentNodeI)
*contentTreeReverseIndexMap
}
func (c *contentTreeReverseIndex) Reset() {
c.contentTreeReverseIndexMap = &contentTreeReverseIndexMap{
m: make(map[any]contentNodeI),
}
}
func (c *contentTreeReverseIndex) Get(key any) contentNodeI {
c.init.Do(func() {
c.m = make(map[any]contentNodeI)
c.initFn(c.contentTreeReverseIndexMap.m)
})
return c.m[key]
}
type contentTreeReverseIndexMap struct {
init sync.Once
m map[any]contentNodeI
}
type sitePagesAssembler struct {
*Site
assembleChanges *WhatChanged
ctx context.Context
}
func (m *pageMap) debugPrint(prefix string, maxLevel int, w io.Writer) {
noshift := false
var prevKey string
pageWalker := &doctree.NodeShiftTreeWalker[contentNodeI]{
NoShift: noshift,
Tree: m.treePages,
Prefix: prefix,
WalkContext: &doctree.WalkContext[contentNodeI]{},
}
resourceWalker := pageWalker.Extend()
resourceWalker.Tree = m.treeResources
pageWalker.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
level := strings.Count(keyPage, "/")
if level > maxLevel {
return false, nil
}
const indentStr = " "
p := n.(*pageState)
s := strings.TrimPrefix(keyPage, paths.CommonDirPath(prevKey, keyPage))
lenIndent := len(keyPage) - len(s)
fmt.Fprint(w, strings.Repeat(indentStr, lenIndent))
info := fmt.Sprintf("%s lm: %s (%s)", s, p.Lastmod().Format("2006-01-02"), p.Kind())
fmt.Fprintln(w, info)
switch p.Kind() {
case kinds.KindTerm:
m.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeNone,
keyPage+"/",
func(s string, n *weightedContentNode) (bool, error) {
fmt.Fprint(w, strings.Repeat(indentStr, lenIndent+4))
fmt.Fprintln(w, s)
return false, nil
},
)
}
isBranch := n.isContentNodeBranch()
prevKey = keyPage
resourceWalker.Prefix = keyPage + "/"
resourceWalker.Handle = func(ss string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if isBranch {
ownerKey, _ := pageWalker.Tree.LongestPrefix(ss, true, nil)
if ownerKey != keyPage {
// Stop walking downwards, someone else owns this resource.
pageWalker.SkipPrefix(ownerKey + "/")
return false, nil
}
}
fmt.Fprint(w, strings.Repeat(indentStr, lenIndent+8))
fmt.Fprintln(w, ss+" (resource)")
return false, nil
}
return false, resourceWalker.Walk(context.Background())
}
err := pageWalker.Walk(context.Background())
if err != nil {
panic(err)
}
}
func (h *HugoSites) dynacacheGCFilenameIfNotWatchedAndDrainMatching(filename string) {
cpss := h.BaseFs.ResolvePaths(filename)
if len(cpss) == 0 {
return
}
// Compile cache busters.
var cacheBusters []func(string) bool
for _, cps := range cpss {
if cps.Watch {
continue
}
np := glob.NormalizePath(path.Join(cps.Component, cps.Path))
g, err := h.ResourceSpec.BuildConfig().MatchCacheBuster(h.Log, np)
if err == nil && g != nil {
cacheBusters = append(cacheBusters, g)
}
}
if len(cacheBusters) == 0 {
return
}
cacheBusterOr := func(s string) bool {
for _, cb := range cacheBusters {
if cb(s) {
return true
}
}
return false
}
h.dynacacheGCCacheBuster(cacheBusterOr)
// We want to avoid that evicted items in the above is considered in the next step server change.
_ = h.MemCache.DrainEvictedIdentitiesMatching(func(ki dynacache.KeyIdentity) bool {
return cacheBusterOr(ki.Key.(string))
})
}
func (h *HugoSites) dynacacheGCCacheBuster(cachebuster func(s string) bool) {
if cachebuster == nil {
return
}
shouldDelete := func(k, v any) bool {
var b bool
if s, ok := k.(string); ok {
b = cachebuster(s)
}
return b
}
h.MemCache.ClearMatching(nil, shouldDelete)
}
func (h *HugoSites) resolveAndClearStateForIdentities(
ctx context.Context,
l logg.LevelLogger,
cachebuster func(s string) bool, changes []identity.Identity,
) error {
h.Log.Debug().Log(logg.StringFunc(
func() string {
var sb strings.Builder
for _, change := range changes {
var key string
if kp, ok := change.(resource.Identifier); ok {
key = " " + kp.Key()
}
sb.WriteString(fmt.Sprintf("Direct dependencies of %q (%T%s) =>\n", change.IdentifierBase(), change, key))
seen := map[string]bool{
change.IdentifierBase(): true,
}
// Print the top level dependencies.
identity.WalkIdentitiesDeep(change, func(level int, id identity.Identity) bool {
if level > 1 {
return true
}
if !seen[id.IdentifierBase()] {
sb.WriteString(fmt.Sprintf(" %s%s\n", strings.Repeat(" ", level), id.IdentifierBase()))
}
seen[id.IdentifierBase()] = true
return false
})
}
return sb.String()
}),
)
for _, id := range changes {
if staler, ok := id.(resource.Staler); ok {
var msgDetail string
if p, ok := id.(*pageState); ok && p.File() != nil {
msgDetail = fmt.Sprintf(" (%s)", p.File().Filename())
}
h.Log.Trace(logg.StringFunc(func() string { return fmt.Sprintf("Marking stale: %s (%T)%s\n", id, id, msgDetail) }))
staler.MarkStale()
}
}
// The order matters here:
// 1. Handle the cache busters first, as those may produce identities for the page reset step.
// 2. Then reset the page outputs, which may mark some resources as stale.
// 3. Then GC the cache.
if cachebuster != nil {
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "gc dynacache cachebuster")
h.dynacacheGCCacheBuster(cachebuster)
return ll, nil
}); err != nil {
return err
}
}
// Drain the cache eviction stack.
evicted := h.Deps.MemCache.DrainEvictedIdentities()
if len(evicted) < 200 {
for _, c := range evicted {
changes = append(changes, c.Identity)
}
} else {
// Mass eviction, we might as well invalidate everything.
changes = []identity.Identity{identity.GenghisKhan}
}
// Remove duplicates
seen := make(map[identity.Identity]bool)
var n int
for _, id := range changes {
if !seen[id] {
seen[id] = true
changes[n] = id
n++
}
}
changes = changes[:n]
if h.pageTrees.treePagesFromTemplateAdapters.LenRaw() > 0 {
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "resolve content adapter change set").WithField("changes", len(changes))
checkedCount := 0
matchCount := 0
depsFinder := identity.NewFinder(identity.FinderConfig{})
h.pageTrees.treePagesFromTemplateAdapters.WalkPrefixRaw(doctree.LockTypeRead, "",
func(s string, n *pagesfromdata.PagesFromTemplate) (bool, error) {
for _, id := range changes {
checkedCount++
if r := depsFinder.Contains(id, n.DependencyManager, 2); r > identity.FinderNotFound {
n.MarkStale()
matchCount++
break
}
}
return false, nil
})
ll = ll.WithField("checked", checkedCount).WithField("matches", matchCount)
return ll, nil
}); err != nil {
return err
}
}
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
// changesLeft: The IDs that the pages is dependent on.
// changesRight: The IDs that the pages depend on.
ll := l.WithField("substep", "resolve page output change set").WithField("changes", len(changes))
checkedCount, matchCount, err := h.resolveAndResetDependententPageOutputs(ctx, changes)
ll = ll.WithField("checked", checkedCount).WithField("matches", matchCount)
return ll, err
}); err != nil {
return err
}
if err := loggers.TimeTrackfn(func() (logg.LevelLogger, error) {
ll := l.WithField("substep", "gc dynacache")
h.MemCache.ClearOnRebuild(changes...)
h.Log.Trace(logg.StringFunc(func() string {
var sb strings.Builder
sb.WriteString("dynacache keys:\n")
for _, key := range h.MemCache.Keys(nil) {
sb.WriteString(fmt.Sprintf(" %s\n", key))
}
return sb.String()
}))
return ll, nil
}); err != nil {
return err
}
return nil
}
// The left change set is the IDs that the pages is dependent on.
// The right change set is the IDs that the pages depend on.
func (h *HugoSites) resolveAndResetDependententPageOutputs(ctx context.Context, changes []identity.Identity) (int, int, error) {
if changes == nil {
return 0, 0, nil
}
// This can be shared (many of the same IDs are repeated).
depsFinder := identity.NewFinder(identity.FinderConfig{})
h.Log.Trace(logg.StringFunc(func() string {
var sb strings.Builder
sb.WriteString("resolve page dependencies: ")
for _, id := range changes {
sb.WriteString(fmt.Sprintf(" %T: %s|", id, id.IdentifierBase()))
}
return sb.String()
}))
var (
resetCounter atomic.Int64
checkedCounter atomic.Int64
)
resetPo := func(po *pageOutput, r identity.FinderResult) {
if po.pco != nil {
po.pco.Reset() // Will invalidate content cache.
}
po.renderState = 0
po.p.resourcesPublishInit = &sync.Once{}
if r == identity.FinderFoundOneOfMany {
// Will force a re-render even in fast render mode.
po.renderOnce = false
}
resetCounter.Add(1)
h.Log.Trace(logg.StringFunc(func() string {
p := po.p
return fmt.Sprintf("Resetting page output %s for %s for output %s\n", p.Kind(), p.Path(), po.f.Name)
}))
}
// This can be a relativeley expensive operations, so we do it in parallel.
g := rungroup.Run[*pageState](ctx, rungroup.Config[*pageState]{
NumWorkers: h.numWorkers,
Handle: func(ctx context.Context, p *pageState) error {
if !p.isRenderedAny() {
// This needs no reset, so no need to check it.
return nil
}
// First check the top level dependency manager.
for _, id := range changes {
checkedCounter.Add(1)
if r := depsFinder.Contains(id, p.dependencyManager, 2); r > identity.FinderFoundOneOfManyRepetition {
for _, po := range p.pageOutputs {
resetPo(po, r)
}
// Done.
return nil
}
}
// Then do a more fine grained reset for each output format.
OUTPUTS:
for _, po := range p.pageOutputs {
if !po.isRendered() {
continue
}
for _, id := range changes {
checkedCounter.Add(1)
if r := depsFinder.Contains(id, po.dependencyManagerOutput, 50); r > identity.FinderFoundOneOfManyRepetition {
resetPo(po, r)
continue OUTPUTS
}
}
}
return nil
},
})
h.withPage(func(s string, p *pageState) bool {
var needToCheck bool
for _, po := range p.pageOutputs {
if po.isRendered() {
needToCheck = true
break
}
}
if needToCheck {
g.Enqueue(p)
}
return false
})
err := g.Wait()
resetCount := int(resetCounter.Load())
checkedCount := int(checkedCounter.Load())
return checkedCount, resetCount, err
}
// Calculate and apply aggregate values to the page tree (e.g. dates, cascades).
func (sa *sitePagesAssembler) applyAggregates() error {
sectionPageCount := map[string]int{}
pw := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: sa.pageMap.treePages,
LockType: doctree.LockTypeRead,
WalkContext: &doctree.WalkContext[contentNodeI]{},
}
rw := pw.Extend()
rw.Tree = sa.pageMap.treeResources
sa.lastmod = time.Time{}
rebuild := sa.s.h.isRebuild()
pw.Handle = func(keyPage string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
pageBundle := n.(*pageState)
if pageBundle.Kind() == kinds.KindTerm {
// Delay this until they're created.
return false, nil
}
if pageBundle.IsPage() {
rootSection := pageBundle.Section()
sectionPageCount[rootSection]++
}
// Handle cascades first to get any default dates set.
var cascade map[page.PageMatcher]maps.Params
if keyPage == "" {
// Home page gets it's cascade from the site config.
cascade = sa.conf.Cascade.Config
if pageBundle.m.pageConfig.CascadeCompiled == nil {
// Pass the site cascade downwards.
pw.WalkContext.Data().Insert(keyPage, cascade)
}
} else {
_, data := pw.WalkContext.Data().LongestPrefix(keyPage)
if data != nil {
cascade = data.(map[page.PageMatcher]maps.Params)
}
}
if rebuild {
if (pageBundle.IsHome() || pageBundle.IsSection()) && pageBundle.m.setMetaPostCount > 0 {
oldDates := pageBundle.m.pageConfig.Dates
// We need to wait until after the walk to determine if any of the dates have changed.
pw.WalkContext.AddPostHook(
func() error {
if oldDates != pageBundle.m.pageConfig.Dates {
sa.assembleChanges.Add(pageBundle)
}
return nil
},
)
}
}
// Combine the cascade map with front matter.
if err := pageBundle.setMetaPost(cascade); err != nil {
return false, err
}
// We receive cascade values from above. If this leads to a change compared
// to the previous value, we need to mark the page and its dependencies as changed.
if rebuild && pageBundle.m.setMetaPostCascadeChanged {
sa.assembleChanges.Add(pageBundle)
}
const eventName = "dates"
if n.isContentNodeBranch() {
if pageBundle.m.pageConfig.CascadeCompiled != nil {
// Pass it down.
pw.WalkContext.Data().Insert(keyPage, pageBundle.m.pageConfig.CascadeCompiled)
}
wasZeroDates := pageBundle.m.pageConfig.Dates.IsAllDatesZero()
if wasZeroDates || pageBundle.IsHome() {
pw.WalkContext.AddEventListener(eventName, keyPage, func(e *doctree.Event[contentNodeI]) {
sp, ok := e.Source.(*pageState)
if !ok {
return
}
if wasZeroDates {
pageBundle.m.pageConfig.Dates.UpdateDateAndLastmodIfAfter(sp.m.pageConfig.Dates)
}
if pageBundle.IsHome() {
if pageBundle.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) {
pageBundle.s.lastmod = pageBundle.m.pageConfig.Dates.Lastmod
}
if sp.m.pageConfig.Dates.Lastmod.After(pageBundle.s.lastmod) {
pageBundle.s.lastmod = sp.m.pageConfig.Dates.Lastmod
}
}
})
}
}
// Send the date info up the tree.
pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: n, Path: keyPage, Name: eventName})
isBranch := n.isContentNodeBranch()
rw.Prefix = keyPage + "/"
rw.Handle = func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if isBranch {
ownerKey, _ := pw.Tree.LongestPrefix(resourceKey, true, nil)
if ownerKey != keyPage {
// Stop walking downwards, someone else owns this resource.
rw.SkipPrefix(ownerKey + "/")
return false, nil
}
}
rs := n.(*resourceSource)
if rs.isPage() {
pageResource := rs.r.(*pageState)
relPath := pageResource.m.pathInfo.BaseRel(pageBundle.m.pathInfo)
pageResource.m.resourcePath = relPath
var cascade map[page.PageMatcher]maps.Params
// Apply cascade (if set) to the page.
_, data := pw.WalkContext.Data().LongestPrefix(resourceKey)
if data != nil {
cascade = data.(map[page.PageMatcher]maps.Params)
}
if err := pageResource.setMetaPost(cascade); err != nil {
return false, err
}
}
return false, nil
}
return false, rw.Walk(sa.ctx)
}
if err := pw.Walk(sa.ctx); err != nil {
return err
}
if err := pw.WalkContext.HandleEventsAndHooks(); err != nil {
return err
}
if !sa.s.conf.C.IsMainSectionsSet() {
var mainSection string
var maxcount int
for section, counter := range sectionPageCount {
if section != "" && counter > maxcount {
mainSection = section
maxcount = counter
}
}
sa.s.conf.C.SetMainSections([]string{mainSection})
}
return nil
}
func (sa *sitePagesAssembler) applyAggregatesToTaxonomiesAndTerms() error {
walkContext := &doctree.WalkContext[contentNodeI]{}
handlePlural := func(key string) error {
var pw *doctree.NodeShiftTreeWalker[contentNodeI]
pw = &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: sa.pageMap.treePages,
Prefix: key, // We also want to include the root taxonomy nodes, so no trailing slash.
LockType: doctree.LockTypeRead,
WalkContext: walkContext,
Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
p := n.(*pageState)
if p.Kind() != kinds.KindTerm {
// The other kinds were handled in applyAggregates.
if p.m.pageConfig.CascadeCompiled != nil {
// Pass it down.
pw.WalkContext.Data().Insert(s, p.m.pageConfig.CascadeCompiled)
}
}
if p.Kind() != kinds.KindTerm && p.Kind() != kinds.KindTaxonomy {
// Already handled.
return false, nil
}
const eventName = "dates"
if p.Kind() == kinds.KindTerm {
var cascade map[page.PageMatcher]maps.Params
_, data := pw.WalkContext.Data().LongestPrefix(s)
if data != nil {
cascade = data.(map[page.PageMatcher]maps.Params)
}
if err := p.setMetaPost(cascade); err != nil {
return false, err
}
if !p.s.shouldBuild(p) {
sa.pageMap.treePages.Delete(s)
sa.pageMap.treeTaxonomyEntries.DeletePrefix(paths.AddTrailingSlash(s))
} else if err := sa.pageMap.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeRead,
paths.AddTrailingSlash(s),
func(ss string, wn *weightedContentNode) (bool, error) {
// Send the date info up the tree.
pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: wn.n, Path: ss, Name: eventName})
return false, nil
},
); err != nil {
return false, err
}
}
// Send the date info up the tree.
pw.WalkContext.SendEvent(&doctree.Event[contentNodeI]{Source: n, Path: s, Name: eventName})
if p.m.pageConfig.Dates.IsAllDatesZero() {
pw.WalkContext.AddEventListener(eventName, s, func(e *doctree.Event[contentNodeI]) {
sp, ok := e.Source.(*pageState)
if !ok {
return
}
p.m.pageConfig.Dates.UpdateDateAndLastmodIfAfter(sp.m.pageConfig.Dates)
})
}
return false, nil
},
}
if err := pw.Walk(sa.ctx); err != nil {
return err
}
return nil
}
for _, viewName := range sa.pageMap.cfg.taxonomyConfig.views {
if err := handlePlural(viewName.pluralTreeKey); err != nil {
return err
}
}
if err := walkContext.HandleEventsAndHooks(); err != nil {
return err
}
return nil
}
func (sa *sitePagesAssembler) assembleTermsAndTranslations() error {
var (
pages = sa.pageMap.treePages
entries = sa.pageMap.treeTaxonomyEntries
views = sa.pageMap.cfg.taxonomyConfig.views
)
lockType := doctree.LockTypeWrite
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: pages,
LockType: lockType,
Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
ps := n.(*pageState)
if ps.m.noLink() {
return false, nil
}
if sa.pageMap.cfg.taxonomyTermDisabled {
return false, nil
}
for _, viewName := range views {
vals := types.ToStringSlicePreserveString(getParam(ps, viewName.plural, false))
if vals == nil {
continue
}
w := getParamToLower(ps, viewName.plural+"_weight")
weight, err := cast.ToIntE(w)
if err != nil {
sa.Log.Warnf("Unable to convert taxonomy weight %#v to int for %q", w, n.Path())
// weight will equal zero, so let the flow continue
}
for i, v := range vals {
if v == "" {
continue
}
viewTermKey := "/" + viewName.plural + "/" + v
pi := sa.Site.Conf.PathParser().Parse(files.ComponentFolderContent, viewTermKey+"/_index.md")
term := pages.Get(pi.Base())
if term == nil {
m := &pageMeta{
term: v,
singular: viewName.singular,
s: sa.Site,
pathInfo: pi,
pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kinds.KindTerm,
},
},
}
n, pi, err := sa.h.newPage(m)
if err != nil {
return false, err
}
pages.InsertIntoValuesDimension(pi.Base(), n)
term = pages.Get(pi.Base())
} else {
m := term.(*pageState).m
m.term = v
m.singular = viewName.singular
}
if s == "" {
// Consider making this the real value.
s = "/"
}
key := pi.Base() + s
entries.Insert(key, &weightedContentNode{
weight: weight,
n: n,
term: &pageWithOrdinal{pageState: term.(*pageState), ordinal: i},
})
}
}
return false, nil
},
}
return w.Walk(sa.ctx)
}
func (sa *sitePagesAssembler) assembleResources() error {
pagesTree := sa.pageMap.treePages
resourcesTree := sa.pageMap.treeResources
lockType := doctree.LockTypeWrite
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: pagesTree,
LockType: lockType,
Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
ps := n.(*pageState)
// This is a little out of place, but is conveniently put here.
// Check if translationKey is set by user.
// This is to support the manual way of setting the translationKey in front matter.
if ps.m.pageConfig.TranslationKey != "" {
sa.s.h.translationKeyPages.Append(ps.m.pageConfig.TranslationKey, ps)
}
// Prepare resources for this page.
ps.shiftToOutputFormat(true, 0)
targetPaths := ps.targetPaths()
baseTarget := targetPaths.SubResourceBaseTarget
duplicateResourceFiles := true
if ps.m.pageConfig.ContentMediaType.IsMarkdown() {
duplicateResourceFiles = ps.s.ContentSpec.Converters.GetMarkupConfig().Goldmark.DuplicateResourceFiles
}
duplicateResourceFiles = duplicateResourceFiles || ps.s.Conf.IsMultihost()
err := sa.pageMap.forEachResourceInPage(
ps, lockType,
!duplicateResourceFiles,
func(resourceKey string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
rs := n.(*resourceSource)
if !match.Has(doctree.DimensionLanguage) {
// We got an alternative language version.
// Clone this and insert it into the tree.
rs = rs.clone()
resourcesTree.InsertIntoCurrentDimension(resourceKey, rs)
}
if rs.r != nil {
return false, nil
}
relPathOriginal := rs.path.Unnormalized().PathRel(ps.m.pathInfo.Unnormalized())
relPath := rs.path.BaseRel(ps.m.pathInfo)
var targetBasePaths []string
if ps.s.Conf.IsMultihost() {
baseTarget = targetPaths.SubResourceBaseLink
// In multihost we need to publish to the lang sub folder.
targetBasePaths = []string{ps.s.GetTargetLanguageBasePath()} // TODO(bep) we don't need this as a slice anymore.
}
if rs.rc != nil && rs.rc.Content.IsResourceValue() {
if rs.rc.Name == "" {
rs.rc.Name = relPathOriginal
}
r, err := ps.m.s.ResourceSpec.NewResourceWrapperFromResourceConfig(rs.rc)
if err != nil {
return false, err
}
rs.r = r
return false, nil
}
var mt media.Type
if rs.rc != nil {
mt = rs.rc.ContentMediaType
}
rd := resources.ResourceSourceDescriptor{
OpenReadSeekCloser: rs.opener,
Path: rs.path,
GroupIdentity: rs.path,
TargetPath: relPathOriginal, // Use the original path for the target path, so the links can be guessed.
TargetBasePaths: targetBasePaths,
BasePathRelPermalink: targetPaths.SubResourceBaseLink,
BasePathTargetPath: baseTarget,
NameNormalized: relPath,
NameOriginal: relPathOriginal,
MediaType: mt,
LazyPublish: !ps.m.pageConfig.Build.PublishResources,
}
if rs.rc != nil {
rc := rs.rc
rd.OpenReadSeekCloser = rc.Content.ValueAsOpenReadSeekCloser()
if rc.Name != "" {
rd.NameNormalized = rc.Name
rd.NameOriginal = rc.Name
}
if rc.Title != "" {
rd.Title = rc.Title
}
rd.Params = rc.Params
}
r, err := ps.m.s.ResourceSpec.NewResource(rd)
if err != nil {
return false, err
}
rs.r = r
return false, nil
},
)
return false, err
},
}
return w.Walk(sa.ctx)
}
func (sa *sitePagesAssembler) assemblePagesStep1(ctx context.Context) error {
if err := sa.addMissingTaxonomies(); err != nil {
return err
}
if err := sa.addMissingRootSections(); err != nil {
return err
}
if err := sa.addStandalonePages(); err != nil {
return err
}
if err := sa.applyAggregates(); err != nil {
return err
}
return nil
}
func (sa *sitePagesAssembler) assemblePagesStep2() error {
if err := sa.removeShouldNotBuild(); err != nil {
return err
}
if err := sa.assembleTermsAndTranslations(); err != nil {
return err
}
if err := sa.applyAggregatesToTaxonomiesAndTerms(); err != nil {
return err
}
return nil
}
func (sa *sitePagesAssembler) assemblePagesStepFinal() error {
if err := sa.assembleResources(); err != nil {
return err
}
return nil
}
// Remove any leftover node that we should not build for some reason (draft, expired, scheduled in the future).
// Note that for the home and section kinds we just disable the nodes to preserve the structure.
func (sa *sitePagesAssembler) removeShouldNotBuild() error {
s := sa.Site
var keys []string
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
LockType: doctree.LockTypeRead,
Tree: sa.pageMap.treePages,
Handle: func(key string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
p := n.(*pageState)
if !s.shouldBuild(p) {
switch p.Kind() {
case kinds.KindHome, kinds.KindSection, kinds.KindTaxonomy:
// We need to keep these for the structure, but disable
// them so they don't get listed/rendered.
(&p.m.pageConfig.Build).Disable()
default:
keys = append(keys, key)
}
}
return false, nil
},
}
if err := w.Walk(sa.ctx); err != nil {
return err
}
if len(keys) == 0 {
return nil
}
sa.pageMap.DeletePageAndResourcesBelow(keys...)
return nil
}
// // Create the fixed output pages, e.g. sitemap.xml, if not already there.
func (sa *sitePagesAssembler) addStandalonePages() error {
s := sa.Site
m := s.pageMap
tree := m.treePages
commit := tree.Lock(true)
defer commit()
addStandalone := func(key, kind string, f output.Format) {
if !s.Conf.IsMultihost() {
switch kind {
case kinds.KindSitemapIndex, kinds.KindRobotsTXT:
// Only one for all languages.
if s.languagei != 0 {
return
}
}
}
if !sa.Site.conf.IsKindEnabled(kind) || tree.Has(key) {
return
}
m := &pageMeta{
s: s,
pathInfo: s.Conf.PathParser().Parse(files.ComponentFolderContent, key+f.MediaType.FirstSuffix.FullSuffix),
pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kind,
},
},
standaloneOutputFormat: f,
}
p, _, _ := s.h.newPage(m)
tree.InsertIntoValuesDimension(key, p)
}
addStandalone("/404", kinds.KindStatus404, output.HTTPStatusHTMLFormat)
if s.conf.EnableRobotsTXT {
if m.i == 0 || s.Conf.IsMultihost() {
addStandalone("/_robots", kinds.KindRobotsTXT, output.RobotsTxtFormat)
}
}
sitemapEnabled := false
for _, s := range s.h.Sites {
if s.conf.IsKindEnabled(kinds.KindSitemap) {
sitemapEnabled = true
break
}
}
if sitemapEnabled {
of := output.SitemapFormat
if s.conf.Sitemap.Filename != "" {
of.BaseName = paths.Filename(s.conf.Sitemap.Filename)
}
addStandalone("/_sitemap", kinds.KindSitemap, of)
skipSitemapIndex := s.Conf.IsMultihost() || !(s.Conf.DefaultContentLanguageInSubdir() || s.Conf.IsMultilingual())
if !skipSitemapIndex {
of = output.SitemapIndexFormat
if s.conf.Sitemap.Filename != "" {
of.BaseName = paths.Filename(s.conf.Sitemap.Filename)
}
addStandalone("/_sitemapindex", kinds.KindSitemapIndex, of)
}
}
return nil
}
func (sa *sitePagesAssembler) addMissingRootSections() error {
var hasHome bool
// Add missing root sections.
seen := map[string]bool{}
var w *doctree.NodeShiftTreeWalker[contentNodeI]
w = &doctree.NodeShiftTreeWalker[contentNodeI]{
LockType: doctree.LockTypeWrite,
Tree: sa.pageMap.treePages,
Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
if n == nil {
panic("n is nil")
}
ps := n.(*pageState)
if ps.Lang() != sa.Lang() {
panic(fmt.Sprintf("lang mismatch: %q: %s != %s", s, ps.Lang(), sa.Lang()))
}
if s == "" {
hasHome = true
sa.home = ps
return false, nil
}
switch ps.Kind() {
case kinds.KindPage, kinds.KindSection:
// OK
default:
// Skip taxonomy nodes etc.
return false, nil
}
p := ps.m.pathInfo
section := p.Section()
if section == "" || seen[section] {
return false, nil
}
seen[section] = true
// Try to preserve the original casing if possible.
sectionUnnormalized := p.Unnormalized().Section()
pth := sa.s.Conf.PathParser().Parse(files.ComponentFolderContent, "/"+sectionUnnormalized+"/_index.md")
nn := w.Tree.Get(pth.Base())
if nn == nil {
m := &pageMeta{
s: sa.Site,
pathInfo: pth,
}
ps, pth, err := sa.h.newPage(m)
if err != nil {
return false, err
}
w.Tree.InsertIntoValuesDimension(pth.Base(), ps)
}
// /a/b, we don't need to walk deeper.
if strings.Count(s, "/") > 1 {
w.SkipPrefix(s + "/")
}
return false, nil
},
}
if err := w.Walk(sa.ctx); err != nil {
return err
}
if !hasHome {
p := sa.Site.Conf.PathParser().Parse(files.ComponentFolderContent, "/_index.md")
m := &pageMeta{
s: sa.Site,
pathInfo: p,
pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kinds.KindHome,
},
},
}
n, p, err := sa.h.newPage(m)
if err != nil {
return err
}
w.Tree.InsertIntoValuesDimensionWithLock(p.Base(), n)
sa.home = n
}
return nil
}
func (sa *sitePagesAssembler) addMissingTaxonomies() error {
if sa.pageMap.cfg.taxonomyDisabled && sa.pageMap.cfg.taxonomyTermDisabled {
return nil
}
tree := sa.pageMap.treePages
commit := tree.Lock(true)
defer commit()
for _, viewName := range sa.pageMap.cfg.taxonomyConfig.views {
key := viewName.pluralTreeKey
if v := tree.Get(key); v == nil {
m := &pageMeta{
s: sa.Site,
pathInfo: sa.Conf.PathParser().Parse(files.ComponentFolderContent, key+"/_index.md"),
pageMetaParams: &pageMetaParams{
pageConfig: &pagemeta.PageConfig{
Kind: kinds.KindTaxonomy,
},
},
singular: viewName.singular,
}
p, _, _ := sa.h.newPage(m)
tree.InsertIntoValuesDimension(key, p)
}
}
return nil
}
func (m *pageMap) CreateSiteTaxonomies(ctx context.Context) error {
m.s.taxonomies = make(page.TaxonomyList)
if m.cfg.taxonomyDisabled && m.cfg.taxonomyTermDisabled {
return nil
}
for _, viewName := range m.cfg.taxonomyConfig.views {
key := viewName.pluralTreeKey
m.s.taxonomies[viewName.plural] = make(page.Taxonomy)
w := &doctree.NodeShiftTreeWalker[contentNodeI]{
Tree: m.treePages,
Prefix: paths.AddTrailingSlash(key),
LockType: doctree.LockTypeRead,
Handle: func(s string, n contentNodeI, match doctree.DimensionFlag) (bool, error) {
p := n.(*pageState)
switch p.Kind() {
case kinds.KindTerm:
if !p.m.shouldList(true) {
return false, nil
}
taxonomy := m.s.taxonomies[viewName.plural]
if taxonomy == nil {
return true, fmt.Errorf("missing taxonomy: %s", viewName.plural)
}
if p.m.term == "" {
panic("term is empty")
}
k := strings.ToLower(p.m.term)
err := m.treeTaxonomyEntries.WalkPrefix(
doctree.LockTypeRead,
paths.AddTrailingSlash(s),
func(ss string, wn *weightedContentNode) (bool, error) {
taxonomy[k] = append(taxonomy[k], page.NewWeightedPage(wn.weight, wn.n.(page.Page), wn.term.Page()))
return false, nil
},
)
if err != nil {
return true, err
}
default:
return false, nil
}
return false, nil
},
}
if err := w.Walk(ctx); err != nil {
return err
}
}
for _, taxonomy := range m.s.taxonomies {
for _, v := range taxonomy {
v.Sort()
}
}
return nil
}
type viewName struct {
singular string // e.g. "category"
plural string // e.g. "categories"
pluralTreeKey string
}
func (v viewName) IsZero() bool {
return v.singular == ""
}