diff --git a/cache/dynacache/dynacache.go b/cache/dynacache/dynacache.go index bb3f7b098..85b360138 100644 --- a/cache/dynacache/dynacache.go +++ b/cache/dynacache/dynacache.go @@ -25,6 +25,7 @@ import ( "github.com/bep/lazycache" "github.com/bep/logg" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/common/paths" @@ -63,11 +64,26 @@ func New(opts Options) *Cache { infol := opts.Log.InfoCommand("dynacache") + evictedIdentities := collections.NewStack[identity.Identity]() + + onEvict := func(k, v any) { + if !opts.Running { + return + } + identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool { + evictedIdentities.Push(id) + return false + }) + resource.MarkStale(v) + } + c := &Cache{ - partitions: make(map[string]PartitionManager), - opts: opts, - stats: stats, - infol: infol, + partitions: make(map[string]PartitionManager), + onEvict: onEvict, + evictedIdentities: evictedIdentities, + opts: opts, + stats: stats, + infol: infol, } c.stop = c.start() @@ -106,14 +122,23 @@ type Cache struct { mu sync.RWMutex partitions map[string]PartitionManager - opts Options - infol logg.LevelLogger + + onEvict func(k, v any) + evictedIdentities *collections.Stack[identity.Identity] + + opts Options + infol logg.LevelLogger stats *stats stopOnce sync.Once stop func() } +// DrainEvictedIdentities drains the evicted identities from the cache. +func (c *Cache) DrainEvictedIdentities() []identity.Identity { + return c.evictedIdentities.Drain() +} + // ClearMatching clears all partition for which the predicate returns true. func (c *Cache) ClearMatching(predicate func(k, v any) bool) { g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{ @@ -318,9 +343,13 @@ func GetOrCreatePartition[K comparable, V any](c *Cache, name string, opts Optio const numberOfPartitionsEstimate = 10 maxSize := opts.CalculateMaxSize(c.opts.MaxSize / numberOfPartitionsEstimate) + onEvict := func(k K, v V) { + c.onEvict(k, v) + } + // Create a new partition and cache it. partition := &Partition[K, V]{ - c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize}), + c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize, OnEvict: onEvict}), maxSize: maxSize, trace: c.opts.Log.Logger().WithLevel(logg.LevelTrace).WithField("partition", name), opts: opts, @@ -445,7 +474,6 @@ func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) { }, ), ) - resource.MarkStale(v) return true } return false @@ -483,6 +511,10 @@ func (p *Partition[K, V]) adjustMaxSize(newMaxSize int) int { if newMaxSize < minMaxSize { newMaxSize = minMaxSize } + oldMaxSize := p.maxSize + if newMaxSize == oldMaxSize { + return 0 + } p.maxSize = newMaxSize // fmt.Println("Adjusting max size of partition from", oldMaxSize, "to", newMaxSize) return p.c.Resize(newMaxSize) @@ -535,7 +567,7 @@ type stats struct { func (s *stats) adjustCurrentMaxSize() bool { newCurrentMaxSize := int(math.Floor(float64(s.opts.MaxSize) * s.adjustmentFactor)) - if newCurrentMaxSize < s.opts.MaxSize { + if newCurrentMaxSize < s.opts.MinMaxSize { newCurrentMaxSize = int(s.opts.MinMaxSize) } changed := newCurrentMaxSize != s.currentMaxSize diff --git a/commands/hugobuilder.go b/commands/hugobuilder.go index ddc92129c..190c12f59 100644 --- a/commands/hugobuilder.go +++ b/commands/hugobuilder.go @@ -949,9 +949,10 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error cfg.Set("environment", c.r.environment) cfg.Set("internal", maps.Params{ - "running": running, - "watch": watch, - "verbose": c.r.isVerbose(), + "running": running, + "watch": watch, + "verbose": c.r.isVerbose(), + "fastRenderMode": c.fastRenderMode, }) conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, cfg)) diff --git a/common/collections/stack.go b/common/collections/stack.go new file mode 100644 index 000000000..0f1581626 --- /dev/null +++ b/common/collections/stack.go @@ -0,0 +1,67 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collections + +import "sync" + +// Stack is a simple LIFO stack that is safe for concurrent use. +type Stack[T any] struct { + items []T + zero T + mu sync.RWMutex +} + +func NewStack[T any]() *Stack[T] { + return &Stack[T]{} +} + +func (s *Stack[T]) Push(item T) { + s.mu.Lock() + defer s.mu.Unlock() + s.items = append(s.items, item) +} + +func (s *Stack[T]) Pop() (T, bool) { + s.mu.Lock() + defer s.mu.Unlock() + if len(s.items) == 0 { + return s.zero, false + } + item := s.items[len(s.items)-1] + s.items = s.items[:len(s.items)-1] + return item, true +} + +func (s *Stack[T]) Peek() (T, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + if len(s.items) == 0 { + return s.zero, false + } + return s.items[len(s.items)-1], true +} + +func (s *Stack[T]) Len() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.items) +} + +func (s *Stack[T]) Drain() []T { + s.mu.Lock() + defer s.mu.Unlock() + items := s.items + s.items = nil + return items +} diff --git a/config/allconfig/allconfig.go b/config/allconfig/allconfig.go index 7052f0abd..4771f5a72 100644 --- a/config/allconfig/allconfig.go +++ b/config/allconfig/allconfig.go @@ -65,6 +65,7 @@ type InternalConfig struct { Verbose bool Clock string Watch bool + FastRenderMode bool LiveReloadPort int } diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index 0b4c74278..2cc80caa8 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -73,6 +73,10 @@ func (c ConfigLanguage) IsMultihost() bool { return c.m.IsMultihost } +func (c ConfigLanguage) FastRenderMode() bool { + return c.config.Internal.FastRenderMode +} + func (c ConfigLanguage) IsMultiLingual() bool { return len(c.m.Languages) > 1 } diff --git a/config/configProvider.go b/config/configProvider.go index 21d832f17..38dde3bb4 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -57,6 +57,7 @@ type AllProvider interface { BuildDrafts() bool Running() bool Watching() bool + FastRenderMode() bool PrintUnusedTemplates() bool EnableMissingTranslationPlaceholders() bool TemplateMetrics() bool diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index 1f4cd0880..570e45bd4 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -1018,14 +1018,6 @@ func (h *HugoSites) resolveAndClearStateForIdentities( b = cachebuster(s) } - if b { - identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool { - // Add them to the change set so we can reset any page that depends on them. - changes = append(changes, id) - return false - }) - } - return b } @@ -1037,6 +1029,15 @@ func (h *HugoSites) resolveAndClearStateForIdentities( } } + // Drain the the cache eviction stack. + evicted := h.Deps.MemCache.DrainEvictedIdentities() + if len(evicted) < 200 { + changes = append(changes, evicted...) + } else { + // Mass eviction, we might as well invalidate everything. + changes = []identity.Identity{identity.GenghisKhan} + } + // Remove duplicates seen := make(map[identity.Identity]bool) var n int diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index ef67b1059..24ff1077f 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -99,6 +99,8 @@ type HugoSites struct { *fatalErrorHandler *buildCounters + // Tracks invocations of the Build method. + buildCounter atomic.Uint64 } // ShouldSkipFileChangeEvent allows skipping filesystem event early before @@ -420,10 +422,9 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool { return false } - fastRenderMode := cfg.RecentlyVisited.Len() > 0 + fastRenderMode := p.s.Conf.FastRenderMode() - if !fastRenderMode { - // Not in fast render mode or first time render. + if !fastRenderMode || p.s.h.buildCounter.Load() == 0 { return shouldRender } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index a15e15504..4b22c1956 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -57,6 +57,9 @@ import ( func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error { infol := h.Log.InfoCommand("build") defer loggers.TimeTrackf(infol, time.Now(), nil, "") + defer func() { + h.buildCounter.Add(1) + }() if h.Deps == nil { panic("must have deps") @@ -769,8 +772,9 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf } case files.ComponentFolderAssets: logger.Println("Asset changed", pathInfo.Path()) - r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base())) + var hasID bool + r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base())) identity.WalkIdentitiesShallow(r, func(level int, rid identity.Identity) bool { hasID = true changes = append(changes, rid)