Handle resource changes when the resources is already evicted from cache

Also fix a logical flaw in the cache resizer that made it too aggressive. After this I haven't been able to reproduce #11988, but I need to look closer.

Closes #11973
Updates #11988
This commit is contained in:
Bjørn Erik Pedersen 2024-02-02 11:20:08 +01:00
parent 53f204310e
commit 609d798e34
9 changed files with 136 additions and 24 deletions

View file

@ -25,6 +25,7 @@ import (
"github.com/bep/lazycache"
"github.com/bep/logg"
"github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
@ -63,11 +64,26 @@ func New(opts Options) *Cache {
infol := opts.Log.InfoCommand("dynacache")
evictedIdentities := collections.NewStack[identity.Identity]()
onEvict := func(k, v any) {
if !opts.Running {
return
}
identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool {
evictedIdentities.Push(id)
return false
})
resource.MarkStale(v)
}
c := &Cache{
partitions: make(map[string]PartitionManager),
opts: opts,
stats: stats,
infol: infol,
partitions: make(map[string]PartitionManager),
onEvict: onEvict,
evictedIdentities: evictedIdentities,
opts: opts,
stats: stats,
infol: infol,
}
c.stop = c.start()
@ -106,14 +122,23 @@ type Cache struct {
mu sync.RWMutex
partitions map[string]PartitionManager
opts Options
infol logg.LevelLogger
onEvict func(k, v any)
evictedIdentities *collections.Stack[identity.Identity]
opts Options
infol logg.LevelLogger
stats *stats
stopOnce sync.Once
stop func()
}
// DrainEvictedIdentities drains the evicted identities from the cache.
func (c *Cache) DrainEvictedIdentities() []identity.Identity {
return c.evictedIdentities.Drain()
}
// ClearMatching clears all partition for which the predicate returns true.
func (c *Cache) ClearMatching(predicate func(k, v any) bool) {
g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
@ -318,9 +343,13 @@ func GetOrCreatePartition[K comparable, V any](c *Cache, name string, opts Optio
const numberOfPartitionsEstimate = 10
maxSize := opts.CalculateMaxSize(c.opts.MaxSize / numberOfPartitionsEstimate)
onEvict := func(k K, v V) {
c.onEvict(k, v)
}
// Create a new partition and cache it.
partition := &Partition[K, V]{
c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize}),
c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize, OnEvict: onEvict}),
maxSize: maxSize,
trace: c.opts.Log.Logger().WithLevel(logg.LevelTrace).WithField("partition", name),
opts: opts,
@ -445,7 +474,6 @@ func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
},
),
)
resource.MarkStale(v)
return true
}
return false
@ -483,6 +511,10 @@ func (p *Partition[K, V]) adjustMaxSize(newMaxSize int) int {
if newMaxSize < minMaxSize {
newMaxSize = minMaxSize
}
oldMaxSize := p.maxSize
if newMaxSize == oldMaxSize {
return 0
}
p.maxSize = newMaxSize
// fmt.Println("Adjusting max size of partition from", oldMaxSize, "to", newMaxSize)
return p.c.Resize(newMaxSize)
@ -535,7 +567,7 @@ type stats struct {
func (s *stats) adjustCurrentMaxSize() bool {
newCurrentMaxSize := int(math.Floor(float64(s.opts.MaxSize) * s.adjustmentFactor))
if newCurrentMaxSize < s.opts.MaxSize {
if newCurrentMaxSize < s.opts.MinMaxSize {
newCurrentMaxSize = int(s.opts.MinMaxSize)
}
changed := newCurrentMaxSize != s.currentMaxSize

View file

@ -949,9 +949,10 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error
cfg.Set("environment", c.r.environment)
cfg.Set("internal", maps.Params{
"running": running,
"watch": watch,
"verbose": c.r.isVerbose(),
"running": running,
"watch": watch,
"verbose": c.r.isVerbose(),
"fastRenderMode": c.fastRenderMode,
})
conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, cfg))

View file

@ -0,0 +1,67 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package collections
import "sync"
// Stack is a simple LIFO stack that is safe for concurrent use.
type Stack[T any] struct {
items []T
zero T
mu sync.RWMutex
}
func NewStack[T any]() *Stack[T] {
return &Stack[T]{}
}
func (s *Stack[T]) Push(item T) {
s.mu.Lock()
defer s.mu.Unlock()
s.items = append(s.items, item)
}
func (s *Stack[T]) Pop() (T, bool) {
s.mu.Lock()
defer s.mu.Unlock()
if len(s.items) == 0 {
return s.zero, false
}
item := s.items[len(s.items)-1]
s.items = s.items[:len(s.items)-1]
return item, true
}
func (s *Stack[T]) Peek() (T, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
if len(s.items) == 0 {
return s.zero, false
}
return s.items[len(s.items)-1], true
}
func (s *Stack[T]) Len() int {
s.mu.RLock()
defer s.mu.RUnlock()
return len(s.items)
}
func (s *Stack[T]) Drain() []T {
s.mu.Lock()
defer s.mu.Unlock()
items := s.items
s.items = nil
return items
}

View file

@ -65,6 +65,7 @@ type InternalConfig struct {
Verbose bool
Clock string
Watch bool
FastRenderMode bool
LiveReloadPort int
}

View file

@ -73,6 +73,10 @@ func (c ConfigLanguage) IsMultihost() bool {
return c.m.IsMultihost
}
func (c ConfigLanguage) FastRenderMode() bool {
return c.config.Internal.FastRenderMode
}
func (c ConfigLanguage) IsMultiLingual() bool {
return len(c.m.Languages) > 1
}

View file

@ -57,6 +57,7 @@ type AllProvider interface {
BuildDrafts() bool
Running() bool
Watching() bool
FastRenderMode() bool
PrintUnusedTemplates() bool
EnableMissingTranslationPlaceholders() bool
TemplateMetrics() bool

View file

@ -1018,14 +1018,6 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
b = cachebuster(s)
}
if b {
identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool {
// Add them to the change set so we can reset any page that depends on them.
changes = append(changes, id)
return false
})
}
return b
}
@ -1037,6 +1029,15 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
}
}
// Drain the the cache eviction stack.
evicted := h.Deps.MemCache.DrainEvictedIdentities()
if len(evicted) < 200 {
changes = append(changes, evicted...)
} else {
// Mass eviction, we might as well invalidate everything.
changes = []identity.Identity{identity.GenghisKhan}
}
// Remove duplicates
seen := make(map[identity.Identity]bool)
var n int

View file

@ -99,6 +99,8 @@ type HugoSites struct {
*fatalErrorHandler
*buildCounters
// Tracks invocations of the Build method.
buildCounter atomic.Uint64
}
// ShouldSkipFileChangeEvent allows skipping filesystem event early before
@ -420,10 +422,9 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool {
return false
}
fastRenderMode := cfg.RecentlyVisited.Len() > 0
fastRenderMode := p.s.Conf.FastRenderMode()
if !fastRenderMode {
// Not in fast render mode or first time render.
if !fastRenderMode || p.s.h.buildCounter.Load() == 0 {
return shouldRender
}

View file

@ -57,6 +57,9 @@ import (
func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
infol := h.Log.InfoCommand("build")
defer loggers.TimeTrackf(infol, time.Now(), nil, "")
defer func() {
h.buildCounter.Add(1)
}()
if h.Deps == nil {
panic("must have deps")
@ -769,8 +772,9 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
}
case files.ComponentFolderAssets:
logger.Println("Asset changed", pathInfo.Path())
r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base()))
var hasID bool
r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base()))
identity.WalkIdentitiesShallow(r, func(level int, rid identity.Identity) bool {
hasID = true
changes = append(changes, rid)