mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-07 20:30:36 -05:00
Handle resource changes when the resources is already evicted from cache
Also fix a logical flaw in the cache resizer that made it too aggressive. After this I haven't been able to reproduce #11988, but I need to look closer. Closes #11973 Updates #11988
This commit is contained in:
parent
53f204310e
commit
609d798e34
9 changed files with 136 additions and 24 deletions
50
cache/dynacache/dynacache.go
vendored
50
cache/dynacache/dynacache.go
vendored
|
@ -25,6 +25,7 @@ import (
|
|||
|
||||
"github.com/bep/lazycache"
|
||||
"github.com/bep/logg"
|
||||
"github.com/gohugoio/hugo/common/collections"
|
||||
"github.com/gohugoio/hugo/common/herrors"
|
||||
"github.com/gohugoio/hugo/common/loggers"
|
||||
"github.com/gohugoio/hugo/common/paths"
|
||||
|
@ -63,11 +64,26 @@ func New(opts Options) *Cache {
|
|||
|
||||
infol := opts.Log.InfoCommand("dynacache")
|
||||
|
||||
evictedIdentities := collections.NewStack[identity.Identity]()
|
||||
|
||||
onEvict := func(k, v any) {
|
||||
if !opts.Running {
|
||||
return
|
||||
}
|
||||
identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool {
|
||||
evictedIdentities.Push(id)
|
||||
return false
|
||||
})
|
||||
resource.MarkStale(v)
|
||||
}
|
||||
|
||||
c := &Cache{
|
||||
partitions: make(map[string]PartitionManager),
|
||||
opts: opts,
|
||||
stats: stats,
|
||||
infol: infol,
|
||||
partitions: make(map[string]PartitionManager),
|
||||
onEvict: onEvict,
|
||||
evictedIdentities: evictedIdentities,
|
||||
opts: opts,
|
||||
stats: stats,
|
||||
infol: infol,
|
||||
}
|
||||
|
||||
c.stop = c.start()
|
||||
|
@ -106,14 +122,23 @@ type Cache struct {
|
|||
mu sync.RWMutex
|
||||
|
||||
partitions map[string]PartitionManager
|
||||
opts Options
|
||||
infol logg.LevelLogger
|
||||
|
||||
onEvict func(k, v any)
|
||||
evictedIdentities *collections.Stack[identity.Identity]
|
||||
|
||||
opts Options
|
||||
infol logg.LevelLogger
|
||||
|
||||
stats *stats
|
||||
stopOnce sync.Once
|
||||
stop func()
|
||||
}
|
||||
|
||||
// DrainEvictedIdentities drains the evicted identities from the cache.
|
||||
func (c *Cache) DrainEvictedIdentities() []identity.Identity {
|
||||
return c.evictedIdentities.Drain()
|
||||
}
|
||||
|
||||
// ClearMatching clears all partition for which the predicate returns true.
|
||||
func (c *Cache) ClearMatching(predicate func(k, v any) bool) {
|
||||
g := rungroup.Run[PartitionManager](context.Background(), rungroup.Config[PartitionManager]{
|
||||
|
@ -318,9 +343,13 @@ func GetOrCreatePartition[K comparable, V any](c *Cache, name string, opts Optio
|
|||
const numberOfPartitionsEstimate = 10
|
||||
maxSize := opts.CalculateMaxSize(c.opts.MaxSize / numberOfPartitionsEstimate)
|
||||
|
||||
onEvict := func(k K, v V) {
|
||||
c.onEvict(k, v)
|
||||
}
|
||||
|
||||
// Create a new partition and cache it.
|
||||
partition := &Partition[K, V]{
|
||||
c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize}),
|
||||
c: lazycache.New(lazycache.Options[K, V]{MaxEntries: maxSize, OnEvict: onEvict}),
|
||||
maxSize: maxSize,
|
||||
trace: c.opts.Log.Logger().WithLevel(logg.LevelTrace).WithField("partition", name),
|
||||
opts: opts,
|
||||
|
@ -445,7 +474,6 @@ func (p *Partition[K, V]) clearOnRebuild(changeset ...identity.Identity) {
|
|||
},
|
||||
),
|
||||
)
|
||||
resource.MarkStale(v)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
@ -483,6 +511,10 @@ func (p *Partition[K, V]) adjustMaxSize(newMaxSize int) int {
|
|||
if newMaxSize < minMaxSize {
|
||||
newMaxSize = minMaxSize
|
||||
}
|
||||
oldMaxSize := p.maxSize
|
||||
if newMaxSize == oldMaxSize {
|
||||
return 0
|
||||
}
|
||||
p.maxSize = newMaxSize
|
||||
// fmt.Println("Adjusting max size of partition from", oldMaxSize, "to", newMaxSize)
|
||||
return p.c.Resize(newMaxSize)
|
||||
|
@ -535,7 +567,7 @@ type stats struct {
|
|||
func (s *stats) adjustCurrentMaxSize() bool {
|
||||
newCurrentMaxSize := int(math.Floor(float64(s.opts.MaxSize) * s.adjustmentFactor))
|
||||
|
||||
if newCurrentMaxSize < s.opts.MaxSize {
|
||||
if newCurrentMaxSize < s.opts.MinMaxSize {
|
||||
newCurrentMaxSize = int(s.opts.MinMaxSize)
|
||||
}
|
||||
changed := newCurrentMaxSize != s.currentMaxSize
|
||||
|
|
|
@ -949,9 +949,10 @@ func (c *hugoBuilder) loadConfig(cd *simplecobra.Commandeer, running bool) error
|
|||
cfg.Set("environment", c.r.environment)
|
||||
|
||||
cfg.Set("internal", maps.Params{
|
||||
"running": running,
|
||||
"watch": watch,
|
||||
"verbose": c.r.isVerbose(),
|
||||
"running": running,
|
||||
"watch": watch,
|
||||
"verbose": c.r.isVerbose(),
|
||||
"fastRenderMode": c.fastRenderMode,
|
||||
})
|
||||
|
||||
conf, err := c.r.ConfigFromProvider(c.r.configVersionID.Load(), flagsToCfg(cd, cfg))
|
||||
|
|
67
common/collections/stack.go
Normal file
67
common/collections/stack.go
Normal file
|
@ -0,0 +1,67 @@
|
|||
// Copyright 2024 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package collections
|
||||
|
||||
import "sync"
|
||||
|
||||
// Stack is a simple LIFO stack that is safe for concurrent use.
|
||||
type Stack[T any] struct {
|
||||
items []T
|
||||
zero T
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func NewStack[T any]() *Stack[T] {
|
||||
return &Stack[T]{}
|
||||
}
|
||||
|
||||
func (s *Stack[T]) Push(item T) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.items = append(s.items, item)
|
||||
}
|
||||
|
||||
func (s *Stack[T]) Pop() (T, bool) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(s.items) == 0 {
|
||||
return s.zero, false
|
||||
}
|
||||
item := s.items[len(s.items)-1]
|
||||
s.items = s.items[:len(s.items)-1]
|
||||
return item, true
|
||||
}
|
||||
|
||||
func (s *Stack[T]) Peek() (T, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if len(s.items) == 0 {
|
||||
return s.zero, false
|
||||
}
|
||||
return s.items[len(s.items)-1], true
|
||||
}
|
||||
|
||||
func (s *Stack[T]) Len() int {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return len(s.items)
|
||||
}
|
||||
|
||||
func (s *Stack[T]) Drain() []T {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
items := s.items
|
||||
s.items = nil
|
||||
return items
|
||||
}
|
|
@ -65,6 +65,7 @@ type InternalConfig struct {
|
|||
Verbose bool
|
||||
Clock string
|
||||
Watch bool
|
||||
FastRenderMode bool
|
||||
LiveReloadPort int
|
||||
}
|
||||
|
||||
|
|
|
@ -73,6 +73,10 @@ func (c ConfigLanguage) IsMultihost() bool {
|
|||
return c.m.IsMultihost
|
||||
}
|
||||
|
||||
func (c ConfigLanguage) FastRenderMode() bool {
|
||||
return c.config.Internal.FastRenderMode
|
||||
}
|
||||
|
||||
func (c ConfigLanguage) IsMultiLingual() bool {
|
||||
return len(c.m.Languages) > 1
|
||||
}
|
||||
|
|
|
@ -57,6 +57,7 @@ type AllProvider interface {
|
|||
BuildDrafts() bool
|
||||
Running() bool
|
||||
Watching() bool
|
||||
FastRenderMode() bool
|
||||
PrintUnusedTemplates() bool
|
||||
EnableMissingTranslationPlaceholders() bool
|
||||
TemplateMetrics() bool
|
||||
|
|
|
@ -1018,14 +1018,6 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
|
|||
b = cachebuster(s)
|
||||
}
|
||||
|
||||
if b {
|
||||
identity.WalkIdentitiesShallow(v, func(level int, id identity.Identity) bool {
|
||||
// Add them to the change set so we can reset any page that depends on them.
|
||||
changes = append(changes, id)
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
|
@ -1037,6 +1029,15 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
|
|||
}
|
||||
}
|
||||
|
||||
// Drain the the cache eviction stack.
|
||||
evicted := h.Deps.MemCache.DrainEvictedIdentities()
|
||||
if len(evicted) < 200 {
|
||||
changes = append(changes, evicted...)
|
||||
} else {
|
||||
// Mass eviction, we might as well invalidate everything.
|
||||
changes = []identity.Identity{identity.GenghisKhan}
|
||||
}
|
||||
|
||||
// Remove duplicates
|
||||
seen := make(map[identity.Identity]bool)
|
||||
var n int
|
||||
|
|
|
@ -99,6 +99,8 @@ type HugoSites struct {
|
|||
|
||||
*fatalErrorHandler
|
||||
*buildCounters
|
||||
// Tracks invocations of the Build method.
|
||||
buildCounter atomic.Uint64
|
||||
}
|
||||
|
||||
// ShouldSkipFileChangeEvent allows skipping filesystem event early before
|
||||
|
@ -420,10 +422,9 @@ func (cfg *BuildCfg) shouldRender(p *pageState) bool {
|
|||
return false
|
||||
}
|
||||
|
||||
fastRenderMode := cfg.RecentlyVisited.Len() > 0
|
||||
fastRenderMode := p.s.Conf.FastRenderMode()
|
||||
|
||||
if !fastRenderMode {
|
||||
// Not in fast render mode or first time render.
|
||||
if !fastRenderMode || p.s.h.buildCounter.Load() == 0 {
|
||||
return shouldRender
|
||||
}
|
||||
|
||||
|
|
|
@ -57,6 +57,9 @@ import (
|
|||
func (h *HugoSites) Build(config BuildCfg, events ...fsnotify.Event) error {
|
||||
infol := h.Log.InfoCommand("build")
|
||||
defer loggers.TimeTrackf(infol, time.Now(), nil, "")
|
||||
defer func() {
|
||||
h.buildCounter.Add(1)
|
||||
}()
|
||||
|
||||
if h.Deps == nil {
|
||||
panic("must have deps")
|
||||
|
@ -769,8 +772,9 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
|
|||
}
|
||||
case files.ComponentFolderAssets:
|
||||
logger.Println("Asset changed", pathInfo.Path())
|
||||
r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base()))
|
||||
|
||||
var hasID bool
|
||||
r, _ := h.ResourceSpec.ResourceCache.Get(context.Background(), dynacache.CleanKey(pathInfo.Base()))
|
||||
identity.WalkIdentitiesShallow(r, func(level int, rid identity.Identity) bool {
|
||||
hasID = true
|
||||
changes = append(changes, rid)
|
||||
|
|
Loading…
Reference in a new issue