Add cache busting config to support Tailwind 3

Fixes #10974

commands/commandeer.go

@@ -162,6 +162,7 @@ func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commo
 				Fs:          fs.Source,
 				Filename:    r.cfgFile,
 				ConfigDir:   r.cfgDir,
+				Logger:      r.logger,
 				Environment: r.environment,
 			},
 		)
@@ -222,6 +223,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
 				Filename:    r.cfgFile,
 				ConfigDir:   r.cfgDir,
 				Environment: r.environment,
+				Logger:      r.logger,
 			},
 		)
 		if err != nil {

config/allconfig/allconfig.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/cache/filecache"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/urls"
 	"github.com/gohugoio/hugo/config"
@@ -184,7 +185,7 @@ type Config struct {
 }
 
 type configCompiler interface {
-	CompileConfig() error
+	CompileConfig(logger loggers.Logger) error
 }
 
 func (c Config) cloneForLang() *Config {
@@ -209,7 +210,7 @@ func (c Config) cloneForLang() *Config {
 	return &x
 }
 
-func (c *Config) CompileConfig() error {
+func (c *Config) CompileConfig(logger loggers.Logger) error {
 	var transientErr error
 	s := c.Timeout
 	if _, err := strconv.Atoi(s); err == nil {
@@ -328,7 +329,7 @@ func (c *Config) CompileConfig() error {
 
 	for _, s := range allDecoderSetups {
 		if getCompiler := s.getCompiler; getCompiler != nil {
-			if err := getCompiler(c).CompileConfig(); err != nil {
+			if err := getCompiler(c).CompileConfig(logger); err != nil {
 				return err
 			}
 		}
@@ -668,8 +669,8 @@ func (c Configs) GetByLang(lang string) config.AllProvider {
 	return nil
 }
 
-// FromLoadConfigResult creates a new Config from res.
+// fromLoadConfigResult creates a new Config from res.
-func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, error) {
+func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadConfigResult) (*Configs, error) {
 	if !res.Cfg.IsSet("languages") {
 		// We need at least one
 		lang := res.Cfg.GetString("defaultContentLanguage")
@@ -690,7 +691,7 @@ func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, e
 	languagesConfig := cfg.GetStringMap("languages")
 	var isMultiHost bool
 
-	if err := all.CompileConfig(); err != nil {
+	if err := all.CompileConfig(logger); err != nil {
 		return nil, err
 	}
 
@@ -769,7 +770,7 @@ func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, e
 			if err := decodeConfigFromParams(fs, bcfg, mergedConfig, clone, differentRootKeys); err != nil {
 				return nil, fmt.Errorf("failed to decode config for language %q: %w", k, err)
 			}
-			if err := clone.CompileConfig(); err != nil {
+			if err := clone.CompileConfig(logger); err != nil {
 				return nil, err
 			}
 			langConfigMap[k] = clone

config/allconfig/alldecoders.go

@@ -92,6 +92,9 @@ var allDecoderSetups = map[string]decodeWeight{
 			p.c.Build = config.DecodeBuildConfig(p.p)
 			return nil
 		},
+		getCompiler: func(c *Config) configCompiler {
+			return &c.Build
+		},
 	},
 	"frontmatter": {
 		key: "frontmatter",

config/allconfig/load.go

@@ -44,6 +44,10 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
 		d.Environ = os.Environ()
 	}
 
+	if d.Logger == nil {
+		d.Logger = loggers.NewErrorLogger()
+	}
+
 	l := &configLoader{ConfigSourceDescriptor: d, cfg: config.New()}
 	// Make sure we always do this, even in error situations,
 	// as we have commands (e.g. "hugo mod init") that will
@@ -54,7 +58,7 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
 		return nil, fmt.Errorf("failed to load config: %w", err)
 	}
 
-	configs, err := FromLoadConfigResult(d.Fs, res)
+	configs, err := fromLoadConfigResult(d.Fs, d.Logger, res)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create config from result: %w", err)
 	}
@@ -67,7 +71,7 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
 	if len(l.ModulesConfigFiles) > 0 {
 		// Config merged in from modules.
 		// Re-read the config.
-		configs, err = FromLoadConfigResult(d.Fs, res)
+		configs, err = fromLoadConfigResult(d.Fs, d.Logger, res)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create config from modules config: %w", err)
 		}

config/commonConfig.go

@@ -15,12 +15,14 @@ package config
 
 import (
 	"fmt"
+	"regexp"
 	"sort"
 	"strings"
 
+	"github.com/gobwas/glob"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/types"
 
-	"github.com/gobwas/glob"
 	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cast"
@@ -77,9 +79,29 @@ type LoadConfigResult struct {
 	BaseConfig  BaseConfig
 }
 
-var DefaultBuild = BuildConfig{
+var defaultBuild = BuildConfig{
 	UseResourceCacheWhen: "fallback",
 	WriteStats:           false,
+
+	CacheBusters: []CacheBuster{
+		{
+			Source: `assets/.*\.(js|ts|jsx|tsx)`,
+			Target: `(js|scripts|javascript)`,
+		},
+		{
+			Source: `assets/.*\.(css|sass|scss)$`,
+			Target: cssTargetCachebusterRe,
+		},
+		{
+			Source: `(postcss|tailwind)\.config\.js`,
+			Target: cssTargetCachebusterRe,
+		},
+		// This is deliberatly coarse grained; it will cache bust resources with "json" in the cache key when js files changes, which is good.
+		{
+			Source: `assets/.*\.(.*)$`,
+			Target: `$1`,
+		},
+	},
 }
 
 // BuildConfig holds some build related configuration.
@@ -93,6 +115,14 @@ type BuildConfig struct {
 	// Can be used to toggle off writing of the intellinsense /assets/jsconfig.js
 	// file.
 	NoJSConfigInAssets bool
+
+	// Can used to control how the resource cache gets evicted on rebuilds.
+	CacheBusters []CacheBuster
+}
+
+func (b BuildConfig) clone() BuildConfig {
+	b.CacheBusters = append([]CacheBuster{}, b.CacheBusters...)
+	return b
 }
 
 func (b BuildConfig) UseResourceCache(err error) bool {
@@ -107,16 +137,47 @@ func (b BuildConfig) UseResourceCache(err error) bool {
 	return true
 }
 
+// MatchCacheBuster returns the cache buster for the given path p, nil if none.
+func (s BuildConfig) MatchCacheBuster(logger loggers.Logger, p string) (func(string) bool, error) {
+	var matchers []func(string) bool
+	for _, cb := range s.CacheBusters {
+		if matcher := cb.compiledSource(p); matcher != nil {
+			matchers = append(matchers, matcher)
+		}
+	}
+	if len(matchers) > 0 {
+		return (func(cacheKey string) bool {
+			for _, m := range matchers {
+				if m(cacheKey) {
+					return true
+				}
+			}
+			return false
+		}), nil
+	}
+	return nil, nil
+}
+
+func (b *BuildConfig) CompileConfig(logger loggers.Logger) error {
+	for i, cb := range b.CacheBusters {
+		if err := cb.CompileConfig(logger); err != nil {
+			return fmt.Errorf("failed to compile cache buster %q: %w", cb.Source, err)
+		}
+		b.CacheBusters[i] = cb
+	}
+	return nil
+}
+
 func DecodeBuildConfig(cfg Provider) BuildConfig {
 	m := cfg.GetStringMap("build")
-	b := DefaultBuild
+	b := defaultBuild.clone()
 	if m == nil {
 		return b
 	}
 
 	err := mapstructure.WeakDecode(m, &b)
 	if err != nil {
-		return DefaultBuild
+		return defaultBuild
 	}
 
 	b.UseResourceCacheWhen = strings.ToLower(b.UseResourceCacheWhen)
@@ -152,7 +213,7 @@ type Server struct {
 	compiledRedirects []glob.Glob
 }
 
-func (s *Server) CompileConfig() error {
+func (s *Server) CompileConfig(logger loggers.Logger) error {
 	if s.compiledHeaders != nil {
 		return nil
 	}
@@ -162,6 +223,7 @@ func (s *Server) CompileConfig() error {
 	for _, r := range s.Redirects {
 		s.compiledRedirects = append(s.compiledRedirects, glob.MustCompile(r.From))
 	}
+
 	return nil
 }
 
@@ -228,10 +290,75 @@ type Redirect struct {
 	Force bool
 }
 
+// CacheBuster configures cache busting for assets.
+type CacheBuster struct {
+	// Trigger for files matching this regexp.
+	Source string
+
+	// Cache bust targets matching this regexp.
+	// This regexp can contain group matches (e.g. $1) from the source regexp.
+	Target string
+
+	compiledSource func(string) func(string) bool
+}
+
+func (c *CacheBuster) CompileConfig(logger loggers.Logger) error {
+	if c.compiledSource != nil {
+		return nil
+	}
+	source := c.Source
+	target := c.Target
+	sourceRe, err := regexp.Compile(source)
+	if err != nil {
+		return fmt.Errorf("failed to compile cache buster source %q: %w", c.Source, err)
+	}
+	var compileErr error
+	c.compiledSource = func(s string) func(string) bool {
+		m := sourceRe.FindStringSubmatch(s)
+		matchString := "no match"
+		match := m != nil
+		if match {
+			matchString = "match!"
+		}
+		logger.Debugf("cachebuster: Matching %q with source %q: %s\n", s, source, matchString)
+		if !match {
+			return nil
+		}
+		groups := m[1:]
+		// Replace $1, $2 etc. in target.
+
+		for i, g := range groups {
+			target = strings.ReplaceAll(target, fmt.Sprintf("$%d", i+1), g)
+		}
+		targetRe, err := regexp.Compile(target)
+		if err != nil {
+			compileErr = fmt.Errorf("failed to compile cache buster target %q: %w", target, err)
+			return nil
+		}
+		return func(s string) bool {
+			match = targetRe.MatchString(s)
+			matchString := "no match"
+			if match {
+				matchString = "match!"
+			}
+			logger.Debugf("cachebuster: Matching %q with target %q: %s\n", s, target, matchString)
+
+			return match
+		}
+
+	}
+	return compileErr
+}
+
 func (r Redirect) IsZero() bool {
 	return r.From == ""
 }
 
+const (
+	// Keep this a little coarse grained, some false positives are OK.
+	cssTargetCachebusterRe = `(css|styles|scss|sass)`
+)
+
 func DecodeServer(cfg Provider) (Server, error) {
 	s := &Server{}
 

config/commonConfig_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/gohugoio/hugo/common/herrors"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/common/types"
 
 	qt "github.com/frankban/quicktest"
@@ -91,7 +92,7 @@ status = 301
 
 	s, err := DecodeServer(cfg)
 	c.Assert(err, qt.IsNil)
-	c.Assert(s.CompileConfig(), qt.IsNil)
+	c.Assert(s.CompileConfig(loggers.NewErrorLogger()), qt.IsNil)
 
 	c.Assert(s.MatchHeaders("/foo.jpg"), qt.DeepEquals, []types.KeyValueStr{
 		{Key: "X-Content-Type-Options", Value: "nosniff"},
@@ -139,3 +140,27 @@ status = 301`,
 
 	}
 }
+
+func TestBuildConfigCacheBusters(t *testing.T) {
+	c := qt.New(t)
+	cfg := New()
+	conf := DecodeBuildConfig(cfg)
+	l := loggers.NewInfoLogger()
+	c.Assert(conf.CompileConfig(l), qt.IsNil)
+
+	m, err := conf.MatchCacheBuster(l, "assets/foo/main.js")
+	c.Assert(err, qt.IsNil)
+	c.Assert(m, qt.IsNotNil)
+	c.Assert(m("scripts"), qt.IsTrue)
+	c.Assert(m("asdf"), qt.IsFalse)
+
+	m, _ = conf.MatchCacheBuster(l, "tailwind.config.js")
+	c.Assert(m("css"), qt.IsTrue)
+	c.Assert(m("js"), qt.IsFalse)
+
+	m, err = conf.MatchCacheBuster(l, "assets/foo.json")
+	c.Assert(err, qt.IsNil)
+	c.Assert(m, qt.IsNotNil)
+	c.Assert(m("json"), qt.IsTrue)
+
+}

hugofs/glob/glob.go

@@ -80,6 +80,31 @@ func (gc *globCache) GetGlob(pattern string) (glob.Glob, error) {
 	return eg.glob, eg.err
 }
 
+// Or creates a new Glob from the given globs.
+func Or(globs ...glob.Glob) glob.Glob {
+	return globSlice{globs: globs}
+}
+
+// MatchesFunc is a convenience type to create a glob.Glob from a function.
+type MatchesFunc func(s string) bool
+
+func (m MatchesFunc) Match(s string) bool {
+	return m(s)
+}
+
+type globSlice struct {
+	globs []glob.Glob
+}
+
+func (g globSlice) Match(s string) bool {
+	for _, g := range g.globs {
+		if g.Match(s) {
+			return true
+		}
+	}
+	return false
+}
+
 type globDecorator struct {
 	// On Windows we may get filenames with Windows slashes to match,
 	// which we need to normalize.

hugolib/alias.go

@@ -82,8 +82,6 @@ func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format
 func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) {
 	handler := newAliasHandler(s.Tmpl(), s.Log, allowRoot)
 
-	s.Log.Debugln("creating alias:", path, "redirecting to", permalink)
-
 	targetPath, err := handler.targetPathAlias(path)
 	if err != nil {
 		return err

hugolib/filesystems/basefs.go

@@ -76,6 +76,8 @@ type BaseFs struct {
 
 	theBigFs *filesystemsCollector
 
+	workingDir string
+
 	// Locks.
 	buildMu Lockable // <project>/.hugo_build.lock
 }
@@ -201,6 +203,27 @@ func (fs *BaseFs) ResolveJSConfigFile(name string) string {
 	return ""
 }
 
+// MakePathRelative creates a relative path from the given filename.
+// It returns both the component name (e.g. layouts) and the path relative to that.
+func (fs *BaseFs) MakePathRelative(filename string) (string, string) {
+	for _, sfs := range fs.FileSystems() {
+		if sfs.Contains(filename) {
+			if s, found := sfs.MakePathRelative(filename); found {
+				return sfs.Name, s
+			}
+		}
+	}
+	// May be a static file.
+	if s := fs.MakeStaticPathRelative(filename); s != "" {
+		return files.ComponentFolderStatic, s
+	}
+	// Fall back to relative to the working dir.
+	if strings.HasPrefix(filename, fs.workingDir) {
+		return "", strings.TrimPrefix(filename, fs.workingDir)
+	}
+	return "", ""
+}
+
 // SourceFilesystems contains the different source file systems. These can be
 // composite file systems (theme and project etc.), and they have all root
 // set to the source type the provides: data, i18n, static, layouts.
@@ -235,6 +258,7 @@ type SourceFilesystems struct {
 func (s *SourceFilesystems) FileSystems() []*SourceFilesystem {
 	return []*SourceFilesystem{
 		s.Content,
+		s.Assets,
 		s.Data,
 		s.I18n,
 		s.Layouts,
@@ -466,6 +490,7 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err
 		WorkDir:         fs.WorkingDirReadOnly,
 		PublishFs:       publishFs,
 		PublishFsStatic: publishFsStatic,
+		workingDir:      p.Cfg.BaseConfig().WorkingDir,
 		buildMu:         buildMu,
 	}
 

hugolib/hugo_sites_build_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/common/loggers"
 	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/resources/page"
 
@@ -1393,7 +1394,7 @@ other = %q
 }
 
 func TestRebuildOnAssetChange(t *testing.T) {
-	b := newTestSitesBuilder(t).Running()
+	b := newTestSitesBuilder(t).Running().WithLogger(loggers.NewInfoLogger())
 	b.WithTemplatesAdded("index.html", `
 {{ (resources.Get "data.json").Content }}
 `)

hugolib/site.go

@@ -22,7 +22,6 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"sort"
 	"strings"
@@ -36,8 +35,6 @@ import (
 
 	"github.com/gohugoio/hugo/common/paths"
 
-	"github.com/gohugoio/hugo/resources"
-
 	"github.com/gohugoio/hugo/identity"
 
 	"github.com/gohugoio/hugo/markup/converter/hooks"
@@ -45,6 +42,7 @@ import (
 	"github.com/gohugoio/hugo/markup/converter"
 
 	"github.com/gohugoio/hugo/hugofs/files"
+	hglob "github.com/gohugoio/hugo/hugofs/glob"
 
 	"github.com/gohugoio/hugo/common/maps"
 
@@ -483,16 +481,6 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event {
 	return filtered
 }
 
-var (
-	// These are only used for cache busting, so false positives are fine.
-	// We also deliberately do not match for file suffixes to also catch
-	// directory names.
-	// TODO(bep) consider this when completing the relevant PR rewrite on this.
-	cssFileRe   = regexp.MustCompile("(css|sass|scss)")
-	cssConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`)
-	jsFileRe    = regexp.MustCompile("(js|ts|jsx|tsx)")
-)
-
 // reBuild partially rebuilds a site given the filesystem events.
 // It returns whatever the content source was changed.
 // TODO(bep) clean up/rewrite this method.
@@ -524,24 +512,16 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
 		logger = helpers.NewDistinctErrorLogger()
 	)
 
-	var cachePartitions []string
+	var cacheBusters []func(string) bool
-	// Special case
+	bcfg := s.conf.Build
-	// TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there.
-	var (
-		evictCSSRe *regexp.Regexp
-		evictJSRe  *regexp.Regexp
-	)
 
 	for _, ev := range events {
-		if assetsFilename, _ := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" {
+		component, relFilename := s.BaseFs.MakePathRelative(ev.Name)
-			cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...)
+		if relFilename != "" {
-			if evictCSSRe == nil {
+			p := hglob.NormalizePath(path.Join(component, relFilename))
-				if cssFileRe.MatchString(assetsFilename) || cssConfigRe.MatchString(assetsFilename) {
+			g, err := bcfg.MatchCacheBuster(s.Log, p)
-					evictCSSRe = cssFileRe
+			if err == nil && g != nil {
-				}
+				cacheBusters = append(cacheBusters, g)
-			}
-			if evictJSRe == nil && jsFileRe.MatchString(assetsFilename) {
-				evictJSRe = jsFileRe
 			}
 		}
 
@@ -586,15 +566,21 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
 		return err
 	}
 
+	var cacheBusterOr func(string) bool
+	if len(cacheBusters) > 0 {
+		cacheBusterOr = func(s string) bool {
+			for _, cb := range cacheBusters {
+				if cb(s) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+
 	// These in memory resource caches will be rebuilt on demand.
-	for _, s := range s.h.Sites {
+	if len(cacheBusters) > 0 {
-		s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...)
+		s.h.ResourceSpec.ResourceCache.DeleteMatches(cacheBusterOr)
-		if evictCSSRe != nil {
-			s.ResourceSpec.ResourceCache.DeleteMatches(evictCSSRe)
-		}
-		if evictJSRe != nil {
-			s.ResourceSpec.ResourceCache.DeleteMatches(evictJSRe)
-		}
 	}
 
 	if tmplChanged || i18nChanged {
@@ -1024,7 +1010,6 @@ func (s *Site) lookupLayouts(layouts ...string) tpl.Template {
 }
 
 func (s *Site) renderAndWriteXML(ctx context.Context, statCounter *uint64, name string, targetPath string, d any, templ tpl.Template) error {
-	s.Log.Debugf("Render XML for %q to %q", name, targetPath)
 	renderBuffer := bp.GetBuffer()
 	defer bp.PutBuffer(renderBuffer)
 
@@ -1046,7 +1031,6 @@ func (s *Site) renderAndWriteXML(ctx context.Context, statCounter *uint64, name
 }
 
 func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *pageState, templ tpl.Template) error {
-	s.Log.Debugf("Render %s to %q", name, targetPath)
 	s.h.IncrPageRender()
 	renderBuffer := bp.GetBuffer()
 	defer bp.PutBuffer(renderBuffer)

resources/resource_cache.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/gohugoio/hugo/helpers"
 
-	"github.com/gohugoio/hugo/hugofs/glob"
+	hglob "github.com/gohugoio/hugo/hugofs/glob"
 
 	"github.com/gohugoio/hugo/resources/resource"
 
@@ -83,7 +83,7 @@ var extAliasKeywords = map[string][]string{
 // e.g. "scss" will also return "sass".
 func ResourceKeyPartitions(filename string) []string {
 	var partitions []string
-	filename = glob.NormalizePath(filename)
+	filename = hglob.NormalizePath(filename)
 	dir, name := path.Split(filename)
 	ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".")
 
@@ -282,7 +282,7 @@ func (c *ResourceCache) DeletePartitions(partitions ...string) {
 	}
 }
 
-func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) {
+func (c *ResourceCache) DeleteMatchesRe(re *regexp.Regexp) {
 	c.Lock()
 	defer c.Unlock()
 
@@ -292,3 +292,14 @@ func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) {
 		}
 	}
 }
+
+func (c *ResourceCache) DeleteMatches(match func(string) bool) {
+	c.Lock()
+	defer c.Unlock()
+
+	for k := range c.cache {
+		if match(k) {
+			delete(c.cache, k)
+		}
+	}
+}