github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-11-14 20:37:55 -05:00
Code Issues Projects Releases Packages Wiki Activity

Add cache busting config to support Tailwind 3

Browse source 
Fixes #10974
This commit is contained in:
Bjørn Erik Pedersen 2023-05-21 14:25:16 +02:00
parent 1292d5a26a
commit 2c3d4dfb74
12 changed files with 266 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
commands/commandeer.go
View file

@ -162,6 +162,7 @@ func (r *rootCommand) ConfigFromConfig(key int32, oldConf *commonConfig) (*commo
Fs: fs.Source,
Filename: r.cfgFile,
ConfigDir: r.cfgDir,
Logger: r.logger,
Environment: r.environment,
},
)
@ -222,6 +223,7 @@ func (r *rootCommand) ConfigFromProvider(key int32, cfg config.Provider) (*commo
Filename: r.cfgFile,
ConfigDir: r.cfgDir,
Environment: r.environment,
Logger: r.logger,
},
)
if err != nil {

15
config/allconfig/allconfig.go
View file

@ -27,6 +27,7 @@ import (
"time"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/urls"
"github.com/gohugoio/hugo/config"
@ -184,7 +185,7 @@ type Config struct {
}
type configCompiler interface {
CompileConfig() error
CompileConfig(logger loggers.Logger) error
}
func (c Config) cloneForLang() *Config {
@ -209,7 +210,7 @@ func (c Config) cloneForLang() *Config {
return &x
}
func (c *Config) CompileConfig() error {
func (c *Config) CompileConfig(logger loggers.Logger) error {
var transientErr error
s := c.Timeout
if _, err := strconv.Atoi(s); err == nil {
@ -328,7 +329,7 @@ func (c *Config) CompileConfig() error {
for _, s := range allDecoderSetups {
if getCompiler := s.getCompiler; getCompiler != nil {
if err := getCompiler(c).CompileConfig(); err != nil {
if err := getCompiler(c).CompileConfig(logger); err != nil {
return err
}
}
@ -668,8 +669,8 @@ func (c Configs) GetByLang(lang string) config.AllProvider {
return nil
}
// FromLoadConfigResult creates a new Config from res.
func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, error) {
// fromLoadConfigResult creates a new Config from res.
func fromLoadConfigResult(fs afero.Fs, logger loggers.Logger, res config.LoadConfigResult) (*Configs, error) {
if !res.Cfg.IsSet("languages") {
// We need at least one
lang := res.Cfg.GetString("defaultContentLanguage")
@ -690,7 +691,7 @@ func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, e
languagesConfig := cfg.GetStringMap("languages")
var isMultiHost bool
if err := all.CompileConfig(); err != nil {
if err := all.CompileConfig(logger); err != nil {
return nil, err
}
@ -769,7 +770,7 @@ func FromLoadConfigResult(fs afero.Fs, res config.LoadConfigResult) (*Configs, e
if err := decodeConfigFromParams(fs, bcfg, mergedConfig, clone, differentRootKeys); err != nil {
return nil, fmt.Errorf("failed to decode config for language %q: %w", k, err)
}
if err := clone.CompileConfig(); err != nil {
if err := clone.CompileConfig(logger); err != nil {
return nil, err
}
langConfigMap[k] = clone

3
config/allconfig/alldecoders.go
View file

@ -92,6 +92,9 @@ var allDecoderSetups = map[string]decodeWeight{
p.c.Build = config.DecodeBuildConfig(p.p)
return nil
},
getCompiler: func(c *Config) configCompiler {
return &c.Build
},
},
"frontmatter": {
key: "frontmatter",

8
config/allconfig/load.go
View file

@ -44,6 +44,10 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
d.Environ = os.Environ()
}
if d.Logger == nil {
d.Logger = loggers.NewErrorLogger()
}
l := &configLoader{ConfigSourceDescriptor: d, cfg: config.New()}
// Make sure we always do this, even in error situations,
// as we have commands (e.g. "hugo mod init") that will
@ -54,7 +58,7 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
return nil, fmt.Errorf("failed to load config: %w", err)
}
configs, err := FromLoadConfigResult(d.Fs, res)
configs, err := fromLoadConfigResult(d.Fs, d.Logger, res)
if err != nil {
return nil, fmt.Errorf("failed to create config from result: %w", err)
}
@ -67,7 +71,7 @@ func LoadConfig(d ConfigSourceDescriptor) (*Configs, error) {
if len(l.ModulesConfigFiles) > 0 {
// Config merged in from modules.
// Re-read the config.
configs, err = FromLoadConfigResult(d.Fs, res)
configs, err = fromLoadConfigResult(d.Fs, d.Logger, res)
if err != nil {
return nil, fmt.Errorf("failed to create config from modules config: %w", err)
}

137
config/commonConfig.go
View file

@ -15,12 +15,14 @@ package config
import (
"fmt"
"regexp"
"sort"
"strings"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/types"
"github.com/gobwas/glob"
"github.com/gohugoio/hugo/common/herrors"
"github.com/mitchellh/mapstructure"
"github.com/spf13/cast"
@ -77,9 +79,29 @@ type LoadConfigResult struct {
BaseConfig BaseConfig
}
var DefaultBuild = BuildConfig{
var defaultBuild = BuildConfig{
UseResourceCacheWhen: "fallback",
WriteStats: false,
CacheBusters: []CacheBuster{
{
Source: `assets/.*\.(js|ts|jsx|tsx)`,
Target: `(js|scripts|javascript)`,
},
{
Source: `assets/.*\.(css|sass|scss)$`,
Target: cssTargetCachebusterRe,
},
{
Source: `(postcss|tailwind)\.config\.js`,
Target: cssTargetCachebusterRe,
},
// This is deliberatly coarse grained; it will cache bust resources with "json" in the cache key when js files changes, which is good.
{
Source: `assets/.*\.(.*)$`,
Target: `$1`,
},
},
}
// BuildConfig holds some build related configuration.
@ -93,6 +115,14 @@ type BuildConfig struct {
// Can be used to toggle off writing of the intellinsense /assets/jsconfig.js
// file.
NoJSConfigInAssets bool
// Can used to control how the resource cache gets evicted on rebuilds.
CacheBusters []CacheBuster
}
func (b BuildConfig) clone() BuildConfig {
b.CacheBusters = append([]CacheBuster{}, b.CacheBusters...)
return b
}
func (b BuildConfig) UseResourceCache(err error) bool {
@ -107,16 +137,47 @@ func (b BuildConfig) UseResourceCache(err error) bool {
return true
}
// MatchCacheBuster returns the cache buster for the given path p, nil if none.
func (s BuildConfig) MatchCacheBuster(logger loggers.Logger, p string) (func(string) bool, error) {
var matchers []func(string) bool
for _, cb := range s.CacheBusters {
if matcher := cb.compiledSource(p); matcher != nil {
matchers = append(matchers, matcher)
}
}
if len(matchers) > 0 {
return (func(cacheKey string) bool {
for _, m := range matchers {
if m(cacheKey) {
return true
}
}
return false
}), nil
}
return nil, nil
}
func (b *BuildConfig) CompileConfig(logger loggers.Logger) error {
for i, cb := range b.CacheBusters {
if err := cb.CompileConfig(logger); err != nil {
return fmt.Errorf("failed to compile cache buster %q: %w", cb.Source, err)
}
b.CacheBusters[i] = cb
}
return nil
}
func DecodeBuildConfig(cfg Provider) BuildConfig {
m := cfg.GetStringMap("build")
b := DefaultBuild
b := defaultBuild.clone()
if m == nil {
return b
}
err := mapstructure.WeakDecode(m, &b)
if err != nil {
return DefaultBuild
return defaultBuild
}
b.UseResourceCacheWhen = strings.ToLower(b.UseResourceCacheWhen)
@ -152,7 +213,7 @@ type Server struct {
compiledRedirects []glob.Glob
}
func (s *Server) CompileConfig() error {
func (s *Server) CompileConfig(logger loggers.Logger) error {
if s.compiledHeaders != nil {
return nil
}
@ -162,6 +223,7 @@ func (s *Server) CompileConfig() error {
for _, r := range s.Redirects {
s.compiledRedirects = append(s.compiledRedirects, glob.MustCompile(r.From))
}
return nil
}
@ -228,10 +290,75 @@ type Redirect struct {
Force bool
}
// CacheBuster configures cache busting for assets.
type CacheBuster struct {
// Trigger for files matching this regexp.
Source string
// Cache bust targets matching this regexp.
// This regexp can contain group matches (e.g. $1) from the source regexp.
Target string
compiledSource func(string) func(string) bool
}
func (c *CacheBuster) CompileConfig(logger loggers.Logger) error {
if c.compiledSource != nil {
return nil
}
source := c.Source
target := c.Target
sourceRe, err := regexp.Compile(source)
if err != nil {
return fmt.Errorf("failed to compile cache buster source %q: %w", c.Source, err)
}
var compileErr error
c.compiledSource = func(s string) func(string) bool {
m := sourceRe.FindStringSubmatch(s)
matchString := "no match"
match := m != nil
if match {
matchString = "match!"
}
logger.Debugf("cachebuster: Matching %q with source %q: %s\n", s, source, matchString)
if !match {
return nil
}
groups := m[1:]
// Replace $1, $2 etc. in target.
for i, g := range groups {
target = strings.ReplaceAll(target, fmt.Sprintf("$%d", i+1), g)
}
targetRe, err := regexp.Compile(target)
if err != nil {
compileErr = fmt.Errorf("failed to compile cache buster target %q: %w", target, err)
return nil
}
return func(s string) bool {
match = targetRe.MatchString(s)
matchString := "no match"
if match {
matchString = "match!"
}
logger.Debugf("cachebuster: Matching %q with target %q: %s\n", s, target, matchString)
return match
}
}
return compileErr
}
func (r Redirect) IsZero() bool {
return r.From == ""
}
const (
// Keep this a little coarse grained, some false positives are OK.
cssTargetCachebusterRe = `(css|styles|scss|sass)`
)
func DecodeServer(cfg Provider) (Server, error) {
s := &Server{}

27
config/commonConfig_test.go
View file

@ -18,6 +18,7 @@ import (
"testing"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/types"
qt "github.com/frankban/quicktest"
@ -91,7 +92,7 @@ status = 301
s, err := DecodeServer(cfg)
c.Assert(err, qt.IsNil)
c.Assert(s.CompileConfig(), qt.IsNil)
c.Assert(s.CompileConfig(loggers.NewErrorLogger()), qt.IsNil)
c.Assert(s.MatchHeaders("/foo.jpg"), qt.DeepEquals, []types.KeyValueStr{
{Key: "X-Content-Type-Options", Value: "nosniff"},
@ -139,3 +140,27 @@ status = 301`,
}
}
func TestBuildConfigCacheBusters(t *testing.T) {
c := qt.New(t)
cfg := New()
conf := DecodeBuildConfig(cfg)
l := loggers.NewInfoLogger()
c.Assert(conf.CompileConfig(l), qt.IsNil)
m, err := conf.MatchCacheBuster(l, "assets/foo/main.js")
c.Assert(err, qt.IsNil)
c.Assert(m, qt.IsNotNil)
c.Assert(m("scripts"), qt.IsTrue)
c.Assert(m("asdf"), qt.IsFalse)
m, _ = conf.MatchCacheBuster(l, "tailwind.config.js")
c.Assert(m("css"), qt.IsTrue)
c.Assert(m("js"), qt.IsFalse)
m, err = conf.MatchCacheBuster(l, "assets/foo.json")
c.Assert(err, qt.IsNil)
c.Assert(m, qt.IsNotNil)
c.Assert(m("json"), qt.IsTrue)
}

25
hugofs/glob/glob.go
View file

@ -80,6 +80,31 @@ func (gc *globCache) GetGlob(pattern string) (glob.Glob, error) {
return eg.glob, eg.err
}
// Or creates a new Glob from the given globs.
func Or(globs ...glob.Glob) glob.Glob {
return globSlice{globs: globs}
}
// MatchesFunc is a convenience type to create a glob.Glob from a function.
type MatchesFunc func(s string) bool
func (m MatchesFunc) Match(s string) bool {
return m(s)
}
type globSlice struct {
globs []glob.Glob
}
func (g globSlice) Match(s string) bool {
for _, g := range g.globs {
if g.Match(s) {
return true
}
}
return false
}
type globDecorator struct {
// On Windows we may get filenames with Windows slashes to match,
// which we need to normalize.

2
hugolib/alias.go
View file

@ -82,8 +82,6 @@ func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format
func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) {
handler := newAliasHandler(s.Tmpl(), s.Log, allowRoot)
s.Log.Debugln("creating alias:", path, "redirecting to", permalink)
targetPath, err := handler.targetPathAlias(path)
if err != nil {
return err

25
hugolib/filesystems/basefs.go
View file

@ -76,6 +76,8 @@ type BaseFs struct {
theBigFs *filesystemsCollector
workingDir string
// Locks.
buildMu Lockable // <project>/.hugo_build.lock
}
@ -201,6 +203,27 @@ func (fs *BaseFs) ResolveJSConfigFile(name string) string {
return ""
}
// MakePathRelative creates a relative path from the given filename.
// It returns both the component name (e.g. layouts) and the path relative to that.
func (fs *BaseFs) MakePathRelative(filename string) (string, string) {
for _, sfs := range fs.FileSystems() {
if sfs.Contains(filename) {
if s, found := sfs.MakePathRelative(filename); found {
return sfs.Name, s
}
}
}
// May be a static file.
if s := fs.MakeStaticPathRelative(filename); s != "" {
return files.ComponentFolderStatic, s
}
// Fall back to relative to the working dir.
if strings.HasPrefix(filename, fs.workingDir) {
return "", strings.TrimPrefix(filename, fs.workingDir)
}
return "", ""
}
// SourceFilesystems contains the different source file systems. These can be
// composite file systems (theme and project etc.), and they have all root
// set to the source type the provides: data, i18n, static, layouts.
@ -235,6 +258,7 @@ type SourceFilesystems struct {
func (s *SourceFilesystems) FileSystems() []*SourceFilesystem {
return []*SourceFilesystem{
s.Content,
s.Assets,
s.Data,
s.I18n,
s.Layouts,
@ -466,6 +490,7 @@ func NewBase(p *paths.Paths, logger loggers.Logger, options ...func(*BaseFs) err
WorkDir: fs.WorkingDirReadOnly,
PublishFs: publishFs,
PublishFsStatic: publishFsStatic,
workingDir: p.Cfg.BaseConfig().WorkingDir,
buildMu: buildMu,
}

3
hugolib/hugo_sites_build_test.go
View file

@ -8,6 +8,7 @@ import (
"time"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/resources/page"
@ -1393,7 +1394,7 @@ other = %q
}
func TestRebuildOnAssetChange(t *testing.T) {
b := newTestSitesBuilder(t).Running()
b := newTestSitesBuilder(t).Running().WithLogger(loggers.NewInfoLogger())
b.WithTemplatesAdded("index.html", `
{{ (resources.Get "data.json").Content }}
`)

62
hugolib/site.go
View file

@ -22,7 +22,6 @@ import (
"net/url"
"path"
"path/filepath"
"regexp"
"runtime"
"sort"
"strings"
@ -36,8 +35,6 @@ import (
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/markup/converter/hooks"
@ -45,6 +42,7 @@ import (
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/hugofs/files"
hglob "github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/common/maps"
@ -483,16 +481,6 @@ func (s *Site) translateFileEvents(events []fsnotify.Event) []fsnotify.Event {
return filtered
}
var (
// These are only used for cache busting, so false positives are fine.
// We also deliberately do not match for file suffixes to also catch
// directory names.
// TODO(bep) consider this when completing the relevant PR rewrite on this.
cssFileRe = regexp.MustCompile("(css|sass|scss)")
cssConfigRe = regexp.MustCompile(`(postcss|tailwind)\.config\.js`)
jsFileRe = regexp.MustCompile("(js|ts|jsx|tsx)")
)
// reBuild partially rebuilds a site given the filesystem events.
// It returns whatever the content source was changed.
// TODO(bep) clean up/rewrite this method.
@ -524,24 +512,16 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
logger = helpers.NewDistinctErrorLogger()
)
var cachePartitions []string
// Special case
// TODO(bep) I have a ongoing branch where I have redone the cache. Consider this there.
var (
evictCSSRe *regexp.Regexp
evictJSRe *regexp.Regexp
)
var cacheBusters []func(string) bool
bcfg := s.conf.Build
for _, ev := range events {
if assetsFilename, _ := s.BaseFs.Assets.MakePathRelative(ev.Name); assetsFilename != "" {
cachePartitions = append(cachePartitions, resources.ResourceKeyPartitions(assetsFilename)...)
if evictCSSRe == nil {
if cssFileRe.MatchString(assetsFilename) || cssConfigRe.MatchString(assetsFilename) {
evictCSSRe = cssFileRe
}
}
if evictJSRe == nil && jsFileRe.MatchString(assetsFilename) {
evictJSRe = jsFileRe
component, relFilename := s.BaseFs.MakePathRelative(ev.Name)
if relFilename != "" {
p := hglob.NormalizePath(path.Join(component, relFilename))
g, err := bcfg.MatchCacheBuster(s.Log, p)
if err == nil && g != nil {
cacheBusters = append(cacheBusters, g)
}
}
@ -586,15 +566,21 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro
return err
}
var cacheBusterOr func(string) bool
if len(cacheBusters) > 0 {
cacheBusterOr = func(s string) bool {
for _, cb := range cacheBusters {
if cb(s) {
return true
}
}
return false
}
}
// These in memory resource caches will be rebuilt on demand.
for _, s := range s.h.Sites {
s.ResourceSpec.ResourceCache.DeletePartitions(cachePartitions...)
if evictCSSRe != nil {
s.ResourceSpec.ResourceCache.DeleteMatches(evictCSSRe)
}
if evictJSRe != nil {
s.ResourceSpec.ResourceCache.DeleteMatches(evictJSRe)
}
if len(cacheBusters) > 0 {
s.h.ResourceSpec.ResourceCache.DeleteMatches(cacheBusterOr)
}
if tmplChanged || i18nChanged {
@ -1024,7 +1010,6 @@ func (s *Site) lookupLayouts(layouts ...string) tpl.Template {
}
func (s *Site) renderAndWriteXML(ctx context.Context, statCounter *uint64, name string, targetPath string, d any, templ tpl.Template) error {
s.Log.Debugf("Render XML for %q to %q", name, targetPath)
renderBuffer := bp.GetBuffer()
defer bp.PutBuffer(renderBuffer)
@ -1046,7 +1031,6 @@ func (s *Site) renderAndWriteXML(ctx context.Context, statCounter *uint64, name
}
func (s *Site) renderAndWritePage(statCounter *uint64, name string, targetPath string, p *pageState, templ tpl.Template) error {
s.Log.Debugf("Render %s to %q", name, targetPath)
s.h.IncrPageRender()
renderBuffer := bp.GetBuffer()
defer bp.PutBuffer(renderBuffer)

17
resources/resource_cache.go
View file

@ -24,7 +24,7 @@ import (
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs/glob"
hglob "github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/resources/resource"
@ -83,7 +83,7 @@ var extAliasKeywords = map[string][]string{
// e.g. "scss" will also return "sass".
func ResourceKeyPartitions(filename string) []string {
var partitions []string
filename = glob.NormalizePath(filename)
filename = hglob.NormalizePath(filename)
dir, name := path.Split(filename)
ext := strings.TrimPrefix(path.Ext(filepath.ToSlash(name)), ".")
@ -282,7 +282,7 @@ func (c *ResourceCache) DeletePartitions(partitions ...string) {
}
}
func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) {
func (c *ResourceCache) DeleteMatchesRe(re *regexp.Regexp) {
c.Lock()
defer c.Unlock()
@ -292,3 +292,14 @@ func (c *ResourceCache) DeleteMatches(re *regexp.Regexp) {
}
}
}
func (c *ResourceCache) DeleteMatches(match func(string) bool) {
c.Lock()
defer c.Unlock()
for k := range c.cache {
if match(k) {
delete(c.cache, k)
}
}
}