Add resources.Match and resources.GetMatch

Fix #6190

hugofs/glob.go

@@ -0,0 +1,85 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/hugofs/glob"
+
+	"github.com/spf13/afero"
+)
+
+// Glob walks the fs and passes all matches to the handle func.
+// The handle func can return true to signal a stop.
+func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error {
+	pattern = glob.NormalizePath(pattern)
+	if pattern == "" {
+		return nil
+	}
+
+	g, err := glob.GetGlob(pattern)
+	if err != nil {
+		return nil
+	}
+
+	hasSuperAsterisk := strings.Contains(pattern, "**")
+	levels := strings.Count(pattern, "/")
+	root := glob.ResolveRootDir(pattern)
+
+	// Signals that we're done.
+	done := errors.New("done")
+
+	wfn := func(p string, info FileMetaInfo, err error) error {
+		p = glob.NormalizePath(p)
+		if info.IsDir() {
+			if !hasSuperAsterisk {
+				// Avoid walking to the bottom if we can avoid it.
+				if p != "" && strings.Count(p, "/") >= levels {
+					return filepath.SkipDir
+				}
+			}
+			return nil
+		}
+
+		if g.Match(p) {
+			d, err := handle(info)
+			if err != nil {
+				return err
+			}
+			if d {
+				return done
+			}
+		}
+
+		return nil
+	}
+
+	w := NewWalkway(WalkwayConfig{
+		Root:   root,
+		Fs:     fs,
+		WalkFn: wfn,
+	})
+
+	err = w.Walk()
+
+	if err != done {
+		return err
+	}
+
+	return nil
+
+}

hugofs/glob/glob.go

@@ -11,13 +11,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package internal
+package glob
 
 import (
+	"path"
+	"path/filepath"
 	"strings"
 	"sync"
 
 	"github.com/gobwas/glob"
+	"github.com/gobwas/glob/syntax"
 )
 
 var (
@@ -46,3 +49,33 @@ func GetGlob(pattern string) (glob.Glob, error) {
 	return g, nil
 
 }
+
+func NormalizePath(p string) string {
+	return strings.Trim(filepath.ToSlash(strings.ToLower(p)), "/.")
+}
+
+// ResolveRootDir takes a normalized path on the form "assets/**.json" and
+// determines any root dir, i.e. any start path without any wildcards.
+func ResolveRootDir(p string) string {
+	parts := strings.Split(path.Dir(p), "/")
+	var roots []string
+	for _, part := range parts {
+		isSpecial := false
+		for i := 0; i < len(part); i++ {
+			if syntax.Special(part[i]) {
+				isSpecial = true
+				break
+			}
+		}
+		if isSpecial {
+			break
+		}
+		roots = append(roots, part)
+	}
+
+	if len(roots) == 0 {
+		return ""
+	}
+
+	return strings.Join(roots, "/")
+}

hugofs/glob/glob_test.go

@@ -0,0 +1,63 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+	"path/filepath"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestResolveRootDir(t *testing.T) {
+	c := qt.New(t)
+
+	for _, test := range []struct {
+		in     string
+		expect string
+	}{
+		{"data/foo.json", "data"},
+		{"a/b/**/foo.json", "a/b"},
+		{"dat?a/foo.json", ""},
+		{"a/b[a-c]/foo.json", "a"},
+	} {
+
+		c.Assert(ResolveRootDir(test.in), qt.Equals, test.expect)
+	}
+}
+
+func TestNormalizePath(t *testing.T) {
+	c := qt.New(t)
+
+	for _, test := range []struct {
+		in     string
+		expect string
+	}{
+		{filepath.FromSlash("data/FOO.json"), "data/foo.json"},
+		{filepath.FromSlash("/data/FOO.json"), "data/foo.json"},
+		{filepath.FromSlash("./FOO.json"), "foo.json"},
+		{"//", ""},
+	} {
+
+		c.Assert(NormalizePath(test.in), qt.Equals, test.expect)
+	}
+}
+
+func TestGetGlob(t *testing.T) {
+	c := qt.New(t)
+	g, err := GetGlob("**.JSON")
+	c.Assert(err, qt.IsNil)
+	c.Assert(g.Match("data/my.json"), qt.Equals, true)
+
+}

hugofs/glob_test.go

@@ -0,0 +1,61 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestGlob(t *testing.T) {
+	c := qt.New(t)
+
+	fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+	create := func(filename string) {
+		err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte("content "+filename), 0777)
+		c.Assert(err, qt.IsNil)
+	}
+
+	collect := func(pattern string) []string {
+		var paths []string
+		h := func(fi FileMetaInfo) (bool, error) {
+			paths = append(paths, fi.Meta().Path())
+			return false, nil
+		}
+		err := Glob(fs, pattern, h)
+		c.Assert(err, qt.IsNil)
+		return paths
+	}
+
+	create("root.json")
+	create("jsonfiles/d1.json")
+	create("jsonfiles/d2.json")
+	create("jsonfiles/sub/d3.json")
+	create("jsonfiles/d1.xml")
+	create("a/b/c/e/f.json")
+
+	c.Assert(collect("**.json"), qt.HasLen, 5)
+	c.Assert(collect("**"), qt.HasLen, 6)
+	c.Assert(collect(""), qt.HasLen, 0)
+	c.Assert(collect("jsonfiles/*.json"), qt.HasLen, 2)
+	c.Assert(collect("*.json"), qt.HasLen, 1)
+	c.Assert(collect("**.xml"), qt.HasLen, 1)
+	c.Assert(collect(filepath.FromSlash("/jsonfiles/*.json")), qt.HasLen, 2)
+
+}

hugolib/resource_chain_test.go

@@ -502,3 +502,33 @@ func TestMultiSiteResource(t *testing.T) {
 	b.AssertFileContent("public/text/pipes.txt", "Hugo Pipes")
 
 }
+
+func TestResourcesMatch(t *testing.T) {
+	t.Parallel()
+
+	b := newTestSitesBuilder(t)
+
+	b.WithContent("page.md", "")
+
+	b.WithSourceFile(
+		"assets/jsons/data1.json", "json1 content",
+		"assets/jsons/data2.json", "json2 content",
+		"assets/jsons/data3.xml", "xml content",
+	)
+
+	b.WithTemplates("index.html", `
+{{ $jsons := (resources.Match "jsons/*.json") }}
+{{ $json := (resources.GetMatch "jsons/*.json") }}
+{{ printf "JSONS: %d"  (len $jsons) }}
+JSON: {{ $json.RelPermalink }}: {{ $json.Content }}
+{{ range $jsons }}
+{{- .RelPermalink }}: {{ .Content }}
+{{ end }}
+`)
+
+	b.Build(BuildCfg{})
+
+	b.AssertFileContent("public/index.html",
+		"JSON: /jsons/data1.json: json1 content",
+		"JSONS: 2", "/jsons/data1.json: json1 content")
+}

resources/resource/resources.go

@@ -17,7 +17,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/gohugoio/hugo/resources/internal"
+	"github.com/gohugoio/hugo/hugofs/glob"
 )
 
 // Resources represents a slice of resources, which can be a mix of different types.
@@ -44,7 +44,7 @@ func (r Resources) ByType(tp string) Resources {
 // GetMatch finds the first Resource matching the given pattern, or nil if none found.
 // See Match for a more complete explanation about the rules used.
 func (r Resources) GetMatch(pattern string) Resource {
-	g, err := internal.GetGlob(pattern)
+	g, err := glob.GetGlob(pattern)
 	if err != nil {
 		return nil
 	}
@@ -68,7 +68,7 @@ func (r Resources) GetMatch(pattern string) Resource {
 // path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
 // See https://github.com/gobwas/glob for the full rules set.
 func (r Resources) Match(pattern string) Resources {
-	g, err := internal.GetGlob(pattern)
+	g, err := glob.GetGlob(pattern)
 	if err != nil {
 		return nil
 	}

resources/resource_cache.go

@@ -37,7 +37,9 @@ type ResourceCache struct {
 	rs *Spec
 
 	sync.RWMutex
-	cache map[string]resource.Resource
+
+	// Either resource.Resource or resource.Resources.
+	cache map[string]interface{}
 
 	fileCache *filecache.Cache
 
@@ -61,7 +63,7 @@ func newResourceCache(rs *Spec) *ResourceCache {
 	return &ResourceCache{
 		rs:        rs,
 		fileCache: rs.FileCaches.AssetsCache(),
-		cache:     make(map[string]resource.Resource),
+		cache:     make(map[string]interface{}),
 		nlocker:   locker.NewLocker(),
 	}
 }
@@ -70,7 +72,7 @@ func (c *ResourceCache) clear() {
 	c.Lock()
 	defer c.Unlock()
 
-	c.cache = make(map[string]resource.Resource)
+	c.cache = make(map[string]interface{})
 	c.nlocker = locker.NewLocker()
 }
 
@@ -84,7 +86,7 @@ func (c *ResourceCache) cleanKey(key string) string {
 	return strings.TrimPrefix(path.Clean(key), "/")
 }
 
-func (c *ResourceCache) get(key string) (resource.Resource, bool) {
+func (c *ResourceCache) get(key string) (interface{}, bool) {
 	c.RLock()
 	defer c.RUnlock()
 	r, found := c.cache[key]
@@ -92,6 +94,22 @@ func (c *ResourceCache) get(key string) (resource.Resource, bool) {
 }
 
 func (c *ResourceCache) GetOrCreate(partition, key string, f func() (resource.Resource, error)) (resource.Resource, error) {
+	r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
+	if r == nil || err != nil {
+		return nil, err
+	}
+	return r.(resource.Resource), nil
+}
+
+func (c *ResourceCache) GetOrCreateResources(partition, key string, f func() (resource.Resources, error)) (resource.Resources, error) {
+	r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
+	if r == nil || err != nil {
+		return nil, err
+	}
+	return r.(resource.Resources), nil
+}
+
+func (c *ResourceCache) getOrCreate(partition, key string, f func() (interface{}, error)) (interface{}, error) {
 	key = c.cleanKey(path.Join(partition, key))
 	// First check in-memory cache.
 	r, found := c.get(key)
@@ -174,7 +192,7 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata)
 
 }
 
-func (c *ResourceCache) set(key string, r resource.Resource) {
+func (c *ResourceCache) set(key string, r interface{}) {
 	c.Lock()
 	defer c.Unlock()
 	c.cache[key] = r

resources/resource_factories/create/create.go

@@ -16,9 +16,12 @@
 package create
 
 import (
+	"path"
 	"path/filepath"
 
-	"github.com/spf13/afero"
+	"github.com/gohugoio/hugo/hugofs/glob"
+
+	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/resources"
@@ -36,18 +39,75 @@ func New(rs *resources.Spec) *Client {
 	return &Client{rs: rs}
 }
 
-// Get creates a new Resource by opening the given filename in the given filesystem.
+// Get creates a new Resource by opening the given filename in the assets filesystem.
-func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
+func (c *Client) Get(filename string) (resource.Resource, error) {
 	filename = filepath.Clean(filename)
 	return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
 		return c.rs.New(resources.ResourceSourceDescriptor{
-			Fs:             fs,
+			Fs:             c.rs.BaseFs.Assets.Fs,
 			LazyPublish:    true,
 			SourceFilename: filename})
 	})
 
 }
 
+// Match gets the resources matching the given pattern from the assets filesystem.
+func (c *Client) Match(pattern string) (resource.Resources, error) {
+	return c.match(pattern, false)
+}
+
+// GetMatch gets first resource matching the given pattern from the assets filesystem.
+func (c *Client) GetMatch(pattern string) (resource.Resource, error) {
+	res, err := c.match(pattern, true)
+	if err != nil || len(res) == 0 {
+		return nil, err
+	}
+	return res[0], err
+}
+
+func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, error) {
+	var partition string
+	if firstOnly {
+		partition = "__get-match"
+	} else {
+		partition = "__match"
+	}
+
+	// TODO(bep) match will be improved as part of https://github.com/gohugoio/hugo/issues/6199
+	partition = path.Join(resources.CACHE_OTHER, partition)
+	key := glob.NormalizePath(pattern)
+
+	return c.rs.ResourceCache.GetOrCreateResources(partition, key, func() (resource.Resources, error) {
+		var res resource.Resources
+
+		handle := func(info hugofs.FileMetaInfo) (bool, error) {
+			meta := info.Meta()
+			r, err := c.rs.New(resources.ResourceSourceDescriptor{
+				LazyPublish: true,
+				OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+					return meta.Open()
+				},
+				RelTargetFilename: meta.Path()})
+
+			if err != nil {
+				return true, err
+			}
+
+			res = append(res, r)
+
+			return firstOnly, nil
+
+		}
+
+		if err := hugofs.Glob(c.rs.BaseFs.Assets.Fs, pattern, handle); err != nil {
+			return nil, err
+		}
+
+		return res, nil
+
+	})
+}
+
 // FromString creates a new Resource from a string with the given relative target path.
 func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
 	return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {

resources/resource_metadata.go

@@ -17,7 +17,7 @@ import (
 	"fmt"
 	"strconv"
 
-	"github.com/gohugoio/hugo/resources/internal"
+	"github.com/gohugoio/hugo/hugofs/glob"
 	"github.com/gohugoio/hugo/resources/resource"
 
 	"github.com/pkg/errors"
@@ -70,7 +70,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res
 
 			srcKey := strings.ToLower(cast.ToString(src))
 
-			glob, err := internal.GetGlob(srcKey)
+			glob, err := glob.GetGlob(srcKey)
 			if err != nil {
 				return errors.Wrap(err, "failed to match resource with metadata")
 			}

tpl/resources/resources.go

@@ -68,7 +68,7 @@ type Namespace struct {
 	templatesClient *templates.Client
 }
 
-// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order)
+// Get locates the filename given in Hugo's assets filesystem
 // and creates a Resource object that can be used for further transformations.
 func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
 	filenamestr, err := cast.ToStringE(filename)
@@ -78,12 +78,50 @@ func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
 
 	filenamestr = filepath.Clean(filenamestr)
 
-	// Resource Get'ing is currently limited to /assets to make it simpler
+	return ns.createClient.Get(filenamestr)
-	// to control the behaviour of publishing and partial rebuilding.
-	return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr)
 
 }
 
+// GetMatch finds the first Resource matching the given pattern, or nil if none found.
+//
+// It looks for files in the assets file system.
+//
+// See Match for a more complete explanation about the rules used.
+func (ns *Namespace) GetMatch(pattern interface{}) (resource.Resource, error) {
+	patternStr, err := cast.ToStringE(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	return ns.createClient.GetMatch(patternStr)
+
+}
+
+// Match gets all resources matching the given base path prefix, e.g
+// "*.png" will match all png files. The "*" does not match path delimiters (/),
+// so if you organize your resources in sub-folders, you need to be explicit about it, e.g.:
+// "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and
+// to match all PNG images below the images folder, use "images/**.jpg".
+//
+// The matching is case insensitive.
+//
+// Match matches by using the files name with path relative to the file system root
+// with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
+//
+// See https://github.com/gobwas/glob for the full rules set.
+//
+// It looks for files in the assets file system.
+//
+// See Match for a more complete explanation about the rules used.
+func (ns *Namespace) Match(pattern interface{}) (resource.Resources, error) {
+	patternStr, err := cast.ToStringE(pattern)
+	if err != nil {
+		return nil, err
+	}
+
+	return ns.createClient.Match(patternStr)
+}
+
 // Concat concatenates a slice of Resource objects. These resources must
 // (currently) be of the same Media Type.
 func (ns *Namespace) Concat(targetPathIn interface{}, r interface{}) (resource.Resource, error) {