Add resources.Match and resources.GetMatch

Fix #6190
This commit is contained in:
Bjørn Erik Pedersen 2019-08-12 16:43:37 +02:00
parent 17ca8f0c4c
commit b64617fe4f
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
10 changed files with 407 additions and 19 deletions

85
hugofs/glob.go Normal file
View file

@ -0,0 +1,85 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugofs
import (
"errors"
"path/filepath"
"strings"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/spf13/afero"
)
// Glob walks the fs and passes all matches to the handle func.
// The handle func can return true to signal a stop.
func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error {
pattern = glob.NormalizePath(pattern)
if pattern == "" {
return nil
}
g, err := glob.GetGlob(pattern)
if err != nil {
return nil
}
hasSuperAsterisk := strings.Contains(pattern, "**")
levels := strings.Count(pattern, "/")
root := glob.ResolveRootDir(pattern)
// Signals that we're done.
done := errors.New("done")
wfn := func(p string, info FileMetaInfo, err error) error {
p = glob.NormalizePath(p)
if info.IsDir() {
if !hasSuperAsterisk {
// Avoid walking to the bottom if we can avoid it.
if p != "" && strings.Count(p, "/") >= levels {
return filepath.SkipDir
}
}
return nil
}
if g.Match(p) {
d, err := handle(info)
if err != nil {
return err
}
if d {
return done
}
}
return nil
}
w := NewWalkway(WalkwayConfig{
Root: root,
Fs: fs,
WalkFn: wfn,
})
err = w.Walk()
if err != done {
return err
}
return nil
}

View file

@ -11,13 +11,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package internal
package glob
import (
"path"
"path/filepath"
"strings"
"sync"
"github.com/gobwas/glob"
"github.com/gobwas/glob/syntax"
)
var (
@ -46,3 +49,33 @@ func GetGlob(pattern string) (glob.Glob, error) {
return g, nil
}
func NormalizePath(p string) string {
return strings.Trim(filepath.ToSlash(strings.ToLower(p)), "/.")
}
// ResolveRootDir takes a normalized path on the form "assets/**.json" and
// determines any root dir, i.e. any start path without any wildcards.
func ResolveRootDir(p string) string {
parts := strings.Split(path.Dir(p), "/")
var roots []string
for _, part := range parts {
isSpecial := false
for i := 0; i < len(part); i++ {
if syntax.Special(part[i]) {
isSpecial = true
break
}
}
if isSpecial {
break
}
roots = append(roots, part)
}
if len(roots) == 0 {
return ""
}
return strings.Join(roots, "/")
}

63
hugofs/glob/glob_test.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package glob
import (
"path/filepath"
"testing"
qt "github.com/frankban/quicktest"
)
func TestResolveRootDir(t *testing.T) {
c := qt.New(t)
for _, test := range []struct {
in string
expect string
}{
{"data/foo.json", "data"},
{"a/b/**/foo.json", "a/b"},
{"dat?a/foo.json", ""},
{"a/b[a-c]/foo.json", "a"},
} {
c.Assert(ResolveRootDir(test.in), qt.Equals, test.expect)
}
}
func TestNormalizePath(t *testing.T) {
c := qt.New(t)
for _, test := range []struct {
in string
expect string
}{
{filepath.FromSlash("data/FOO.json"), "data/foo.json"},
{filepath.FromSlash("/data/FOO.json"), "data/foo.json"},
{filepath.FromSlash("./FOO.json"), "foo.json"},
{"//", ""},
} {
c.Assert(NormalizePath(test.in), qt.Equals, test.expect)
}
}
func TestGetGlob(t *testing.T) {
c := qt.New(t)
g, err := GetGlob("**.JSON")
c.Assert(err, qt.IsNil)
c.Assert(g.Match("data/my.json"), qt.Equals, true)
}

61
hugofs/glob_test.go Normal file
View file

@ -0,0 +1,61 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugofs
import (
"path/filepath"
"testing"
"github.com/spf13/afero"
qt "github.com/frankban/quicktest"
)
func TestGlob(t *testing.T) {
c := qt.New(t)
fs := NewBaseFileDecorator(afero.NewMemMapFs())
create := func(filename string) {
err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte("content "+filename), 0777)
c.Assert(err, qt.IsNil)
}
collect := func(pattern string) []string {
var paths []string
h := func(fi FileMetaInfo) (bool, error) {
paths = append(paths, fi.Meta().Path())
return false, nil
}
err := Glob(fs, pattern, h)
c.Assert(err, qt.IsNil)
return paths
}
create("root.json")
create("jsonfiles/d1.json")
create("jsonfiles/d2.json")
create("jsonfiles/sub/d3.json")
create("jsonfiles/d1.xml")
create("a/b/c/e/f.json")
c.Assert(collect("**.json"), qt.HasLen, 5)
c.Assert(collect("**"), qt.HasLen, 6)
c.Assert(collect(""), qt.HasLen, 0)
c.Assert(collect("jsonfiles/*.json"), qt.HasLen, 2)
c.Assert(collect("*.json"), qt.HasLen, 1)
c.Assert(collect("**.xml"), qt.HasLen, 1)
c.Assert(collect(filepath.FromSlash("/jsonfiles/*.json")), qt.HasLen, 2)
}

View file

@ -502,3 +502,33 @@ func TestMultiSiteResource(t *testing.T) {
b.AssertFileContent("public/text/pipes.txt", "Hugo Pipes")
}
func TestResourcesMatch(t *testing.T) {
t.Parallel()
b := newTestSitesBuilder(t)
b.WithContent("page.md", "")
b.WithSourceFile(
"assets/jsons/data1.json", "json1 content",
"assets/jsons/data2.json", "json2 content",
"assets/jsons/data3.xml", "xml content",
)
b.WithTemplates("index.html", `
{{ $jsons := (resources.Match "jsons/*.json") }}
{{ $json := (resources.GetMatch "jsons/*.json") }}
{{ printf "JSONS: %d" (len $jsons) }}
JSON: {{ $json.RelPermalink }}: {{ $json.Content }}
{{ range $jsons }}
{{- .RelPermalink }}: {{ .Content }}
{{ end }}
`)
b.Build(BuildCfg{})
b.AssertFileContent("public/index.html",
"JSON: /jsons/data1.json: json1 content",
"JSONS: 2", "/jsons/data1.json: json1 content")
}

View file

@ -17,7 +17,7 @@ import (
"fmt"
"strings"
"github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/hugofs/glob"
)
// Resources represents a slice of resources, which can be a mix of different types.
@ -44,7 +44,7 @@ func (r Resources) ByType(tp string) Resources {
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
// See Match for a more complete explanation about the rules used.
func (r Resources) GetMatch(pattern string) Resource {
g, err := internal.GetGlob(pattern)
g, err := glob.GetGlob(pattern)
if err != nil {
return nil
}
@ -68,7 +68,7 @@ func (r Resources) GetMatch(pattern string) Resource {
// path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
// See https://github.com/gobwas/glob for the full rules set.
func (r Resources) Match(pattern string) Resources {
g, err := internal.GetGlob(pattern)
g, err := glob.GetGlob(pattern)
if err != nil {
return nil
}

View file

@ -37,7 +37,9 @@ type ResourceCache struct {
rs *Spec
sync.RWMutex
cache map[string]resource.Resource
// Either resource.Resource or resource.Resources.
cache map[string]interface{}
fileCache *filecache.Cache
@ -61,7 +63,7 @@ func newResourceCache(rs *Spec) *ResourceCache {
return &ResourceCache{
rs: rs,
fileCache: rs.FileCaches.AssetsCache(),
cache: make(map[string]resource.Resource),
cache: make(map[string]interface{}),
nlocker: locker.NewLocker(),
}
}
@ -70,7 +72,7 @@ func (c *ResourceCache) clear() {
c.Lock()
defer c.Unlock()
c.cache = make(map[string]resource.Resource)
c.cache = make(map[string]interface{})
c.nlocker = locker.NewLocker()
}
@ -84,7 +86,7 @@ func (c *ResourceCache) cleanKey(key string) string {
return strings.TrimPrefix(path.Clean(key), "/")
}
func (c *ResourceCache) get(key string) (resource.Resource, bool) {
func (c *ResourceCache) get(key string) (interface{}, bool) {
c.RLock()
defer c.RUnlock()
r, found := c.cache[key]
@ -92,6 +94,22 @@ func (c *ResourceCache) get(key string) (resource.Resource, bool) {
}
func (c *ResourceCache) GetOrCreate(partition, key string, f func() (resource.Resource, error)) (resource.Resource, error) {
r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
if r == nil || err != nil {
return nil, err
}
return r.(resource.Resource), nil
}
func (c *ResourceCache) GetOrCreateResources(partition, key string, f func() (resource.Resources, error)) (resource.Resources, error) {
r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
if r == nil || err != nil {
return nil, err
}
return r.(resource.Resources), nil
}
func (c *ResourceCache) getOrCreate(partition, key string, f func() (interface{}, error)) (interface{}, error) {
key = c.cleanKey(path.Join(partition, key))
// First check in-memory cache.
r, found := c.get(key)
@ -174,7 +192,7 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata)
}
func (c *ResourceCache) set(key string, r resource.Resource) {
func (c *ResourceCache) set(key string, r interface{}) {
c.Lock()
defer c.Unlock()
c.cache[key] = r

View file

@ -16,9 +16,12 @@
package create
import (
"path"
"path/filepath"
"github.com/spf13/afero"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/resources"
@ -36,18 +39,75 @@ func New(rs *resources.Spec) *Client {
return &Client{rs: rs}
}
// Get creates a new Resource by opening the given filename in the given filesystem.
func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
// Get creates a new Resource by opening the given filename in the assets filesystem.
func (c *Client) Get(filename string) (resource.Resource, error) {
filename = filepath.Clean(filename)
return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
return c.rs.New(resources.ResourceSourceDescriptor{
Fs: fs,
Fs: c.rs.BaseFs.Assets.Fs,
LazyPublish: true,
SourceFilename: filename})
})
}
// Match gets the resources matching the given pattern from the assets filesystem.
func (c *Client) Match(pattern string) (resource.Resources, error) {
return c.match(pattern, false)
}
// GetMatch gets first resource matching the given pattern from the assets filesystem.
func (c *Client) GetMatch(pattern string) (resource.Resource, error) {
res, err := c.match(pattern, true)
if err != nil || len(res) == 0 {
return nil, err
}
return res[0], err
}
func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, error) {
var partition string
if firstOnly {
partition = "__get-match"
} else {
partition = "__match"
}
// TODO(bep) match will be improved as part of https://github.com/gohugoio/hugo/issues/6199
partition = path.Join(resources.CACHE_OTHER, partition)
key := glob.NormalizePath(pattern)
return c.rs.ResourceCache.GetOrCreateResources(partition, key, func() (resource.Resources, error) {
var res resource.Resources
handle := func(info hugofs.FileMetaInfo) (bool, error) {
meta := info.Meta()
r, err := c.rs.New(resources.ResourceSourceDescriptor{
LazyPublish: true,
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
return meta.Open()
},
RelTargetFilename: meta.Path()})
if err != nil {
return true, err
}
res = append(res, r)
return firstOnly, nil
}
if err := hugofs.Glob(c.rs.BaseFs.Assets.Fs, pattern, handle); err != nil {
return nil, err
}
return res, nil
})
}
// FromString creates a new Resource from a string with the given relative target path.
func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {

View file

@ -17,7 +17,7 @@ import (
"fmt"
"strconv"
"github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/resources/resource"
"github.com/pkg/errors"
@ -70,7 +70,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res
srcKey := strings.ToLower(cast.ToString(src))
glob, err := internal.GetGlob(srcKey)
glob, err := glob.GetGlob(srcKey)
if err != nil {
return errors.Wrap(err, "failed to match resource with metadata")
}

View file

@ -68,7 +68,7 @@ type Namespace struct {
templatesClient *templates.Client
}
// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order)
// Get locates the filename given in Hugo's assets filesystem
// and creates a Resource object that can be used for further transformations.
func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
filenamestr, err := cast.ToStringE(filename)
@ -78,12 +78,50 @@ func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
filenamestr = filepath.Clean(filenamestr)
// Resource Get'ing is currently limited to /assets to make it simpler
// to control the behaviour of publishing and partial rebuilding.
return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr)
return ns.createClient.Get(filenamestr)
}
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
//
// It looks for files in the assets file system.
//
// See Match for a more complete explanation about the rules used.
func (ns *Namespace) GetMatch(pattern interface{}) (resource.Resource, error) {
patternStr, err := cast.ToStringE(pattern)
if err != nil {
return nil, err
}
return ns.createClient.GetMatch(patternStr)
}
// Match gets all resources matching the given base path prefix, e.g
// "*.png" will match all png files. The "*" does not match path delimiters (/),
// so if you organize your resources in sub-folders, you need to be explicit about it, e.g.:
// "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and
// to match all PNG images below the images folder, use "images/**.jpg".
//
// The matching is case insensitive.
//
// Match matches by using the files name with path relative to the file system root
// with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
//
// See https://github.com/gobwas/glob for the full rules set.
//
// It looks for files in the assets file system.
//
// See Match for a more complete explanation about the rules used.
func (ns *Namespace) Match(pattern interface{}) (resource.Resources, error) {
patternStr, err := cast.ToStringE(pattern)
if err != nil {
return nil, err
}
return ns.createClient.Match(patternStr)
}
// Concat concatenates a slice of Resource objects. These resources must
// (currently) be of the same Media Type.
func (ns *Namespace) Concat(targetPathIn interface{}, r interface{}) (resource.Resource, error) {