mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-21 20:46:30 -05:00
parent
17ca8f0c4c
commit
b64617fe4f
10 changed files with 407 additions and 19 deletions
85
hugofs/glob.go
Normal file
85
hugofs/glob.go
Normal file
|
@ -0,0 +1,85 @@
|
|||
// Copyright 2019 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package hugofs
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs/glob"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// Glob walks the fs and passes all matches to the handle func.
|
||||
// The handle func can return true to signal a stop.
|
||||
func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error {
|
||||
pattern = glob.NormalizePath(pattern)
|
||||
if pattern == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
g, err := glob.GetGlob(pattern)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
hasSuperAsterisk := strings.Contains(pattern, "**")
|
||||
levels := strings.Count(pattern, "/")
|
||||
root := glob.ResolveRootDir(pattern)
|
||||
|
||||
// Signals that we're done.
|
||||
done := errors.New("done")
|
||||
|
||||
wfn := func(p string, info FileMetaInfo, err error) error {
|
||||
p = glob.NormalizePath(p)
|
||||
if info.IsDir() {
|
||||
if !hasSuperAsterisk {
|
||||
// Avoid walking to the bottom if we can avoid it.
|
||||
if p != "" && strings.Count(p, "/") >= levels {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if g.Match(p) {
|
||||
d, err := handle(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d {
|
||||
return done
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
w := NewWalkway(WalkwayConfig{
|
||||
Root: root,
|
||||
Fs: fs,
|
||||
WalkFn: wfn,
|
||||
})
|
||||
|
||||
err = w.Walk()
|
||||
|
||||
if err != done {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
|
@ -11,13 +11,16 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package internal
|
||||
package glob
|
||||
|
||||
import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
"github.com/gobwas/glob/syntax"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -46,3 +49,33 @@ func GetGlob(pattern string) (glob.Glob, error) {
|
|||
return g, nil
|
||||
|
||||
}
|
||||
|
||||
func NormalizePath(p string) string {
|
||||
return strings.Trim(filepath.ToSlash(strings.ToLower(p)), "/.")
|
||||
}
|
||||
|
||||
// ResolveRootDir takes a normalized path on the form "assets/**.json" and
|
||||
// determines any root dir, i.e. any start path without any wildcards.
|
||||
func ResolveRootDir(p string) string {
|
||||
parts := strings.Split(path.Dir(p), "/")
|
||||
var roots []string
|
||||
for _, part := range parts {
|
||||
isSpecial := false
|
||||
for i := 0; i < len(part); i++ {
|
||||
if syntax.Special(part[i]) {
|
||||
isSpecial = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if isSpecial {
|
||||
break
|
||||
}
|
||||
roots = append(roots, part)
|
||||
}
|
||||
|
||||
if len(roots) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
return strings.Join(roots, "/")
|
||||
}
|
63
hugofs/glob/glob_test.go
Normal file
63
hugofs/glob/glob_test.go
Normal file
|
@ -0,0 +1,63 @@
|
|||
// Copyright 2019 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package glob
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
)
|
||||
|
||||
func TestResolveRootDir(t *testing.T) {
|
||||
c := qt.New(t)
|
||||
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expect string
|
||||
}{
|
||||
{"data/foo.json", "data"},
|
||||
{"a/b/**/foo.json", "a/b"},
|
||||
{"dat?a/foo.json", ""},
|
||||
{"a/b[a-c]/foo.json", "a"},
|
||||
} {
|
||||
|
||||
c.Assert(ResolveRootDir(test.in), qt.Equals, test.expect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizePath(t *testing.T) {
|
||||
c := qt.New(t)
|
||||
|
||||
for _, test := range []struct {
|
||||
in string
|
||||
expect string
|
||||
}{
|
||||
{filepath.FromSlash("data/FOO.json"), "data/foo.json"},
|
||||
{filepath.FromSlash("/data/FOO.json"), "data/foo.json"},
|
||||
{filepath.FromSlash("./FOO.json"), "foo.json"},
|
||||
{"//", ""},
|
||||
} {
|
||||
|
||||
c.Assert(NormalizePath(test.in), qt.Equals, test.expect)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetGlob(t *testing.T) {
|
||||
c := qt.New(t)
|
||||
g, err := GetGlob("**.JSON")
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(g.Match("data/my.json"), qt.Equals, true)
|
||||
|
||||
}
|
61
hugofs/glob_test.go
Normal file
61
hugofs/glob_test.go
Normal file
|
@ -0,0 +1,61 @@
|
|||
// Copyright 2019 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package hugofs
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
)
|
||||
|
||||
func TestGlob(t *testing.T) {
|
||||
c := qt.New(t)
|
||||
|
||||
fs := NewBaseFileDecorator(afero.NewMemMapFs())
|
||||
|
||||
create := func(filename string) {
|
||||
err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte("content "+filename), 0777)
|
||||
c.Assert(err, qt.IsNil)
|
||||
}
|
||||
|
||||
collect := func(pattern string) []string {
|
||||
var paths []string
|
||||
h := func(fi FileMetaInfo) (bool, error) {
|
||||
paths = append(paths, fi.Meta().Path())
|
||||
return false, nil
|
||||
}
|
||||
err := Glob(fs, pattern, h)
|
||||
c.Assert(err, qt.IsNil)
|
||||
return paths
|
||||
}
|
||||
|
||||
create("root.json")
|
||||
create("jsonfiles/d1.json")
|
||||
create("jsonfiles/d2.json")
|
||||
create("jsonfiles/sub/d3.json")
|
||||
create("jsonfiles/d1.xml")
|
||||
create("a/b/c/e/f.json")
|
||||
|
||||
c.Assert(collect("**.json"), qt.HasLen, 5)
|
||||
c.Assert(collect("**"), qt.HasLen, 6)
|
||||
c.Assert(collect(""), qt.HasLen, 0)
|
||||
c.Assert(collect("jsonfiles/*.json"), qt.HasLen, 2)
|
||||
c.Assert(collect("*.json"), qt.HasLen, 1)
|
||||
c.Assert(collect("**.xml"), qt.HasLen, 1)
|
||||
c.Assert(collect(filepath.FromSlash("/jsonfiles/*.json")), qt.HasLen, 2)
|
||||
|
||||
}
|
|
@ -502,3 +502,33 @@ func TestMultiSiteResource(t *testing.T) {
|
|||
b.AssertFileContent("public/text/pipes.txt", "Hugo Pipes")
|
||||
|
||||
}
|
||||
|
||||
func TestResourcesMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := newTestSitesBuilder(t)
|
||||
|
||||
b.WithContent("page.md", "")
|
||||
|
||||
b.WithSourceFile(
|
||||
"assets/jsons/data1.json", "json1 content",
|
||||
"assets/jsons/data2.json", "json2 content",
|
||||
"assets/jsons/data3.xml", "xml content",
|
||||
)
|
||||
|
||||
b.WithTemplates("index.html", `
|
||||
{{ $jsons := (resources.Match "jsons/*.json") }}
|
||||
{{ $json := (resources.GetMatch "jsons/*.json") }}
|
||||
{{ printf "JSONS: %d" (len $jsons) }}
|
||||
JSON: {{ $json.RelPermalink }}: {{ $json.Content }}
|
||||
{{ range $jsons }}
|
||||
{{- .RelPermalink }}: {{ .Content }}
|
||||
{{ end }}
|
||||
`)
|
||||
|
||||
b.Build(BuildCfg{})
|
||||
|
||||
b.AssertFileContent("public/index.html",
|
||||
"JSON: /jsons/data1.json: json1 content",
|
||||
"JSONS: 2", "/jsons/data1.json: json1 content")
|
||||
}
|
||||
|
|
|
@ -17,7 +17,7 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/resources/internal"
|
||||
"github.com/gohugoio/hugo/hugofs/glob"
|
||||
)
|
||||
|
||||
// Resources represents a slice of resources, which can be a mix of different types.
|
||||
|
@ -44,7 +44,7 @@ func (r Resources) ByType(tp string) Resources {
|
|||
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
|
||||
// See Match for a more complete explanation about the rules used.
|
||||
func (r Resources) GetMatch(pattern string) Resource {
|
||||
g, err := internal.GetGlob(pattern)
|
||||
g, err := glob.GetGlob(pattern)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
@ -68,7 +68,7 @@ func (r Resources) GetMatch(pattern string) Resource {
|
|||
// path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
|
||||
// See https://github.com/gobwas/glob for the full rules set.
|
||||
func (r Resources) Match(pattern string) Resources {
|
||||
g, err := internal.GetGlob(pattern)
|
||||
g, err := glob.GetGlob(pattern)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -37,7 +37,9 @@ type ResourceCache struct {
|
|||
rs *Spec
|
||||
|
||||
sync.RWMutex
|
||||
cache map[string]resource.Resource
|
||||
|
||||
// Either resource.Resource or resource.Resources.
|
||||
cache map[string]interface{}
|
||||
|
||||
fileCache *filecache.Cache
|
||||
|
||||
|
@ -61,7 +63,7 @@ func newResourceCache(rs *Spec) *ResourceCache {
|
|||
return &ResourceCache{
|
||||
rs: rs,
|
||||
fileCache: rs.FileCaches.AssetsCache(),
|
||||
cache: make(map[string]resource.Resource),
|
||||
cache: make(map[string]interface{}),
|
||||
nlocker: locker.NewLocker(),
|
||||
}
|
||||
}
|
||||
|
@ -70,7 +72,7 @@ func (c *ResourceCache) clear() {
|
|||
c.Lock()
|
||||
defer c.Unlock()
|
||||
|
||||
c.cache = make(map[string]resource.Resource)
|
||||
c.cache = make(map[string]interface{})
|
||||
c.nlocker = locker.NewLocker()
|
||||
}
|
||||
|
||||
|
@ -84,7 +86,7 @@ func (c *ResourceCache) cleanKey(key string) string {
|
|||
return strings.TrimPrefix(path.Clean(key), "/")
|
||||
}
|
||||
|
||||
func (c *ResourceCache) get(key string) (resource.Resource, bool) {
|
||||
func (c *ResourceCache) get(key string) (interface{}, bool) {
|
||||
c.RLock()
|
||||
defer c.RUnlock()
|
||||
r, found := c.cache[key]
|
||||
|
@ -92,6 +94,22 @@ func (c *ResourceCache) get(key string) (resource.Resource, bool) {
|
|||
}
|
||||
|
||||
func (c *ResourceCache) GetOrCreate(partition, key string, f func() (resource.Resource, error)) (resource.Resource, error) {
|
||||
r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
|
||||
if r == nil || err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.(resource.Resource), nil
|
||||
}
|
||||
|
||||
func (c *ResourceCache) GetOrCreateResources(partition, key string, f func() (resource.Resources, error)) (resource.Resources, error) {
|
||||
r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })
|
||||
if r == nil || err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.(resource.Resources), nil
|
||||
}
|
||||
|
||||
func (c *ResourceCache) getOrCreate(partition, key string, f func() (interface{}, error)) (interface{}, error) {
|
||||
key = c.cleanKey(path.Join(partition, key))
|
||||
// First check in-memory cache.
|
||||
r, found := c.get(key)
|
||||
|
@ -174,7 +192,7 @@ func (c *ResourceCache) writeMeta(key string, meta transformedResourceMetadata)
|
|||
|
||||
}
|
||||
|
||||
func (c *ResourceCache) set(key string, r resource.Resource) {
|
||||
func (c *ResourceCache) set(key string, r interface{}) {
|
||||
c.Lock()
|
||||
defer c.Unlock()
|
||||
c.cache[key] = r
|
||||
|
|
|
@ -16,9 +16,12 @@
|
|||
package create
|
||||
|
||||
import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/gohugoio/hugo/hugofs/glob"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
|
||||
"github.com/gohugoio/hugo/common/hugio"
|
||||
"github.com/gohugoio/hugo/resources"
|
||||
|
@ -36,18 +39,75 @@ func New(rs *resources.Spec) *Client {
|
|||
return &Client{rs: rs}
|
||||
}
|
||||
|
||||
// Get creates a new Resource by opening the given filename in the given filesystem.
|
||||
func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {
|
||||
// Get creates a new Resource by opening the given filename in the assets filesystem.
|
||||
func (c *Client) Get(filename string) (resource.Resource, error) {
|
||||
filename = filepath.Clean(filename)
|
||||
return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) {
|
||||
return c.rs.New(resources.ResourceSourceDescriptor{
|
||||
Fs: fs,
|
||||
Fs: c.rs.BaseFs.Assets.Fs,
|
||||
LazyPublish: true,
|
||||
SourceFilename: filename})
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
// Match gets the resources matching the given pattern from the assets filesystem.
|
||||
func (c *Client) Match(pattern string) (resource.Resources, error) {
|
||||
return c.match(pattern, false)
|
||||
}
|
||||
|
||||
// GetMatch gets first resource matching the given pattern from the assets filesystem.
|
||||
func (c *Client) GetMatch(pattern string) (resource.Resource, error) {
|
||||
res, err := c.match(pattern, true)
|
||||
if err != nil || len(res) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
return res[0], err
|
||||
}
|
||||
|
||||
func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, error) {
|
||||
var partition string
|
||||
if firstOnly {
|
||||
partition = "__get-match"
|
||||
} else {
|
||||
partition = "__match"
|
||||
}
|
||||
|
||||
// TODO(bep) match will be improved as part of https://github.com/gohugoio/hugo/issues/6199
|
||||
partition = path.Join(resources.CACHE_OTHER, partition)
|
||||
key := glob.NormalizePath(pattern)
|
||||
|
||||
return c.rs.ResourceCache.GetOrCreateResources(partition, key, func() (resource.Resources, error) {
|
||||
var res resource.Resources
|
||||
|
||||
handle := func(info hugofs.FileMetaInfo) (bool, error) {
|
||||
meta := info.Meta()
|
||||
r, err := c.rs.New(resources.ResourceSourceDescriptor{
|
||||
LazyPublish: true,
|
||||
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
|
||||
return meta.Open()
|
||||
},
|
||||
RelTargetFilename: meta.Path()})
|
||||
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
|
||||
res = append(res, r)
|
||||
|
||||
return firstOnly, nil
|
||||
|
||||
}
|
||||
|
||||
if err := hugofs.Glob(c.rs.BaseFs.Assets.Fs, pattern, handle); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return res, nil
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
// FromString creates a new Resource from a string with the given relative target path.
|
||||
func (c *Client) FromString(targetPath, content string) (resource.Resource, error) {
|
||||
return c.rs.ResourceCache.GetOrCreate(resources.CACHE_OTHER, targetPath, func() (resource.Resource, error) {
|
||||
|
|
|
@ -17,7 +17,7 @@ import (
|
|||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/gohugoio/hugo/resources/internal"
|
||||
"github.com/gohugoio/hugo/hugofs/glob"
|
||||
"github.com/gohugoio/hugo/resources/resource"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
@ -70,7 +70,7 @@ func AssignMetadata(metadata []map[string]interface{}, resources ...resource.Res
|
|||
|
||||
srcKey := strings.ToLower(cast.ToString(src))
|
||||
|
||||
glob, err := internal.GetGlob(srcKey)
|
||||
glob, err := glob.GetGlob(srcKey)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to match resource with metadata")
|
||||
}
|
||||
|
|
|
@ -68,7 +68,7 @@ type Namespace struct {
|
|||
templatesClient *templates.Client
|
||||
}
|
||||
|
||||
// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order)
|
||||
// Get locates the filename given in Hugo's assets filesystem
|
||||
// and creates a Resource object that can be used for further transformations.
|
||||
func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
|
||||
filenamestr, err := cast.ToStringE(filename)
|
||||
|
@ -78,12 +78,50 @@ func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
|
|||
|
||||
filenamestr = filepath.Clean(filenamestr)
|
||||
|
||||
// Resource Get'ing is currently limited to /assets to make it simpler
|
||||
// to control the behaviour of publishing and partial rebuilding.
|
||||
return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr)
|
||||
return ns.createClient.Get(filenamestr)
|
||||
|
||||
}
|
||||
|
||||
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
|
||||
//
|
||||
// It looks for files in the assets file system.
|
||||
//
|
||||
// See Match for a more complete explanation about the rules used.
|
||||
func (ns *Namespace) GetMatch(pattern interface{}) (resource.Resource, error) {
|
||||
patternStr, err := cast.ToStringE(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ns.createClient.GetMatch(patternStr)
|
||||
|
||||
}
|
||||
|
||||
// Match gets all resources matching the given base path prefix, e.g
|
||||
// "*.png" will match all png files. The "*" does not match path delimiters (/),
|
||||
// so if you organize your resources in sub-folders, you need to be explicit about it, e.g.:
|
||||
// "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and
|
||||
// to match all PNG images below the images folder, use "images/**.jpg".
|
||||
//
|
||||
// The matching is case insensitive.
|
||||
//
|
||||
// Match matches by using the files name with path relative to the file system root
|
||||
// with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
|
||||
//
|
||||
// See https://github.com/gobwas/glob for the full rules set.
|
||||
//
|
||||
// It looks for files in the assets file system.
|
||||
//
|
||||
// See Match for a more complete explanation about the rules used.
|
||||
func (ns *Namespace) Match(pattern interface{}) (resource.Resources, error) {
|
||||
patternStr, err := cast.ToStringE(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ns.createClient.Match(patternStr)
|
||||
}
|
||||
|
||||
// Concat concatenates a slice of Resource objects. These resources must
|
||||
// (currently) be of the same Media Type.
|
||||
func (ns *Namespace) Concat(targetPathIn interface{}, r interface{}) (resource.Resource, error) {
|
||||
|
|
Loading…
Reference in a new issue