mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-07 20:30:36 -05:00
parent
43f9df0194
commit
822dc627a1
20 changed files with 633 additions and 74 deletions
84
cache/namedmemcache/named_cache.go
vendored
Normal file
84
cache/namedmemcache/named_cache.go
vendored
Normal file
|
@ -0,0 +1,84 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package namedmemcache provides a memory cache with a named lock. This is suitable
|
||||
// for situations where creating the cached resource can be time consuming or otherwise
|
||||
// resource hungry, or in situations where a "once only per key" is a requirement.
|
||||
package namedmemcache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/BurntSushi/locker"
|
||||
)
|
||||
|
||||
// Cache holds the cached values.
|
||||
type Cache struct {
|
||||
nlocker *locker.Locker
|
||||
cache map[string]cacheEntry
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
type cacheEntry struct {
|
||||
value interface{}
|
||||
err error
|
||||
}
|
||||
|
||||
// New creates a new cache.
|
||||
func New() *Cache {
|
||||
return &Cache{
|
||||
nlocker: locker.NewLocker(),
|
||||
cache: make(map[string]cacheEntry),
|
||||
}
|
||||
}
|
||||
|
||||
// Clear clears the cache state.
|
||||
func (c *Cache) Clear() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.cache = make(map[string]cacheEntry)
|
||||
c.nlocker = locker.NewLocker()
|
||||
|
||||
}
|
||||
|
||||
// GetOrCreate tries to get the value with the given cache key, if not found
|
||||
// create will be called and cached.
|
||||
// This method is thread safe. It also guarantees that the create func for a given
|
||||
// key is invoced only once for this cache.
|
||||
func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) {
|
||||
c.mu.RLock()
|
||||
entry, found := c.cache[key]
|
||||
c.mu.RUnlock()
|
||||
|
||||
if found {
|
||||
return entry.value, entry.err
|
||||
}
|
||||
|
||||
c.nlocker.Lock(key)
|
||||
defer c.nlocker.Unlock(key)
|
||||
|
||||
// Double check
|
||||
if entry, found := c.cache[key]; found {
|
||||
return entry.value, entry.err
|
||||
}
|
||||
|
||||
// Create it.
|
||||
value, err := create()
|
||||
|
||||
c.mu.Lock()
|
||||
c.cache[key] = cacheEntry{value: value, err: err}
|
||||
c.mu.Unlock()
|
||||
|
||||
return value, err
|
||||
}
|
80
cache/namedmemcache/named_cache_test.go
vendored
Normal file
80
cache/namedmemcache/named_cache_test.go
vendored
Normal file
|
@ -0,0 +1,80 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package namedmemcache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNamedCache(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert := require.New(t)
|
||||
|
||||
cache := New()
|
||||
|
||||
counter := 0
|
||||
create := func() (interface{}, error) {
|
||||
counter++
|
||||
return counter, nil
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
v1, err := cache.GetOrCreate("a1", create)
|
||||
assert.NoError(err)
|
||||
assert.Equal(1, v1)
|
||||
v2, err := cache.GetOrCreate("a2", create)
|
||||
assert.NoError(err)
|
||||
assert.Equal(2, v2)
|
||||
}
|
||||
|
||||
cache.Clear()
|
||||
|
||||
v3, err := cache.GetOrCreate("a2", create)
|
||||
assert.NoError(err)
|
||||
assert.Equal(3, v3)
|
||||
}
|
||||
|
||||
func TestNamedCacheConcurrent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert := require.New(t)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
cache := New()
|
||||
|
||||
create := func(i int) func() (interface{}, error) {
|
||||
return func() (interface{}, error) {
|
||||
return i, nil
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 100; j++ {
|
||||
id := fmt.Sprintf("id%d", j)
|
||||
v, err := cache.GetOrCreate(id, create(j))
|
||||
assert.NoError(err)
|
||||
assert.Equal(j, v)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
11
deps/deps.go
vendored
11
deps/deps.go
vendored
|
@ -123,6 +123,9 @@ type Listeners struct {
|
|||
|
||||
// Add adds a function to a Listeners instance.
|
||||
func (b *Listeners) Add(f func()) {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
b.Lock()
|
||||
defer b.Unlock()
|
||||
b.listeners = append(b.listeners, f)
|
||||
|
@ -192,6 +195,14 @@ func New(cfg DepsCfg) (*Deps, error) {
|
|||
fs = hugofs.NewDefault(cfg.Language)
|
||||
}
|
||||
|
||||
if cfg.MediaTypes == nil {
|
||||
cfg.MediaTypes = media.DefaultTypes
|
||||
}
|
||||
|
||||
if cfg.OutputFormats == nil {
|
||||
cfg.OutputFormats = output.DefaultFormats
|
||||
}
|
||||
|
||||
ps, err := helpers.NewPathSpec(fs, cfg.Language)
|
||||
|
||||
if err != nil {
|
||||
|
|
|
@ -394,11 +394,10 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) {
|
|||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// MD5FromFile creates a MD5 hash from the given file.
|
||||
// It will not close the file.
|
||||
func MD5FromFile(f afero.File) (string, error) {
|
||||
// MD5FromReader creates a MD5 hash from the given reader.
|
||||
func MD5FromReader(r io.Reader) (string, error) {
|
||||
h := md5.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
if _, err := io.Copy(h, r); err != nil {
|
||||
return "", nil
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
|
|
|
@ -272,7 +272,7 @@ func TestFastMD5FromFile(t *testing.T) {
|
|||
req.NoError(err)
|
||||
req.NotEqual(m3, m4)
|
||||
|
||||
m5, err := MD5FromFile(bf2)
|
||||
m5, err := MD5FromReader(bf2)
|
||||
req.NoError(err)
|
||||
req.NotEqual(m4, m5)
|
||||
}
|
||||
|
@ -293,7 +293,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) {
|
|||
}
|
||||
b.StartTimer()
|
||||
if full {
|
||||
if _, err := MD5FromFile(f); err != nil {
|
||||
if _, err := MD5FromReader(f); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
|
|
|
@ -339,6 +339,16 @@ Publish 2: {{ $cssPublish2.Permalink }}
|
|||
assert.False(b.CheckExists("public/inline.min.css"), "Inline content should not be copied to /public")
|
||||
}},
|
||||
|
||||
{"unmarshal", func() bool { return true }, func(b *sitesBuilder) {
|
||||
b.WithTemplates("home.html", `
|
||||
{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }}
|
||||
Slogan: {{ $toml.slogan }}
|
||||
|
||||
`)
|
||||
}, func(b *sitesBuilder) {
|
||||
b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`)
|
||||
}},
|
||||
|
||||
{"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) {
|
||||
}},
|
||||
}
|
||||
|
|
|
@ -135,6 +135,8 @@ var (
|
|||
XMLType = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter}
|
||||
SVGType = Type{MainType: "image", SubType: "svg", mimeSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter}
|
||||
TextType = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter}
|
||||
TOMLType = Type{MainType: "application", SubType: "toml", Suffixes: []string{"toml"}, Delimiter: defaultDelimiter}
|
||||
YAMLType = Type{MainType: "application", SubType: "yaml", Suffixes: []string{"yaml", "yml"}, Delimiter: defaultDelimiter}
|
||||
|
||||
OctetType = Type{MainType: "application", SubType: "octet-stream"}
|
||||
)
|
||||
|
@ -154,6 +156,8 @@ var DefaultTypes = Types{
|
|||
SVGType,
|
||||
TextType,
|
||||
OctetType,
|
||||
YAMLType,
|
||||
TOMLType,
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
|
@ -39,6 +39,8 @@ func TestDefaultTypes(t *testing.T) {
|
|||
{SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"},
|
||||
{TextType, "text", "plain", "txt", "text/plain", "text/plain"},
|
||||
{XMLType, "application", "xml", "xml", "application/xml", "application/xml"},
|
||||
{TOMLType, "application", "toml", "toml", "application/toml", "application/toml"},
|
||||
{YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"},
|
||||
} {
|
||||
require.Equal(t, test.expectedMainType, test.tp.MainType)
|
||||
require.Equal(t, test.expectedSubType, test.tp.SubType)
|
||||
|
@ -50,6 +52,8 @@ func TestDefaultTypes(t *testing.T) {
|
|||
|
||||
}
|
||||
|
||||
require.Equal(t, 15, len(DefaultTypes))
|
||||
|
||||
}
|
||||
|
||||
func TestGetByType(t *testing.T) {
|
||||
|
|
|
@ -17,6 +17,8 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/media"
|
||||
|
||||
"github.com/gohugoio/hugo/parser/pageparser"
|
||||
)
|
||||
|
||||
|
@ -55,6 +57,18 @@ func FormatFromString(formatStr string) Format {
|
|||
|
||||
}
|
||||
|
||||
// FormatFromMediaType gets the Format given a MIME type, empty string
|
||||
// if unknown.
|
||||
func FormatFromMediaType(m media.Type) Format {
|
||||
for _, suffix := range m.Suffixes {
|
||||
if f := FormatFromString(suffix); f != "" {
|
||||
return f
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// FormatFromFrontMatterType will return empty if not supported.
|
||||
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
|
||||
switch typ {
|
||||
|
@ -70,3 +84,39 @@ func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
|
|||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// FormatFromContentString tries to detect the format (JSON, YAML or TOML)
|
||||
// in the given string.
|
||||
// It return an empty string if no format could be detected.
|
||||
func FormatFromContentString(data string) Format {
|
||||
jsonIdx := strings.Index(data, "{")
|
||||
yamlIdx := strings.Index(data, ":")
|
||||
tomlIdx := strings.Index(data, "=")
|
||||
|
||||
if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) {
|
||||
return JSON
|
||||
}
|
||||
|
||||
if isLowerIndexThan(yamlIdx, tomlIdx) {
|
||||
return YAML
|
||||
}
|
||||
|
||||
if tomlIdx != -1 {
|
||||
return TOML
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func isLowerIndexThan(first int, others ...int) bool {
|
||||
if first == -1 {
|
||||
return false
|
||||
}
|
||||
for _, other := range others {
|
||||
if other != -1 && other < first {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
|
|
@ -17,6 +17,8 @@ import (
|
|||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/media"
|
||||
|
||||
"github.com/gohugoio/hugo/parser/pageparser"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
@ -41,6 +43,21 @@ func TestFormatFromString(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFormatFromMediaType(t *testing.T) {
|
||||
assert := require.New(t)
|
||||
for i, test := range []struct {
|
||||
m media.Type
|
||||
expect Format
|
||||
}{
|
||||
{media.JSONType, JSON},
|
||||
{media.YAMLType, YAML},
|
||||
{media.TOMLType, TOML},
|
||||
{media.CalendarType, ""},
|
||||
} {
|
||||
assert.Equal(test.expect, FormatFromMediaType(test.m), fmt.Sprintf("t%d", i))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatFromFrontMatterType(t *testing.T) {
|
||||
assert := require.New(t)
|
||||
for i, test := range []struct {
|
||||
|
@ -56,3 +73,28 @@ func TestFormatFromFrontMatterType(t *testing.T) {
|
|||
assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatFromContentString(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert := require.New(t)
|
||||
|
||||
for i, test := range []struct {
|
||||
data string
|
||||
expect interface{}
|
||||
}{
|
||||
{`foo = "bar"`, TOML},
|
||||
{` foo = "bar"`, TOML},
|
||||
{`foo="bar"`, TOML},
|
||||
{`foo: "bar"`, YAML},
|
||||
{`foo:"bar"`, YAML},
|
||||
{`{ "foo": "bar"`, JSON},
|
||||
{`asdfasdf`, Format("")},
|
||||
{``, Format("")},
|
||||
} {
|
||||
errMsg := fmt.Sprintf("[%d] %s", i, test.data)
|
||||
|
||||
result := FormatFromContentString(test.data)
|
||||
|
||||
assert.Equal(test.expect, result, errMsg)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -50,6 +50,7 @@ var (
|
|||
_ ResourcesLanguageMerger = (*Resources)(nil)
|
||||
_ permalinker = (*genericResource)(nil)
|
||||
_ collections.Slicer = (*genericResource)(nil)
|
||||
_ Identifier = (*genericResource)(nil)
|
||||
)
|
||||
|
||||
var noData = make(map[string]interface{})
|
||||
|
@ -76,6 +77,8 @@ type Cloner interface {
|
|||
|
||||
// Resource represents a linkable resource, i.e. a content page, image etc.
|
||||
type Resource interface {
|
||||
resourceBase
|
||||
|
||||
// Permalink represents the absolute link to this resource.
|
||||
Permalink() string
|
||||
|
||||
|
@ -87,9 +90,6 @@ type Resource interface {
|
|||
// For content pages, this value is "page".
|
||||
ResourceType() string
|
||||
|
||||
// MediaType is this resource's MIME type.
|
||||
MediaType() media.Type
|
||||
|
||||
// Name is the logical name of this resource. This can be set in the front matter
|
||||
// metadata for this resource. If not set, Hugo will assign a value.
|
||||
// This will in most cases be the base filename.
|
||||
|
@ -109,6 +109,13 @@ type Resource interface {
|
|||
Params() map[string]interface{}
|
||||
}
|
||||
|
||||
// resourceBase pulls out the minimal set of operations to define a Resource,
|
||||
// to simplify testing etc.
|
||||
type resourceBase interface {
|
||||
// MediaType is this resource's MIME type.
|
||||
MediaType() media.Type
|
||||
}
|
||||
|
||||
// ResourcesLanguageMerger describes an interface for merging resources from a
|
||||
// different language.
|
||||
type ResourcesLanguageMerger interface {
|
||||
|
@ -121,12 +128,17 @@ type translatedResource interface {
|
|||
TranslationKey() string
|
||||
}
|
||||
|
||||
// Identifier identifies a resource.
|
||||
type Identifier interface {
|
||||
Key() string
|
||||
}
|
||||
|
||||
// ContentResource represents a Resource that provides a way to get to its content.
|
||||
// Most Resource types in Hugo implements this interface, including Page.
|
||||
// This should be used with care, as it will read the file content into memory, but it
|
||||
// should be cached as effectively as possible by the implementation.
|
||||
type ContentResource interface {
|
||||
Resource
|
||||
resourceBase
|
||||
|
||||
// Content returns this resource's content. It will be equivalent to reading the content
|
||||
// that RelPermalink points to in the published folder.
|
||||
|
@ -143,7 +155,7 @@ type OpenReadSeekCloser func() (hugio.ReadSeekCloser, error)
|
|||
|
||||
// ReadSeekCloserResource is a Resource that supports loading its content.
|
||||
type ReadSeekCloserResource interface {
|
||||
Resource
|
||||
resourceBase
|
||||
ReadSeekCloser() (hugio.ReadSeekCloser, error)
|
||||
}
|
||||
|
||||
|
@ -716,6 +728,10 @@ func (l *genericResource) RelPermalink() string {
|
|||
return l.relPermalinkFor(l.relTargetDirFile.path())
|
||||
}
|
||||
|
||||
func (l *genericResource) Key() string {
|
||||
return l.relTargetDirFile.path()
|
||||
}
|
||||
|
||||
func (l *genericResource) relPermalinkFor(target string) string {
|
||||
return l.relPermalinkForRel(target, false)
|
||||
|
||||
|
|
|
@ -50,6 +50,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) {
|
|||
|
||||
assert.Equal("https://example.com/foo/foo.css", r.Permalink())
|
||||
assert.Equal("/foo/foo.css", r.RelPermalink())
|
||||
assert.Equal("foo.css", r.Key())
|
||||
assert.Equal("css", r.ResourceType())
|
||||
}
|
||||
|
||||
|
|
|
@ -38,6 +38,7 @@ var (
|
|||
_ ContentResource = (*transformedResource)(nil)
|
||||
_ ReadSeekCloserResource = (*transformedResource)(nil)
|
||||
_ collections.Slicer = (*transformedResource)(nil)
|
||||
_ Identifier = (*transformedResource)(nil)
|
||||
)
|
||||
|
||||
func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) {
|
||||
|
@ -249,6 +250,13 @@ func (r *transformedResource) MediaType() media.Type {
|
|||
return m
|
||||
}
|
||||
|
||||
func (r *transformedResource) Key() string {
|
||||
if err := r.initTransform(false, false); err != nil {
|
||||
return ""
|
||||
}
|
||||
return r.linker.relPermalinkFor(r.Target)
|
||||
}
|
||||
|
||||
func (r *transformedResource) Permalink() string {
|
||||
if err := r.initTransform(false, true); err != nil {
|
||||
return ""
|
||||
|
@ -481,8 +489,8 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) {
|
|||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (r *transformedResource) initTransform(setContent, publish bool) error {
|
||||
r.transformInit.Do(func() {
|
||||
r.published = publish
|
||||
|
|
|
@ -95,6 +95,14 @@ func init() {
|
|||
},
|
||||
)
|
||||
|
||||
ns.AddMethodMapping(ctx.Unmarshal,
|
||||
[]string{"unmarshal"},
|
||||
[][2]string{
|
||||
{`{{ "hello = \"Hello World\"" | transform.Unmarshal }}`, "map[hello:Hello World]"},
|
||||
{`{{ "hello = \"Hello World\"" | resources.FromString "data/greetings.toml" | transform.Unmarshal }}`, "map[hello:Hello World]"},
|
||||
},
|
||||
)
|
||||
|
||||
return ns
|
||||
|
||||
}
|
||||
|
|
|
@ -2,9 +2,10 @@ package transform
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/gohugoio/hugo/parser"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
"github.com/spf13/cast"
|
||||
|
@ -34,9 +35,9 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
|
|||
return "", err
|
||||
}
|
||||
|
||||
fromFormat, err := detectFormat(from)
|
||||
if err != nil {
|
||||
return "", err
|
||||
fromFormat := metadecoders.FormatFromContentString(from)
|
||||
if fromFormat == "" {
|
||||
return "", errors.New("failed to detect format from content")
|
||||
}
|
||||
|
||||
meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
|
||||
|
@ -56,24 +57,3 @@ func toFormatMark(format string) (metadecoders.Format, error) {
|
|||
|
||||
return "", errors.New("failed to detect target data serialization format")
|
||||
}
|
||||
|
||||
func detectFormat(data string) (metadecoders.Format, error) {
|
||||
jsonIdx := strings.Index(data, "{")
|
||||
yamlIdx := strings.Index(data, ":")
|
||||
tomlIdx := strings.Index(data, "=")
|
||||
|
||||
if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {
|
||||
return metadecoders.JSON, nil
|
||||
}
|
||||
|
||||
if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {
|
||||
return metadecoders.YAML, nil
|
||||
}
|
||||
|
||||
if tomlIdx != -1 {
|
||||
return metadecoders.TOML, nil
|
||||
}
|
||||
|
||||
return "", errors.New("failed to detect data serialization format")
|
||||
|
||||
}
|
||||
|
|
|
@ -18,7 +18,6 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -171,34 +170,3 @@ func TestTestRemarshalError(t *testing.T) {
|
|||
assert.Error(err)
|
||||
|
||||
}
|
||||
|
||||
func TestRemarshalDetectFormat(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert := require.New(t)
|
||||
|
||||
for i, test := range []struct {
|
||||
data string
|
||||
expect interface{}
|
||||
}{
|
||||
{`foo = "bar"`, metadecoders.TOML},
|
||||
{` foo = "bar"`, metadecoders.TOML},
|
||||
{`foo="bar"`, metadecoders.TOML},
|
||||
{`foo: "bar"`, metadecoders.YAML},
|
||||
{`foo:"bar"`, metadecoders.YAML},
|
||||
{`{ "foo": "bar"`, metadecoders.JSON},
|
||||
{`asdfasdf`, false},
|
||||
{``, false},
|
||||
} {
|
||||
errMsg := fmt.Sprintf("[%d] %s", i, test.data)
|
||||
|
||||
result, err := detectFormat(test.data)
|
||||
|
||||
if b, ok := test.expect.(bool); ok && !b {
|
||||
assert.Error(err, errMsg)
|
||||
continue
|
||||
}
|
||||
|
||||
assert.NoError(err, errMsg)
|
||||
assert.Equal(test.expect, result)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,6 +19,8 @@ import (
|
|||
"html"
|
||||
"html/template"
|
||||
|
||||
"github.com/gohugoio/hugo/cache/namedmemcache"
|
||||
|
||||
"github.com/gohugoio/hugo/deps"
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/spf13/cast"
|
||||
|
@ -26,14 +28,22 @@ import (
|
|||
|
||||
// New returns a new instance of the transform-namespaced template functions.
|
||||
func New(deps *deps.Deps) *Namespace {
|
||||
cache := namedmemcache.New()
|
||||
deps.BuildStartListeners.Add(
|
||||
func() {
|
||||
cache.Clear()
|
||||
})
|
||||
|
||||
return &Namespace{
|
||||
deps: deps,
|
||||
cache: cache,
|
||||
deps: deps,
|
||||
}
|
||||
}
|
||||
|
||||
// Namespace provides template functions for the "transform" namespace.
|
||||
type Namespace struct {
|
||||
deps *deps.Deps
|
||||
cache *namedmemcache.Cache
|
||||
deps *deps.Deps
|
||||
}
|
||||
|
||||
// Emojify returns a copy of s with all emoji codes replaced with actual emojis.
|
||||
|
|
|
@ -34,7 +34,6 @@ func TestEmojify(t *testing.T) {
|
|||
t.Parallel()
|
||||
|
||||
v := viper.New()
|
||||
v.Set("contentDir", "content")
|
||||
ns := New(newDeps(v))
|
||||
|
||||
for i, test := range []struct {
|
||||
|
@ -215,7 +214,6 @@ func TestPlainify(t *testing.T) {
|
|||
t.Parallel()
|
||||
|
||||
v := viper.New()
|
||||
v.Set("contentDir", "content")
|
||||
ns := New(newDeps(v))
|
||||
|
||||
for i, test := range []struct {
|
||||
|
@ -241,8 +239,11 @@ func TestPlainify(t *testing.T) {
|
|||
}
|
||||
|
||||
func newDeps(cfg config.Provider) *deps.Deps {
|
||||
cfg.Set("contentDir", "content")
|
||||
cfg.Set("i18nDir", "i18n")
|
||||
|
||||
l := langs.NewLanguage("en", cfg)
|
||||
l.Set("i18nDir", "i18n")
|
||||
|
||||
cs, err := helpers.NewContentSpec(l)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
98
tpl/transform/unmarshal.go
Normal file
98
tpl/transform/unmarshal.go
Normal file
|
@ -0,0 +1,98 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package transform
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/gohugoio/hugo/common/hugio"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
"github.com/gohugoio/hugo/resource"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
// Unmarshal unmarshals the data given, which can be either a string
|
||||
// or a Resource. Supported formats are JSON, TOML and YAML.
|
||||
func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) {
|
||||
|
||||
// All the relevant Resource types implements ReadSeekCloserResource,
|
||||
// which should be the most effective way to get the content.
|
||||
if r, ok := data.(resource.ReadSeekCloserResource); ok {
|
||||
var key string
|
||||
var reader hugio.ReadSeekCloser
|
||||
|
||||
if k, ok := r.(resource.Identifier); ok {
|
||||
key = k.Key()
|
||||
}
|
||||
|
||||
if key == "" {
|
||||
reader, err := r.ReadSeekCloser()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
key, err = helpers.MD5FromReader(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader.Seek(0, 0)
|
||||
}
|
||||
|
||||
return ns.cache.GetOrCreate(key, func() (interface{}, error) {
|
||||
f := metadecoders.FormatFromMediaType(r.MediaType())
|
||||
if f == "" {
|
||||
return nil, errors.Errorf("MIME %q not supported", r.MediaType())
|
||||
}
|
||||
|
||||
if reader == nil {
|
||||
var err error
|
||||
reader, err = r.ReadSeekCloser()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
}
|
||||
|
||||
b, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return metadecoders.Unmarshal(b, f)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
dataStr, err := cast.ToStringE(data)
|
||||
if err != nil {
|
||||
return nil, errors.Errorf("type %T not supported", data)
|
||||
}
|
||||
|
||||
key := helpers.MD5String(dataStr)
|
||||
|
||||
return ns.cache.GetOrCreate(key, func() (interface{}, error) {
|
||||
f := metadecoders.FormatFromContentString(dataStr)
|
||||
if f == "" {
|
||||
return nil, errors.New("unknown format")
|
||||
}
|
||||
|
||||
return metadecoders.Unmarshal([]byte(dataStr), f)
|
||||
})
|
||||
}
|
185
tpl/transform/unmarshal_test.go
Normal file
185
tpl/transform/unmarshal_test.go
Normal file
|
@ -0,0 +1,185 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package transform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/common/hugio"
|
||||
|
||||
"github.com/gohugoio/hugo/media"
|
||||
|
||||
"github.com/gohugoio/hugo/resource"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testJSON = `
|
||||
|
||||
{
|
||||
"ROOT_KEY": {
|
||||
"title": "example glossary",
|
||||
"GlossDiv": {
|
||||
"title": "S",
|
||||
"GlossList": {
|
||||
"GlossEntry": {
|
||||
"ID": "SGML",
|
||||
"SortAs": "SGML",
|
||||
"GlossTerm": "Standard Generalized Markup Language",
|
||||
"Acronym": "SGML",
|
||||
"Abbrev": "ISO 8879:1986",
|
||||
"GlossDef": {
|
||||
"para": "A meta-markup language, used to create markup languages such as DocBook.",
|
||||
"GlossSeeAlso": ["GML", "XML"]
|
||||
},
|
||||
"GlossSee": "markup"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
`
|
||||
)
|
||||
|
||||
var _ resource.ReadSeekCloserResource = (*testContentResource)(nil)
|
||||
|
||||
type testContentResource struct {
|
||||
content string
|
||||
mime media.Type
|
||||
|
||||
key string
|
||||
}
|
||||
|
||||
func (t testContentResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
|
||||
return hugio.NewReadSeekerNoOpCloserFromString(t.content), nil
|
||||
}
|
||||
|
||||
func (t testContentResource) MediaType() media.Type {
|
||||
return t.mime
|
||||
}
|
||||
|
||||
func (t testContentResource) Key() string {
|
||||
return t.key
|
||||
}
|
||||
|
||||
func TestUnmarshal(t *testing.T) {
|
||||
|
||||
v := viper.New()
|
||||
ns := New(newDeps(v))
|
||||
assert := require.New(t)
|
||||
|
||||
assertSlogan := func(m map[string]interface{}) {
|
||||
assert.Equal("Hugo Rocks!", m["slogan"])
|
||||
}
|
||||
|
||||
for i, test := range []struct {
|
||||
data interface{}
|
||||
expect interface{}
|
||||
}{
|
||||
{`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) {
|
||||
assertSlogan(m)
|
||||
}},
|
||||
{`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) {
|
||||
assertSlogan(m)
|
||||
}},
|
||||
{`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) {
|
||||
assertSlogan(m)
|
||||
}},
|
||||
{testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) {
|
||||
assertSlogan(m)
|
||||
}},
|
||||
{testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) {
|
||||
assertSlogan(m)
|
||||
}},
|
||||
{testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) {
|
||||
assertSlogan(m)
|
||||
}},
|
||||
// errors
|
||||
{"thisisnotavaliddataformat", false},
|
||||
{testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false},
|
||||
{testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false},
|
||||
{"thisisnotavaliddataformat", false},
|
||||
{`{ notjson }`, false},
|
||||
{tstNoStringer{}, false},
|
||||
} {
|
||||
errMsg := fmt.Sprintf("[%d]", i)
|
||||
|
||||
result, err := ns.Unmarshal(test.data)
|
||||
|
||||
if b, ok := test.expect.(bool); ok && !b {
|
||||
assert.Error(err, errMsg)
|
||||
} else if fn, ok := test.expect.(func(m map[string]interface{})); ok {
|
||||
assert.NoError(err, errMsg)
|
||||
m, ok := result.(map[string]interface{})
|
||||
assert.True(ok, errMsg)
|
||||
fn(m)
|
||||
} else {
|
||||
assert.NoError(err, errMsg)
|
||||
assert.Equal(test.expect, result, errMsg)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalString(b *testing.B) {
|
||||
v := viper.New()
|
||||
ns := New(newDeps(v))
|
||||
|
||||
const numJsons = 100
|
||||
|
||||
var jsons [numJsons]string
|
||||
for i := 0; i < numJsons; i++ {
|
||||
jsons[i] = strings.Replace(testJSON, "ROOT_KEY", fmt.Sprintf("root%d", i), 1)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if result == nil {
|
||||
b.Fatal("no result")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalResource(b *testing.B) {
|
||||
v := viper.New()
|
||||
ns := New(newDeps(v))
|
||||
|
||||
const numJsons = 100
|
||||
|
||||
var jsons [numJsons]testContentResource
|
||||
for i := 0; i < numJsons; i++ {
|
||||
key := fmt.Sprintf("root%d", i)
|
||||
jsons[i] = testContentResource{key: key, content: strings.Replace(testJSON, "ROOT_KEY", key, 1), mime: media.JSONType}
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)])
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if result == nil {
|
||||
b.Fatal("no result")
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue