diff --git a/cache/namedmemcache/named_cache.go b/cache/namedmemcache/named_cache.go new file mode 100644 index 000000000..18fbea391 --- /dev/null +++ b/cache/namedmemcache/named_cache.go @@ -0,0 +1,84 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package namedmemcache provides a memory cache with a named lock. This is suitable +// for situations where creating the cached resource can be time consuming or otherwise +// resource hungry, or in situations where a "once only per key" is a requirement. +package namedmemcache + +import ( + "sync" + + "github.com/BurntSushi/locker" +) + +// Cache holds the cached values. +type Cache struct { + nlocker *locker.Locker + cache map[string]cacheEntry + mu sync.RWMutex +} + +type cacheEntry struct { + value interface{} + err error +} + +// New creates a new cache. +func New() *Cache { + return &Cache{ + nlocker: locker.NewLocker(), + cache: make(map[string]cacheEntry), + } +} + +// Clear clears the cache state. +func (c *Cache) Clear() { + c.mu.Lock() + defer c.mu.Unlock() + + c.cache = make(map[string]cacheEntry) + c.nlocker = locker.NewLocker() + +} + +// GetOrCreate tries to get the value with the given cache key, if not found +// create will be called and cached. +// This method is thread safe. It also guarantees that the create func for a given +// key is invoced only once for this cache. +func (c *Cache) GetOrCreate(key string, create func() (interface{}, error)) (interface{}, error) { + c.mu.RLock() + entry, found := c.cache[key] + c.mu.RUnlock() + + if found { + return entry.value, entry.err + } + + c.nlocker.Lock(key) + defer c.nlocker.Unlock(key) + + // Double check + if entry, found := c.cache[key]; found { + return entry.value, entry.err + } + + // Create it. + value, err := create() + + c.mu.Lock() + c.cache[key] = cacheEntry{value: value, err: err} + c.mu.Unlock() + + return value, err +} diff --git a/cache/namedmemcache/named_cache_test.go b/cache/namedmemcache/named_cache_test.go new file mode 100644 index 000000000..cf64aa210 --- /dev/null +++ b/cache/namedmemcache/named_cache_test.go @@ -0,0 +1,80 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package namedmemcache + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNamedCache(t *testing.T) { + t.Parallel() + assert := require.New(t) + + cache := New() + + counter := 0 + create := func() (interface{}, error) { + counter++ + return counter, nil + } + + for i := 0; i < 5; i++ { + v1, err := cache.GetOrCreate("a1", create) + assert.NoError(err) + assert.Equal(1, v1) + v2, err := cache.GetOrCreate("a2", create) + assert.NoError(err) + assert.Equal(2, v2) + } + + cache.Clear() + + v3, err := cache.GetOrCreate("a2", create) + assert.NoError(err) + assert.Equal(3, v3) +} + +func TestNamedCacheConcurrent(t *testing.T) { + t.Parallel() + + assert := require.New(t) + + var wg sync.WaitGroup + + cache := New() + + create := func(i int) func() (interface{}, error) { + return func() (interface{}, error) { + return i, nil + } + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 100; j++ { + id := fmt.Sprintf("id%d", j) + v, err := cache.GetOrCreate(id, create(j)) + assert.NoError(err) + assert.Equal(j, v) + } + }() + } + wg.Wait() +} diff --git a/deps/deps.go b/deps/deps.go index 46f4f7730..7fba0e153 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -123,6 +123,9 @@ type Listeners struct { // Add adds a function to a Listeners instance. func (b *Listeners) Add(f func()) { + if b == nil { + return + } b.Lock() defer b.Unlock() b.listeners = append(b.listeners, f) @@ -192,6 +195,14 @@ func New(cfg DepsCfg) (*Deps, error) { fs = hugofs.NewDefault(cfg.Language) } + if cfg.MediaTypes == nil { + cfg.MediaTypes = media.DefaultTypes + } + + if cfg.OutputFormats == nil { + cfg.OutputFormats = output.DefaultFormats + } + ps, err := helpers.NewPathSpec(fs, cfg.Language) if err != nil { diff --git a/helpers/general.go b/helpers/general.go index cfabab5a9..00caf1ecc 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -394,11 +394,10 @@ func MD5FromFileFast(r io.ReadSeeker) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } -// MD5FromFile creates a MD5 hash from the given file. -// It will not close the file. -func MD5FromFile(f afero.File) (string, error) { +// MD5FromReader creates a MD5 hash from the given reader. +func MD5FromReader(r io.Reader) (string, error) { h := md5.New() - if _, err := io.Copy(h, f); err != nil { + if _, err := io.Copy(h, r); err != nil { return "", nil } return hex.EncodeToString(h.Sum(nil)), nil diff --git a/helpers/general_test.go b/helpers/general_test.go index 08fe4890e..1279df439 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -272,7 +272,7 @@ func TestFastMD5FromFile(t *testing.T) { req.NoError(err) req.NotEqual(m3, m4) - m5, err := MD5FromFile(bf2) + m5, err := MD5FromReader(bf2) req.NoError(err) req.NotEqual(m4, m5) } @@ -293,7 +293,7 @@ func BenchmarkMD5FromFileFast(b *testing.B) { } b.StartTimer() if full { - if _, err := MD5FromFile(f); err != nil { + if _, err := MD5FromReader(f); err != nil { b.Fatal(err) } } else { diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 66a0a7ce6..74129dc17 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -339,6 +339,16 @@ Publish 2: {{ $cssPublish2.Permalink }} assert.False(b.CheckExists("public/inline.min.css"), "Inline content should not be copied to /public") }}, + {"unmarshal", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", ` +{{ $toml := "slogan = \"Hugo Rocks!\"" | resources.FromString "slogan.toml" | transform.Unmarshal }} +Slogan: {{ $toml.slogan }} + +`) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `Slogan: Hugo Rocks!`) + }}, + {"template", func() bool { return true }, func(b *sitesBuilder) {}, func(b *sitesBuilder) { }}, } diff --git a/media/mediaType.go b/media/mediaType.go index 9f7673ecc..01a6b9582 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -135,6 +135,8 @@ var ( XMLType = Type{MainType: "application", SubType: "xml", Suffixes: []string{"xml"}, Delimiter: defaultDelimiter} SVGType = Type{MainType: "image", SubType: "svg", mimeSuffix: "xml", Suffixes: []string{"svg"}, Delimiter: defaultDelimiter} TextType = Type{MainType: "text", SubType: "plain", Suffixes: []string{"txt"}, Delimiter: defaultDelimiter} + TOMLType = Type{MainType: "application", SubType: "toml", Suffixes: []string{"toml"}, Delimiter: defaultDelimiter} + YAMLType = Type{MainType: "application", SubType: "yaml", Suffixes: []string{"yaml", "yml"}, Delimiter: defaultDelimiter} OctetType = Type{MainType: "application", SubType: "octet-stream"} ) @@ -154,6 +156,8 @@ var DefaultTypes = Types{ SVGType, TextType, OctetType, + YAMLType, + TOMLType, } func init() { diff --git a/media/mediaType_test.go b/media/mediaType_test.go index bf356582f..ea6499a14 100644 --- a/media/mediaType_test.go +++ b/media/mediaType_test.go @@ -39,6 +39,8 @@ func TestDefaultTypes(t *testing.T) { {SVGType, "image", "svg", "svg", "image/svg+xml", "image/svg+xml"}, {TextType, "text", "plain", "txt", "text/plain", "text/plain"}, {XMLType, "application", "xml", "xml", "application/xml", "application/xml"}, + {TOMLType, "application", "toml", "toml", "application/toml", "application/toml"}, + {YAMLType, "application", "yaml", "yaml", "application/yaml", "application/yaml"}, } { require.Equal(t, test.expectedMainType, test.tp.MainType) require.Equal(t, test.expectedSubType, test.tp.SubType) @@ -50,6 +52,8 @@ func TestDefaultTypes(t *testing.T) { } + require.Equal(t, 15, len(DefaultTypes)) + } func TestGetByType(t *testing.T) { diff --git a/parser/metadecoders/format.go b/parser/metadecoders/format.go index 3f5a8a5c1..4a30898fe 100644 --- a/parser/metadecoders/format.go +++ b/parser/metadecoders/format.go @@ -17,6 +17,8 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/parser/pageparser" ) @@ -55,6 +57,18 @@ func FormatFromString(formatStr string) Format { } +// FormatFromMediaType gets the Format given a MIME type, empty string +// if unknown. +func FormatFromMediaType(m media.Type) Format { + for _, suffix := range m.Suffixes { + if f := FormatFromString(suffix); f != "" { + return f + } + } + + return "" +} + // FormatFromFrontMatterType will return empty if not supported. func FormatFromFrontMatterType(typ pageparser.ItemType) Format { switch typ { @@ -70,3 +84,39 @@ func FormatFromFrontMatterType(typ pageparser.ItemType) Format { return "" } } + +// FormatFromContentString tries to detect the format (JSON, YAML or TOML) +// in the given string. +// It return an empty string if no format could be detected. +func FormatFromContentString(data string) Format { + jsonIdx := strings.Index(data, "{") + yamlIdx := strings.Index(data, ":") + tomlIdx := strings.Index(data, "=") + + if isLowerIndexThan(jsonIdx, yamlIdx, tomlIdx) { + return JSON + } + + if isLowerIndexThan(yamlIdx, tomlIdx) { + return YAML + } + + if tomlIdx != -1 { + return TOML + } + + return "" +} + +func isLowerIndexThan(first int, others ...int) bool { + if first == -1 { + return false + } + for _, other := range others { + if other != -1 && other < first { + return false + } + } + + return true +} diff --git a/parser/metadecoders/format_test.go b/parser/metadecoders/format_test.go index a22e84f98..6243b3f1e 100644 --- a/parser/metadecoders/format_test.go +++ b/parser/metadecoders/format_test.go @@ -17,6 +17,8 @@ import ( "fmt" "testing" + "github.com/gohugoio/hugo/media" + "github.com/gohugoio/hugo/parser/pageparser" "github.com/stretchr/testify/require" @@ -41,6 +43,21 @@ func TestFormatFromString(t *testing.T) { } } +func TestFormatFromMediaType(t *testing.T) { + assert := require.New(t) + for i, test := range []struct { + m media.Type + expect Format + }{ + {media.JSONType, JSON}, + {media.YAMLType, YAML}, + {media.TOMLType, TOML}, + {media.CalendarType, ""}, + } { + assert.Equal(test.expect, FormatFromMediaType(test.m), fmt.Sprintf("t%d", i)) + } +} + func TestFormatFromFrontMatterType(t *testing.T) { assert := require.New(t) for i, test := range []struct { @@ -56,3 +73,28 @@ func TestFormatFromFrontMatterType(t *testing.T) { assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i)) } } + +func TestFormatFromContentString(t *testing.T) { + t.Parallel() + assert := require.New(t) + + for i, test := range []struct { + data string + expect interface{} + }{ + {`foo = "bar"`, TOML}, + {` foo = "bar"`, TOML}, + {`foo="bar"`, TOML}, + {`foo: "bar"`, YAML}, + {`foo:"bar"`, YAML}, + {`{ "foo": "bar"`, JSON}, + {`asdfasdf`, Format("")}, + {``, Format("")}, + } { + errMsg := fmt.Sprintf("[%d] %s", i, test.data) + + result := FormatFromContentString(test.data) + + assert.Equal(test.expect, result, errMsg) + } +} diff --git a/resource/resource.go b/resource/resource.go index a8f9dde06..0f5a43648 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -50,6 +50,7 @@ var ( _ ResourcesLanguageMerger = (*Resources)(nil) _ permalinker = (*genericResource)(nil) _ collections.Slicer = (*genericResource)(nil) + _ Identifier = (*genericResource)(nil) ) var noData = make(map[string]interface{}) @@ -76,6 +77,8 @@ type Cloner interface { // Resource represents a linkable resource, i.e. a content page, image etc. type Resource interface { + resourceBase + // Permalink represents the absolute link to this resource. Permalink() string @@ -87,9 +90,6 @@ type Resource interface { // For content pages, this value is "page". ResourceType() string - // MediaType is this resource's MIME type. - MediaType() media.Type - // Name is the logical name of this resource. This can be set in the front matter // metadata for this resource. If not set, Hugo will assign a value. // This will in most cases be the base filename. @@ -109,6 +109,13 @@ type Resource interface { Params() map[string]interface{} } +// resourceBase pulls out the minimal set of operations to define a Resource, +// to simplify testing etc. +type resourceBase interface { + // MediaType is this resource's MIME type. + MediaType() media.Type +} + // ResourcesLanguageMerger describes an interface for merging resources from a // different language. type ResourcesLanguageMerger interface { @@ -121,12 +128,17 @@ type translatedResource interface { TranslationKey() string } +// Identifier identifies a resource. +type Identifier interface { + Key() string +} + // ContentResource represents a Resource that provides a way to get to its content. // Most Resource types in Hugo implements this interface, including Page. // This should be used with care, as it will read the file content into memory, but it // should be cached as effectively as possible by the implementation. type ContentResource interface { - Resource + resourceBase // Content returns this resource's content. It will be equivalent to reading the content // that RelPermalink points to in the published folder. @@ -143,7 +155,7 @@ type OpenReadSeekCloser func() (hugio.ReadSeekCloser, error) // ReadSeekCloserResource is a Resource that supports loading its content. type ReadSeekCloserResource interface { - Resource + resourceBase ReadSeekCloser() (hugio.ReadSeekCloser, error) } @@ -716,6 +728,10 @@ func (l *genericResource) RelPermalink() string { return l.relPermalinkFor(l.relTargetDirFile.path()) } +func (l *genericResource) Key() string { + return l.relTargetDirFile.path() +} + func (l *genericResource) relPermalinkFor(target string) string { return l.relPermalinkForRel(target, false) diff --git a/resource/resource_test.go b/resource/resource_test.go index b76f0a604..b3f6035b6 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -50,6 +50,7 @@ func TestGenericResourceWithLinkFacory(t *testing.T) { assert.Equal("https://example.com/foo/foo.css", r.Permalink()) assert.Equal("/foo/foo.css", r.RelPermalink()) + assert.Equal("foo.css", r.Key()) assert.Equal("css", r.ResourceType()) } diff --git a/resource/transform.go b/resource/transform.go index 796c7ee23..bd59d0658 100644 --- a/resource/transform.go +++ b/resource/transform.go @@ -38,6 +38,7 @@ var ( _ ContentResource = (*transformedResource)(nil) _ ReadSeekCloserResource = (*transformedResource)(nil) _ collections.Slicer = (*transformedResource)(nil) + _ Identifier = (*transformedResource)(nil) ) func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) { @@ -249,6 +250,13 @@ func (r *transformedResource) MediaType() media.Type { return m } +func (r *transformedResource) Key() string { + if err := r.initTransform(false, false); err != nil { + return "" + } + return r.linker.relPermalinkFor(r.Target) +} + func (r *transformedResource) Permalink() string { if err := r.initTransform(false, true); err != nil { return "" @@ -481,8 +489,8 @@ func (r *transformedResource) transform(setContent, publish bool) (err error) { } return nil - } + func (r *transformedResource) initTransform(setContent, publish bool) error { r.transformInit.Do(func() { r.published = publish diff --git a/tpl/transform/init.go b/tpl/transform/init.go index 86951c253..62cb0a9c3 100644 --- a/tpl/transform/init.go +++ b/tpl/transform/init.go @@ -95,6 +95,14 @@ func init() { }, ) + ns.AddMethodMapping(ctx.Unmarshal, + []string{"unmarshal"}, + [][2]string{ + {`{{ "hello = \"Hello World\"" | transform.Unmarshal }}`, "map[hello:Hello World]"}, + {`{{ "hello = \"Hello World\"" | resources.FromString "data/greetings.toml" | transform.Unmarshal }}`, "map[hello:Hello World]"}, + }, + ) + return ns } diff --git a/tpl/transform/remarshal.go b/tpl/transform/remarshal.go index fd0742b7f..144964f0a 100644 --- a/tpl/transform/remarshal.go +++ b/tpl/transform/remarshal.go @@ -2,9 +2,10 @@ package transform import ( "bytes" - "errors" "strings" + "github.com/pkg/errors" + "github.com/gohugoio/hugo/parser" "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/cast" @@ -34,9 +35,9 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error) return "", err } - fromFormat, err := detectFormat(from) - if err != nil { - return "", err + fromFormat := metadecoders.FormatFromContentString(from) + if fromFormat == "" { + return "", errors.New("failed to detect format from content") } meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat) @@ -56,24 +57,3 @@ func toFormatMark(format string) (metadecoders.Format, error) { return "", errors.New("failed to detect target data serialization format") } - -func detectFormat(data string) (metadecoders.Format, error) { - jsonIdx := strings.Index(data, "{") - yamlIdx := strings.Index(data, ":") - tomlIdx := strings.Index(data, "=") - - if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) { - return metadecoders.JSON, nil - } - - if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) { - return metadecoders.YAML, nil - } - - if tomlIdx != -1 { - return metadecoders.TOML, nil - } - - return "", errors.New("failed to detect data serialization format") - -} diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go index 1416afff3..07414ccb4 100644 --- a/tpl/transform/remarshal_test.go +++ b/tpl/transform/remarshal_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/parser/metadecoders" "github.com/spf13/viper" "github.com/stretchr/testify/require" ) @@ -171,34 +170,3 @@ func TestTestRemarshalError(t *testing.T) { assert.Error(err) } - -func TestRemarshalDetectFormat(t *testing.T) { - t.Parallel() - assert := require.New(t) - - for i, test := range []struct { - data string - expect interface{} - }{ - {`foo = "bar"`, metadecoders.TOML}, - {` foo = "bar"`, metadecoders.TOML}, - {`foo="bar"`, metadecoders.TOML}, - {`foo: "bar"`, metadecoders.YAML}, - {`foo:"bar"`, metadecoders.YAML}, - {`{ "foo": "bar"`, metadecoders.JSON}, - {`asdfasdf`, false}, - {``, false}, - } { - errMsg := fmt.Sprintf("[%d] %s", i, test.data) - - result, err := detectFormat(test.data) - - if b, ok := test.expect.(bool); ok && !b { - assert.Error(err, errMsg) - continue - } - - assert.NoError(err, errMsg) - assert.Equal(test.expect, result) - } -} diff --git a/tpl/transform/transform.go b/tpl/transform/transform.go index 777e31c3e..42e36eb0f 100644 --- a/tpl/transform/transform.go +++ b/tpl/transform/transform.go @@ -19,6 +19,8 @@ import ( "html" "html/template" + "github.com/gohugoio/hugo/cache/namedmemcache" + "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/helpers" "github.com/spf13/cast" @@ -26,14 +28,22 @@ import ( // New returns a new instance of the transform-namespaced template functions. func New(deps *deps.Deps) *Namespace { + cache := namedmemcache.New() + deps.BuildStartListeners.Add( + func() { + cache.Clear() + }) + return &Namespace{ - deps: deps, + cache: cache, + deps: deps, } } // Namespace provides template functions for the "transform" namespace. type Namespace struct { - deps *deps.Deps + cache *namedmemcache.Cache + deps *deps.Deps } // Emojify returns a copy of s with all emoji codes replaced with actual emojis. diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index 34de4a6fd..a09ec6fbd 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -34,7 +34,6 @@ func TestEmojify(t *testing.T) { t.Parallel() v := viper.New() - v.Set("contentDir", "content") ns := New(newDeps(v)) for i, test := range []struct { @@ -215,7 +214,6 @@ func TestPlainify(t *testing.T) { t.Parallel() v := viper.New() - v.Set("contentDir", "content") ns := New(newDeps(v)) for i, test := range []struct { @@ -241,8 +239,11 @@ func TestPlainify(t *testing.T) { } func newDeps(cfg config.Provider) *deps.Deps { + cfg.Set("contentDir", "content") + cfg.Set("i18nDir", "i18n") + l := langs.NewLanguage("en", cfg) - l.Set("i18nDir", "i18n") + cs, err := helpers.NewContentSpec(l) if err != nil { panic(err) diff --git a/tpl/transform/unmarshal.go b/tpl/transform/unmarshal.go new file mode 100644 index 000000000..bf7db8920 --- /dev/null +++ b/tpl/transform/unmarshal.go @@ -0,0 +1,98 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transform + +import ( + "io/ioutil" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/helpers" + "github.com/gohugoio/hugo/parser/metadecoders" + "github.com/gohugoio/hugo/resource" + "github.com/pkg/errors" + + "github.com/spf13/cast" +) + +// Unmarshal unmarshals the data given, which can be either a string +// or a Resource. Supported formats are JSON, TOML and YAML. +func (ns *Namespace) Unmarshal(data interface{}) (interface{}, error) { + + // All the relevant Resource types implements ReadSeekCloserResource, + // which should be the most effective way to get the content. + if r, ok := data.(resource.ReadSeekCloserResource); ok { + var key string + var reader hugio.ReadSeekCloser + + if k, ok := r.(resource.Identifier); ok { + key = k.Key() + } + + if key == "" { + reader, err := r.ReadSeekCloser() + if err != nil { + return nil, err + } + defer reader.Close() + + key, err = helpers.MD5FromReader(reader) + if err != nil { + return nil, err + } + + reader.Seek(0, 0) + } + + return ns.cache.GetOrCreate(key, func() (interface{}, error) { + f := metadecoders.FormatFromMediaType(r.MediaType()) + if f == "" { + return nil, errors.Errorf("MIME %q not supported", r.MediaType()) + } + + if reader == nil { + var err error + reader, err = r.ReadSeekCloser() + if err != nil { + return nil, err + } + defer reader.Close() + } + + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + return metadecoders.Unmarshal(b, f) + }) + + } + + dataStr, err := cast.ToStringE(data) + if err != nil { + return nil, errors.Errorf("type %T not supported", data) + } + + key := helpers.MD5String(dataStr) + + return ns.cache.GetOrCreate(key, func() (interface{}, error) { + f := metadecoders.FormatFromContentString(dataStr) + if f == "" { + return nil, errors.New("unknown format") + } + + return metadecoders.Unmarshal([]byte(dataStr), f) + }) +} diff --git a/tpl/transform/unmarshal_test.go b/tpl/transform/unmarshal_test.go new file mode 100644 index 000000000..77e14edad --- /dev/null +++ b/tpl/transform/unmarshal_test.go @@ -0,0 +1,185 @@ +// Copyright 2018 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transform + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "github.com/gohugoio/hugo/common/hugio" + + "github.com/gohugoio/hugo/media" + + "github.com/gohugoio/hugo/resource" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +const ( + testJSON = ` + +{ + "ROOT_KEY": { + "title": "example glossary", + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +} + + ` +) + +var _ resource.ReadSeekCloserResource = (*testContentResource)(nil) + +type testContentResource struct { + content string + mime media.Type + + key string +} + +func (t testContentResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloserFromString(t.content), nil +} + +func (t testContentResource) MediaType() media.Type { + return t.mime +} + +func (t testContentResource) Key() string { + return t.key +} + +func TestUnmarshal(t *testing.T) { + + v := viper.New() + ns := New(newDeps(v)) + assert := require.New(t) + + assertSlogan := func(m map[string]interface{}) { + assert.Equal("Hugo Rocks!", m["slogan"]) + } + + for i, test := range []struct { + data interface{} + expect interface{} + }{ + {`{ "slogan": "Hugo Rocks!" }`, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {`slogan: "Hugo Rocks!"`, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {`slogan = "Hugo Rocks!"`, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {testContentResource{content: `slogan: "Hugo Rocks!"`, mime: media.YAMLType}, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {testContentResource{content: `{ "slogan": "Hugo Rocks!" }`, mime: media.JSONType}, func(m map[string]interface{}) { + assertSlogan(m) + }}, + {testContentResource{content: `slogan = "Hugo Rocks!"`, mime: media.TOMLType}, func(m map[string]interface{}) { + assertSlogan(m) + }}, + // errors + {"thisisnotavaliddataformat", false}, + {testContentResource{content: `invalid&toml"`, mime: media.TOMLType}, false}, + {testContentResource{content: `unsupported: MIME"`, mime: media.CalendarType}, false}, + {"thisisnotavaliddataformat", false}, + {`{ notjson }`, false}, + {tstNoStringer{}, false}, + } { + errMsg := fmt.Sprintf("[%d]", i) + + result, err := ns.Unmarshal(test.data) + + if b, ok := test.expect.(bool); ok && !b { + assert.Error(err, errMsg) + } else if fn, ok := test.expect.(func(m map[string]interface{})); ok { + assert.NoError(err, errMsg) + m, ok := result.(map[string]interface{}) + assert.True(ok, errMsg) + fn(m) + } else { + assert.NoError(err, errMsg) + assert.Equal(test.expect, result, errMsg) + } + + } +} + +func BenchmarkUnmarshalString(b *testing.B) { + v := viper.New() + ns := New(newDeps(v)) + + const numJsons = 100 + + var jsons [numJsons]string + for i := 0; i < numJsons; i++ { + jsons[i] = strings.Replace(testJSON, "ROOT_KEY", fmt.Sprintf("root%d", i), 1) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)]) + if err != nil { + b.Fatal(err) + } + if result == nil { + b.Fatal("no result") + } + } +} + +func BenchmarkUnmarshalResource(b *testing.B) { + v := viper.New() + ns := New(newDeps(v)) + + const numJsons = 100 + + var jsons [numJsons]testContentResource + for i := 0; i < numJsons; i++ { + key := fmt.Sprintf("root%d", i) + jsons[i] = testContentResource{key: key, content: strings.Replace(testJSON, "ROOT_KEY", key, 1), mime: media.JSONType} + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := ns.Unmarshal(jsons[rand.Intn(numJsons)]) + if err != nil { + b.Fatal(err) + } + if result == nil { + b.Fatal("no result") + } + } +}