2018-10-18 08:21:23 +00:00
|
|
|
// Copyright 2018 The Hugo Authors. All rights reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package metadecoders
|
|
|
|
|
|
|
|
import (
|
2018-12-23 09:40:32 +00:00
|
|
|
"bytes"
|
|
|
|
"encoding/csv"
|
2018-10-18 08:21:23 +00:00
|
|
|
"encoding/json"
|
2018-10-20 09:16:18 +00:00
|
|
|
"fmt"
|
2020-06-26 21:52:12 +00:00
|
|
|
"regexp"
|
2018-12-23 20:34:17 +00:00
|
|
|
"strings"
|
2018-10-18 08:21:23 +00:00
|
|
|
|
2018-10-23 06:54:10 +00:00
|
|
|
"github.com/gohugoio/hugo/common/herrors"
|
2019-06-04 10:21:25 +00:00
|
|
|
"github.com/niklasfasching/go-org/org"
|
2018-10-23 06:54:10 +00:00
|
|
|
|
2018-10-18 08:21:23 +00:00
|
|
|
"github.com/BurntSushi/toml"
|
|
|
|
"github.com/pkg/errors"
|
2018-11-15 08:28:02 +00:00
|
|
|
"github.com/spf13/afero"
|
2018-10-20 09:16:18 +00:00
|
|
|
"github.com/spf13/cast"
|
2019-06-04 10:21:25 +00:00
|
|
|
jww "github.com/spf13/jwalterweatherman"
|
2018-10-19 09:30:57 +00:00
|
|
|
yaml "gopkg.in/yaml.v2"
|
2018-10-18 08:21:23 +00:00
|
|
|
)
|
|
|
|
|
2018-12-23 09:40:32 +00:00
|
|
|
// Decoder provides some configuration options for the decoders.
|
|
|
|
type Decoder struct {
|
2018-12-23 20:08:12 +00:00
|
|
|
// Delimiter is the field delimiter used in the CSV decoder. It defaults to ','.
|
|
|
|
Delimiter rune
|
2018-12-23 09:40:32 +00:00
|
|
|
|
|
|
|
// Comment, if not 0, is the comment character ued in the CSV decoder. Lines beginning with the
|
|
|
|
// Comment character without preceding whitespace are ignored.
|
|
|
|
Comment rune
|
|
|
|
}
|
|
|
|
|
2018-12-23 20:34:17 +00:00
|
|
|
// OptionsKey is used in cache keys.
|
|
|
|
func (d Decoder) OptionsKey() string {
|
|
|
|
var sb strings.Builder
|
|
|
|
sb.WriteRune(d.Delimiter)
|
|
|
|
sb.WriteRune(d.Comment)
|
|
|
|
return sb.String()
|
|
|
|
}
|
|
|
|
|
2018-12-23 09:40:32 +00:00
|
|
|
// Default is a Decoder in its default configuration.
|
|
|
|
var Default = Decoder{
|
2018-12-23 20:08:12 +00:00
|
|
|
Delimiter: ',',
|
2018-12-23 09:40:32 +00:00
|
|
|
}
|
|
|
|
|
2018-10-18 08:21:23 +00:00
|
|
|
// UnmarshalToMap will unmarshall data in format f into a new map. This is
|
|
|
|
// what's needed for Hugo's front matter decoding.
|
2018-12-23 09:40:32 +00:00
|
|
|
func (d Decoder) UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
|
2018-10-18 08:21:23 +00:00
|
|
|
m := make(map[string]interface{})
|
|
|
|
if data == nil {
|
|
|
|
return m, nil
|
|
|
|
}
|
|
|
|
|
2020-06-30 14:11:05 +00:00
|
|
|
err := d.UnmarshalTo(data, f, &m)
|
2018-10-20 09:16:18 +00:00
|
|
|
|
|
|
|
return m, err
|
2018-11-15 08:28:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// UnmarshalFileToMap is the same as UnmarshalToMap, but reads the data from
|
|
|
|
// the given filename.
|
2018-12-23 09:40:32 +00:00
|
|
|
func (d Decoder) UnmarshalFileToMap(fs afero.Fs, filename string) (map[string]interface{}, error) {
|
2018-11-15 08:28:02 +00:00
|
|
|
format := FormatFromString(filename)
|
|
|
|
if format == "" {
|
|
|
|
return nil, errors.Errorf("%q is not a valid configuration format", filename)
|
|
|
|
}
|
2018-10-20 09:16:18 +00:00
|
|
|
|
2018-11-15 08:28:02 +00:00
|
|
|
data, err := afero.ReadFile(fs, filename)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2018-12-23 09:40:32 +00:00
|
|
|
return d.UnmarshalToMap(data, format)
|
2018-10-20 09:16:18 +00:00
|
|
|
}
|
|
|
|
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 07:16:58 +00:00
|
|
|
// UnmarshalStringTo tries to unmarshal data to a new instance of type typ.
|
|
|
|
func (d Decoder) UnmarshalStringTo(data string, typ interface{}) (interface{}, error) {
|
|
|
|
data = strings.TrimSpace(data)
|
|
|
|
// We only check for the possible types in YAML, JSON and TOML.
|
|
|
|
switch typ.(type) {
|
|
|
|
case string:
|
|
|
|
return data, nil
|
|
|
|
case map[string]interface{}:
|
|
|
|
format := d.FormatFromContentString(data)
|
|
|
|
return d.UnmarshalToMap([]byte(data), format)
|
|
|
|
case []interface{}:
|
|
|
|
// A standalone slice. Let YAML handle it.
|
|
|
|
return d.Unmarshal([]byte(data), YAML)
|
|
|
|
case bool:
|
|
|
|
return cast.ToBoolE(data)
|
|
|
|
case int:
|
|
|
|
return cast.ToIntE(data)
|
|
|
|
case int64:
|
|
|
|
return cast.ToInt64E(data)
|
|
|
|
case float64:
|
|
|
|
return cast.ToFloat64E(data)
|
|
|
|
default:
|
2020-12-16 11:11:32 +00:00
|
|
|
return nil, errors.Errorf("unmarshal: %T not supported", typ)
|
Add Hugo Modules
This commit implements Hugo Modules.
This is a broad subject, but some keywords include:
* A new `module` configuration section where you can import almost anything. You can configure both your own file mounts nd the file mounts of the modules you import. This is the new recommended way of configuring what you earlier put in `configDir`, `staticDir` etc. And it also allows you to mount folders in non-Hugo-projects, e.g. the `SCSS` folder in the Bootstrap GitHub project.
* A module consists of a set of mounts to the standard 7 component types in Hugo: `static`, `content`, `layouts`, `data`, `assets`, `i18n`, and `archetypes`. Yes, Theme Components can now include content, which should be very useful, especially in bigger multilingual projects.
* Modules not in your local file cache will be downloaded automatically and even "hot replaced" while the server is running.
* Hugo Modules supports and encourages semver versioned modules, and uses the minimal version selection algorithm to resolve versions.
* A new set of CLI commands are provided to manage all of this: `hugo mod init`, `hugo mod get`, `hugo mod graph`, `hugo mod tidy`, and `hugo mod vendor`.
All of the above is backed by Go Modules.
Fixes #5973
Fixes #5996
Fixes #6010
Fixes #5911
Fixes #5940
Fixes #6074
Fixes #6082
Fixes #6092
2019-05-03 07:16:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-20 09:16:18 +00:00
|
|
|
// Unmarshal will unmarshall data in format f into an interface{}.
|
|
|
|
// This is what's needed for Hugo's /data handling.
|
2018-12-23 09:40:32 +00:00
|
|
|
func (d Decoder) Unmarshal(data []byte, f Format) (interface{}, error) {
|
2018-10-20 09:16:18 +00:00
|
|
|
if data == nil {
|
2018-12-23 09:40:32 +00:00
|
|
|
switch f {
|
|
|
|
case CSV:
|
|
|
|
return make([][]string, 0), nil
|
|
|
|
default:
|
|
|
|
return make(map[string]interface{}), nil
|
|
|
|
}
|
2018-10-20 09:16:18 +00:00
|
|
|
}
|
|
|
|
var v interface{}
|
2020-06-30 14:11:05 +00:00
|
|
|
err := d.UnmarshalTo(data, f, &v)
|
2018-10-20 09:16:18 +00:00
|
|
|
|
|
|
|
return v, err
|
|
|
|
}
|
|
|
|
|
2020-06-30 14:11:05 +00:00
|
|
|
// UnmarshalTo unmarshals data in format f into v.
|
|
|
|
func (d Decoder) UnmarshalTo(data []byte, f Format, v interface{}) error {
|
2018-10-18 08:21:23 +00:00
|
|
|
var err error
|
|
|
|
|
|
|
|
switch f {
|
|
|
|
case ORG:
|
2019-06-04 10:21:25 +00:00
|
|
|
err = d.unmarshalORG(data, v)
|
2018-10-18 08:21:23 +00:00
|
|
|
case JSON:
|
2018-10-20 09:16:18 +00:00
|
|
|
err = json.Unmarshal(data, v)
|
2018-10-18 08:21:23 +00:00
|
|
|
case TOML:
|
2018-10-20 09:16:18 +00:00
|
|
|
err = toml.Unmarshal(data, v)
|
2018-10-18 08:21:23 +00:00
|
|
|
case YAML:
|
2018-10-20 09:16:18 +00:00
|
|
|
err = yaml.Unmarshal(data, v)
|
2018-10-20 17:09:03 +00:00
|
|
|
if err != nil {
|
2018-10-23 06:54:10 +00:00
|
|
|
return toFileError(f, errors.Wrap(err, "failed to unmarshal YAML"))
|
2018-10-20 17:09:03 +00:00
|
|
|
}
|
2018-10-18 08:21:23 +00:00
|
|
|
|
2018-10-20 09:16:18 +00:00
|
|
|
// To support boolean keys, the YAML package unmarshals maps to
|
2018-10-18 08:21:23 +00:00
|
|
|
// map[interface{}]interface{}. Here we recurse through the result
|
|
|
|
// and change all maps to map[string]interface{} like we would've
|
|
|
|
// gotten from `json`.
|
2018-10-20 09:16:18 +00:00
|
|
|
var ptr interface{}
|
|
|
|
switch v.(type) {
|
|
|
|
case *map[string]interface{}:
|
|
|
|
ptr = *v.(*map[string]interface{})
|
|
|
|
case *interface{}:
|
|
|
|
ptr = *v.(*interface{})
|
|
|
|
default:
|
2020-06-30 14:11:05 +00:00
|
|
|
// Not a map.
|
2018-10-20 09:16:18 +00:00
|
|
|
}
|
|
|
|
|
2020-06-30 14:11:05 +00:00
|
|
|
if ptr != nil {
|
|
|
|
if mm, changed := stringifyMapKeys(ptr); changed {
|
|
|
|
switch v.(type) {
|
|
|
|
case *map[string]interface{}:
|
|
|
|
*v.(*map[string]interface{}) = mm.(map[string]interface{})
|
|
|
|
case *interface{}:
|
|
|
|
*v.(*interface{}) = mm
|
|
|
|
}
|
2018-10-18 08:21:23 +00:00
|
|
|
}
|
|
|
|
}
|
2018-12-23 09:40:32 +00:00
|
|
|
case CSV:
|
|
|
|
return d.unmarshalCSV(data, v)
|
|
|
|
|
2018-10-18 08:21:23 +00:00
|
|
|
default:
|
2018-10-20 09:16:18 +00:00
|
|
|
return errors.Errorf("unmarshal of format %q is not supported", f)
|
2018-10-18 08:21:23 +00:00
|
|
|
}
|
|
|
|
|
2018-10-23 06:54:10 +00:00
|
|
|
if err == nil {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return toFileError(f, errors.Wrap(err, "unmarshal failed"))
|
|
|
|
}
|
2018-10-18 08:21:23 +00:00
|
|
|
|
2018-12-23 09:40:32 +00:00
|
|
|
func (d Decoder) unmarshalCSV(data []byte, v interface{}) error {
|
|
|
|
r := csv.NewReader(bytes.NewReader(data))
|
2018-12-23 20:08:12 +00:00
|
|
|
r.Comma = d.Delimiter
|
2018-12-23 09:40:32 +00:00
|
|
|
r.Comment = d.Comment
|
|
|
|
|
|
|
|
records, err := r.ReadAll()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
switch v.(type) {
|
|
|
|
case *interface{}:
|
|
|
|
*v.(*interface{}) = records
|
|
|
|
default:
|
|
|
|
return errors.Errorf("CSV cannot be unmarshaled into %T", v)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2020-06-26 21:52:12 +00:00
|
|
|
func parseORGDate(s string) string {
|
|
|
|
r := regexp.MustCompile(`[<\[](\d{4}-\d{2}-\d{2}) .*[>\]]`)
|
|
|
|
if m := r.FindStringSubmatch(s); m != nil {
|
|
|
|
return m[1]
|
|
|
|
}
|
|
|
|
return s
|
|
|
|
}
|
|
|
|
|
2019-06-04 10:21:25 +00:00
|
|
|
func (d Decoder) unmarshalORG(data []byte, v interface{}) error {
|
|
|
|
config := org.New()
|
|
|
|
config.Log = jww.WARN
|
|
|
|
document := config.Parse(bytes.NewReader(data), "")
|
|
|
|
if document.Error != nil {
|
|
|
|
return document.Error
|
|
|
|
}
|
|
|
|
frontMatter := make(map[string]interface{}, len(document.BufferSettings))
|
|
|
|
for k, v := range document.BufferSettings {
|
|
|
|
k = strings.ToLower(k)
|
2019-06-04 10:21:48 +00:00
|
|
|
if strings.HasSuffix(k, "[]") {
|
|
|
|
frontMatter[k[:len(k)-2]] = strings.Fields(v)
|
|
|
|
} else if k == "tags" || k == "categories" || k == "aliases" {
|
|
|
|
jww.WARN.Printf("Please use '#+%s[]:' notation, automatic conversion is deprecated.", k)
|
2019-06-04 10:21:25 +00:00
|
|
|
frontMatter[k] = strings.Fields(v)
|
2020-06-26 21:52:12 +00:00
|
|
|
} else if k == "date" {
|
|
|
|
frontMatter[k] = parseORGDate(v)
|
2019-06-04 10:21:25 +00:00
|
|
|
} else {
|
|
|
|
frontMatter[k] = v
|
|
|
|
}
|
|
|
|
}
|
|
|
|
switch v.(type) {
|
|
|
|
case *map[string]interface{}:
|
|
|
|
*v.(*map[string]interface{}) = frontMatter
|
|
|
|
default:
|
|
|
|
*v.(*interface{}) = frontMatter
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-10-23 06:54:10 +00:00
|
|
|
func toFileError(f Format, err error) error {
|
|
|
|
return herrors.ToFileError(string(f), err)
|
2018-10-20 09:16:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// stringifyMapKeys recurses into in and changes all instances of
|
|
|
|
// map[interface{}]interface{} to map[string]interface{}. This is useful to
|
2018-11-13 17:28:40 +00:00
|
|
|
// work around the impedance mismatch between JSON and YAML unmarshaling that's
|
2018-10-20 09:16:18 +00:00
|
|
|
// described here: https://github.com/go-yaml/yaml/issues/139
|
|
|
|
//
|
|
|
|
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
|
|
|
|
func stringifyMapKeys(in interface{}) (interface{}, bool) {
|
|
|
|
switch in := in.(type) {
|
|
|
|
case []interface{}:
|
|
|
|
for i, v := range in {
|
|
|
|
if vv, replaced := stringifyMapKeys(v); replaced {
|
|
|
|
in[i] = vv
|
|
|
|
}
|
|
|
|
}
|
|
|
|
case map[string]interface{}:
|
|
|
|
for k, v := range in {
|
|
|
|
if vv, changed := stringifyMapKeys(v); changed {
|
|
|
|
in[k] = vv
|
|
|
|
}
|
|
|
|
}
|
|
|
|
case map[interface{}]interface{}:
|
|
|
|
res := make(map[string]interface{})
|
|
|
|
var (
|
|
|
|
ok bool
|
|
|
|
err error
|
|
|
|
)
|
|
|
|
for k, v := range in {
|
|
|
|
var ks string
|
|
|
|
|
|
|
|
if ks, ok = k.(string); !ok {
|
|
|
|
ks, err = cast.ToStringE(k)
|
|
|
|
if err != nil {
|
|
|
|
ks = fmt.Sprintf("%v", k)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if vv, replaced := stringifyMapKeys(v); replaced {
|
|
|
|
res[ks] = vv
|
|
|
|
} else {
|
|
|
|
res[ks] = v
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return res, true
|
|
|
|
}
|
2018-10-18 08:21:23 +00:00
|
|
|
|
2018-10-20 09:16:18 +00:00
|
|
|
return nil, false
|
2018-10-18 08:21:23 +00:00
|
|
|
}
|