mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-07 20:30:36 -05:00
parent
44da60d869
commit
129c27ee6e
22 changed files with 624 additions and 808 deletions
|
@ -14,8 +14,19 @@
|
|||
package commands
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
|
||||
"github.com/gohugoio/hugo/parser"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
"github.com/gohugoio/hugo/parser/pageparser"
|
||||
|
||||
src "github.com/gohugoio/hugo/source"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
|
@ -23,7 +34,6 @@ import (
|
|||
|
||||
"path/filepath"
|
||||
|
||||
"github.com/gohugoio/hugo/parser"
|
||||
"github.com/spf13/cast"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
@ -60,7 +70,7 @@ See convert's subcommands toJSON, toTOML and toYAML for more information.`,
|
|||
Long: `toJSON converts all front matter in the content directory
|
||||
to use JSON for the front matter.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return cc.convertContents(rune([]byte(parser.JSONLead)[0]))
|
||||
return cc.convertContents(metadecoders.JSON)
|
||||
},
|
||||
},
|
||||
&cobra.Command{
|
||||
|
@ -69,7 +79,7 @@ to use JSON for the front matter.`,
|
|||
Long: `toTOML converts all front matter in the content directory
|
||||
to use TOML for the front matter.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return cc.convertContents(rune([]byte(parser.TOMLLead)[0]))
|
||||
return cc.convertContents(metadecoders.TOML)
|
||||
},
|
||||
},
|
||||
&cobra.Command{
|
||||
|
@ -78,7 +88,7 @@ to use TOML for the front matter.`,
|
|||
Long: `toYAML converts all front matter in the content directory
|
||||
to use YAML for the front matter.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return cc.convertContents(rune([]byte(parser.YAMLLead)[0]))
|
||||
return cc.convertContents(metadecoders.YAML)
|
||||
},
|
||||
},
|
||||
)
|
||||
|
@ -91,7 +101,7 @@ to use YAML for the front matter.`,
|
|||
return cc
|
||||
}
|
||||
|
||||
func (cc *convertCmd) convertContents(mark rune) error {
|
||||
func (cc *convertCmd) convertContents(format metadecoders.Format) error {
|
||||
if cc.outputDir == "" && !cc.unsafe {
|
||||
return newUserError("Unsafe operation not allowed, use --unsafe or set a different output path")
|
||||
}
|
||||
|
@ -114,17 +124,17 @@ func (cc *convertCmd) convertContents(mark rune) error {
|
|||
|
||||
site.Log.FEEDBACK.Println("processing", len(site.AllPages), "content files")
|
||||
for _, p := range site.AllPages {
|
||||
if err := cc.convertAndSavePage(p, site, mark); err != nil {
|
||||
if err := cc.convertAndSavePage(p, site, format); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, mark rune) error {
|
||||
func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, targetFormat metadecoders.Format) error {
|
||||
// The resources are not in .Site.AllPages.
|
||||
for _, r := range p.Resources.ByType("page") {
|
||||
if err := cc.convertAndSavePage(r.(*hugolib.Page), site, mark); err != nil {
|
||||
if err := cc.convertAndSavePage(r.(*hugolib.Page), site, targetFormat); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -134,37 +144,56 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma
|
|||
return nil
|
||||
}
|
||||
|
||||
errMsg := fmt.Errorf("Error processing file %q", p.Path())
|
||||
|
||||
site.Log.INFO.Println("Attempting to convert", p.LogicalName())
|
||||
newPage, err := site.NewPage(p.LogicalName())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, _ := p.File.(src.ReadableFile)
|
||||
file, err := f.Open()
|
||||
if err != nil {
|
||||
site.Log.ERROR.Println("Error reading file:", p.Path())
|
||||
site.Log.ERROR.Println(errMsg)
|
||||
file.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
psr, err := parser.ReadFrom(file)
|
||||
psr, err := pageparser.Parse(file)
|
||||
if err != nil {
|
||||
site.Log.ERROR.Println("Error processing file:", p.Path())
|
||||
site.Log.ERROR.Println(errMsg)
|
||||
file.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
file.Close()
|
||||
|
||||
metadata, err := psr.Metadata()
|
||||
var sourceFormat, sourceContent []byte
|
||||
var fromFormat metadecoders.Format
|
||||
|
||||
iter := psr.Iterator()
|
||||
|
||||
walkFn := func(item pageparser.Item) bool {
|
||||
if sourceFormat != nil {
|
||||
// The rest is content.
|
||||
sourceContent = psr.Input()[item.Pos:]
|
||||
// Done
|
||||
return false
|
||||
} else if item.IsFrontMatter() {
|
||||
fromFormat = metadecoders.FormatFromFrontMatterType(item.Type)
|
||||
sourceFormat = item.Val
|
||||
}
|
||||
return true
|
||||
|
||||
}
|
||||
|
||||
iter.PeekWalk(walkFn)
|
||||
|
||||
metadata, err := metadecoders.UnmarshalToMap(sourceFormat, fromFormat)
|
||||
if err != nil {
|
||||
site.Log.ERROR.Println("Error processing file:", p.Path())
|
||||
site.Log.ERROR.Println(errMsg)
|
||||
return err
|
||||
}
|
||||
|
||||
// better handling of dates in formats that don't have support for them
|
||||
if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
|
||||
if fromFormat == metadecoders.JSON || fromFormat == metadecoders.YAML || fromFormat == metadecoders.TOML {
|
||||
newMetadata := cast.ToStringMap(metadata)
|
||||
for k, v := range newMetadata {
|
||||
switch vv := v.(type) {
|
||||
|
@ -175,18 +204,26 @@ func (cc *convertCmd) convertAndSavePage(p *hugolib.Page, site *hugolib.Site, ma
|
|||
metadata = newMetadata
|
||||
}
|
||||
|
||||
newPage.SetSourceContent(psr.Content())
|
||||
if err = newPage.SetSourceMetaData(metadata, mark); err != nil {
|
||||
site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", newPage.FullFilePath(), err)
|
||||
return nil
|
||||
var newContent bytes.Buffer
|
||||
err = parser.InterfaceToFrontMatter2(metadata, targetFormat, &newContent)
|
||||
if err != nil {
|
||||
site.Log.ERROR.Println(errMsg)
|
||||
return err
|
||||
}
|
||||
|
||||
newContent.Write(sourceContent)
|
||||
|
||||
newFilename := p.Filename()
|
||||
|
||||
if cc.outputDir != "" {
|
||||
newFilename = filepath.Join(cc.outputDir, p.Dir(), newPage.LogicalName())
|
||||
contentDir := strings.TrimSuffix(newFilename, p.Path())
|
||||
contentDir = filepath.Base(contentDir)
|
||||
|
||||
newFilename = filepath.Join(cc.outputDir, contentDir, p.Path())
|
||||
}
|
||||
|
||||
if err = newPage.SaveSourceAs(newFilename); err != nil {
|
||||
fs := hugofs.Os
|
||||
if err := helpers.WriteToDisk(newFilename, &newContent, fs); err != nil {
|
||||
return errors.Wrapf(err, "Failed to save file %q:", newFilename)
|
||||
}
|
||||
|
||||
|
|
|
@ -42,7 +42,7 @@ import (
|
|||
|
||||
"github.com/gohugoio/hugo/config"
|
||||
|
||||
"github.com/gohugoio/hugo/parser"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
flag "github.com/spf13/pflag"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
|
@ -1017,7 +1017,7 @@ func (c *commandeer) isThemeVsHugoVersionMismatch(fs afero.Fs) (dir string, mism
|
|||
|
||||
b, err := afero.ReadFile(fs, path)
|
||||
|
||||
tomlMeta, err := parser.HandleTOMLMetaData(b)
|
||||
tomlMeta, err := metadecoders.UnmarshalToMap(b, metadecoders.TOML)
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
|
|
|
@ -25,6 +25,8 @@ import (
|
|||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/hugofs"
|
||||
"github.com/gohugoio/hugo/hugolib"
|
||||
|
@ -253,7 +255,7 @@ func (i *importCmd) loadJekyllConfig(fs afero.Fs, jekyllRoot string) map[string]
|
|||
return nil
|
||||
}
|
||||
|
||||
c, err := parser.HandleYAMLMetaData(b)
|
||||
c, err := metadecoders.UnmarshalToMap(b, metadecoders.YAML)
|
||||
|
||||
if err != nil {
|
||||
return nil
|
||||
|
|
3
go.mod
3
go.mod
|
@ -51,7 +51,7 @@ require (
|
|||
github.com/spf13/nitro v0.0.0-20131003134307-24d7ef30a12d
|
||||
github.com/spf13/pflag v1.0.2
|
||||
github.com/spf13/viper v1.2.0
|
||||
github.com/stretchr/testify v1.2.2
|
||||
github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9
|
||||
github.com/tdewolff/minify v2.3.5+incompatible
|
||||
github.com/tdewolff/parse v2.3.3+incompatible // indirect
|
||||
github.com/tdewolff/test v0.0.0-20171106182207-265427085153 // indirect
|
||||
|
@ -63,6 +63,7 @@ require (
|
|||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect
|
||||
golang.org/x/text v0.3.0
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
|
||||
gopkg.in/yaml.v2 v2.2.1
|
||||
)
|
||||
|
||||
|
|
5
go.sum
5
go.sum
|
@ -65,6 +65,7 @@ github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk=
|
|||
github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
|
||||
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
|
||||
github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
|
||||
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
|
@ -118,6 +119,8 @@ github.com/spf13/viper v1.2.0 h1:M4Rzxlu+RgU4pyBRKhKaVN1VeYOm8h2jgyXnAseDgCc=
|
|||
github.com/spf13/viper v1.2.0/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9 h1:kcVw9CGDqYBy0TTpIq2+BNR4W9poqiwEPBh/OYX5CaU=
|
||||
github.com/stretchr/testify v1.2.3-0.20181002232621-f2347ac6c9c9/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/tdewolff/minify v2.3.5+incompatible h1:oFxBKxTIY1F/1DEJhLeh/T507W56JqZtWVrawFcdadI=
|
||||
github.com/tdewolff/minify v2.3.5+incompatible/go.mod h1:9Ov578KJUmAWpS6NeZwRZyT56Uf6o3Mcz9CEsg8USYs=
|
||||
github.com/tdewolff/parse v2.3.3+incompatible h1:q6OSjvHtvBucLb34z24OH1xl5wGdw1mI9Vd38Qj9evs=
|
||||
|
@ -143,5 +146,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
|
||||
gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
|
||||
gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
|
@ -44,6 +44,7 @@ import (
|
|||
"github.com/gohugoio/hugo/config"
|
||||
|
||||
"github.com/gohugoio/hugo/media"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
|
||||
"github.com/markbates/inflect"
|
||||
|
||||
|
@ -53,7 +54,6 @@ import (
|
|||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/hugolib/pagemeta"
|
||||
"github.com/gohugoio/hugo/output"
|
||||
"github.com/gohugoio/hugo/parser"
|
||||
"github.com/gohugoio/hugo/related"
|
||||
"github.com/gohugoio/hugo/source"
|
||||
"github.com/gohugoio/hugo/tpl"
|
||||
|
@ -949,16 +949,8 @@ func (s *Site) readData(f source.ReadableFile) (interface{}, error) {
|
|||
defer file.Close()
|
||||
content := helpers.ReaderToBytes(file)
|
||||
|
||||
switch f.Extension() {
|
||||
case "yaml", "yml":
|
||||
return parser.HandleYAMLData(content)
|
||||
case "json":
|
||||
return parser.HandleJSONData(content)
|
||||
case "toml":
|
||||
return parser.HandleTOMLMetaData(content)
|
||||
default:
|
||||
return nil, fmt.Errorf("Data not supported for extension '%s'", f.Extension())
|
||||
}
|
||||
format := metadecoders.FormatFromString(f.Extension())
|
||||
return metadecoders.Unmarshal(content, format)
|
||||
}
|
||||
|
||||
func (s *Site) readDataFromSourceFS() error {
|
||||
|
|
|
@ -19,16 +19,12 @@ import (
|
|||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
|
||||
"github.com/spf13/cast"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/chaseadamsio/goorgeous"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
@ -79,6 +75,82 @@ func InterfaceToConfig(in interface{}, mark rune, w io.Writer) error {
|
|||
}
|
||||
}
|
||||
|
||||
func InterfaceToConfig2(in interface{}, format metadecoders.Format, w io.Writer) error {
|
||||
if in == nil {
|
||||
return errors.New("input was nil")
|
||||
}
|
||||
|
||||
switch format {
|
||||
case metadecoders.YAML:
|
||||
b, err := yaml.Marshal(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.Write(b)
|
||||
return err
|
||||
|
||||
case metadecoders.TOML:
|
||||
return toml.NewEncoder(w).Encode(in)
|
||||
case metadecoders.JSON:
|
||||
b, err := json.MarshalIndent(in, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.Write(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.Write([]byte{'\n'})
|
||||
return err
|
||||
|
||||
default:
|
||||
return errors.New("Unsupported Format provided")
|
||||
}
|
||||
}
|
||||
|
||||
func InterfaceToFrontMatter2(in interface{}, format metadecoders.Format, w io.Writer) error {
|
||||
if in == nil {
|
||||
return errors.New("input was nil")
|
||||
}
|
||||
|
||||
switch format {
|
||||
case metadecoders.YAML:
|
||||
_, err := w.Write([]byte(YAMLDelimUnix))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = InterfaceToConfig2(in, format, w)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.Write([]byte(YAMLDelimUnix))
|
||||
return err
|
||||
|
||||
case metadecoders.TOML:
|
||||
_, err := w.Write([]byte(TOMLDelimUnix))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = InterfaceToConfig2(in, format, w)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = w.Write([]byte("\n" + TOMLDelimUnix))
|
||||
return err
|
||||
|
||||
default:
|
||||
return InterfaceToConfig2(in, format, w)
|
||||
}
|
||||
}
|
||||
|
||||
// InterfaceToFrontMatter encodes a given input into a frontmatter
|
||||
// representation based upon the mark with the appropriate front matter delimiters
|
||||
// surrounding the output, which is written to w.
|
||||
|
@ -155,34 +227,6 @@ func FormatSanitize(kind string) string {
|
|||
}
|
||||
}
|
||||
|
||||
// DetectFrontMatter detects the type of frontmatter analysing its first character.
|
||||
func DetectFrontMatter(mark rune) (f *FrontmatterType) {
|
||||
switch mark {
|
||||
case '-':
|
||||
return &FrontmatterType{HandleYAMLMetaData, []byte(YAMLDelim), []byte(YAMLDelim), false}
|
||||
case '+':
|
||||
return &FrontmatterType{HandleTOMLMetaData, []byte(TOMLDelim), []byte(TOMLDelim), false}
|
||||
case '{':
|
||||
return &FrontmatterType{HandleJSONMetaData, []byte{'{'}, []byte{'}'}, true}
|
||||
case '#':
|
||||
return &FrontmatterType{HandleOrgMetaData, []byte("#+"), []byte("\n"), false}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// HandleTOMLMetaData unmarshals TOML-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
func HandleTOMLMetaData(datum []byte) (map[string]interface{}, error) {
|
||||
m := map[string]interface{}{}
|
||||
datum = removeTOMLIdentifier(datum)
|
||||
|
||||
_, err := toml.Decode(string(datum), &m)
|
||||
|
||||
return m, err
|
||||
|
||||
}
|
||||
|
||||
// removeTOMLIdentifier removes, if necessary, beginning and ending TOML
|
||||
// frontmatter delimiters from a byte slice.
|
||||
func removeTOMLIdentifier(datum []byte) []byte {
|
||||
|
@ -200,125 +244,3 @@ func removeTOMLIdentifier(datum []byte) []byte {
|
|||
b = bytes.Trim(b, "\r\n")
|
||||
return bytes.TrimSuffix(b, []byte(TOMLDelim))
|
||||
}
|
||||
|
||||
// HandleYAMLMetaData unmarshals YAML-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
// TODO(bep) 2errors remove these handlers (and hopefully package)
|
||||
func HandleYAMLMetaData(datum []byte) (map[string]interface{}, error) {
|
||||
m := map[string]interface{}{}
|
||||
err := yaml.Unmarshal(datum, &m)
|
||||
|
||||
// To support boolean keys, the `yaml` package unmarshals maps to
|
||||
// map[interface{}]interface{}. Here we recurse through the result
|
||||
// and change all maps to map[string]interface{} like we would've
|
||||
// gotten from `json`.
|
||||
if err == nil {
|
||||
for k, v := range m {
|
||||
if vv, changed := stringifyMapKeys(v); changed {
|
||||
m[k] = vv
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return m, err
|
||||
}
|
||||
|
||||
// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
func HandleYAMLData(datum []byte) (interface{}, error) {
|
||||
var m interface{}
|
||||
err := yaml.Unmarshal(datum, &m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// To support boolean keys, the `yaml` package unmarshals maps to
|
||||
// map[interface{}]interface{}. Here we recurse through the result
|
||||
// and change all maps to map[string]interface{} like we would've
|
||||
// gotten from `json`.
|
||||
if mm, changed := stringifyMapKeys(m); changed {
|
||||
return mm, nil
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// stringifyMapKeys recurses into in and changes all instances of
|
||||
// map[interface{}]interface{} to map[string]interface{}. This is useful to
|
||||
// work around the impedence mismatch between JSON and YAML unmarshaling that's
|
||||
// described here: https://github.com/go-yaml/yaml/issues/139
|
||||
//
|
||||
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
|
||||
func stringifyMapKeys(in interface{}) (interface{}, bool) {
|
||||
switch in := in.(type) {
|
||||
case []interface{}:
|
||||
for i, v := range in {
|
||||
if vv, replaced := stringifyMapKeys(v); replaced {
|
||||
in[i] = vv
|
||||
}
|
||||
}
|
||||
case map[interface{}]interface{}:
|
||||
res := make(map[string]interface{})
|
||||
var (
|
||||
ok bool
|
||||
err error
|
||||
)
|
||||
for k, v := range in {
|
||||
var ks string
|
||||
|
||||
if ks, ok = k.(string); !ok {
|
||||
ks, err = cast.ToStringE(k)
|
||||
if err != nil {
|
||||
ks = fmt.Sprintf("%v", k)
|
||||
}
|
||||
// TODO(bep) added in Hugo 0.37, remove some time in the future.
|
||||
helpers.DistinctFeedbackLog.Printf("WARNING: YAML data/frontmatter with keys of type %T is since Hugo 0.37 converted to strings", k)
|
||||
}
|
||||
if vv, replaced := stringifyMapKeys(v); replaced {
|
||||
res[ks] = vv
|
||||
} else {
|
||||
res[ks] = v
|
||||
}
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// HandleJSONMetaData unmarshals JSON-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
func HandleJSONMetaData(datum []byte) (map[string]interface{}, error) {
|
||||
m := make(map[string]interface{})
|
||||
|
||||
if datum == nil {
|
||||
// Package json returns on error on nil input.
|
||||
// Return an empty map to be consistent with our other supported
|
||||
// formats.
|
||||
return m, nil
|
||||
}
|
||||
|
||||
err := json.Unmarshal(datum, &m)
|
||||
return m, err
|
||||
}
|
||||
|
||||
// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
func HandleJSONData(datum []byte) (interface{}, error) {
|
||||
if datum == nil {
|
||||
// Package json returns on error on nil input.
|
||||
// Return an empty map to be consistent with our other supported
|
||||
// formats.
|
||||
return make(map[string]interface{}), nil
|
||||
}
|
||||
|
||||
var f interface{}
|
||||
err := json.Unmarshal(datum, &f)
|
||||
return f, err
|
||||
}
|
||||
|
||||
// HandleOrgMetaData unmarshals org-mode encoded datum and returns a Go
|
||||
// interface representing the encoded data structure.
|
||||
func HandleOrgMetaData(datum []byte) (map[string]interface{}, error) {
|
||||
return goorgeous.OrgHeaders(datum)
|
||||
}
|
||||
|
|
|
@ -132,116 +132,6 @@ func TestInterfaceToFrontMatter(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestHandleTOMLMetaData(t *testing.T) {
|
||||
cases := []struct {
|
||||
input []byte
|
||||
want interface{}
|
||||
isErr bool
|
||||
}{
|
||||
{nil, map[string]interface{}{}, false},
|
||||
{[]byte("title = \"test 1\""), map[string]interface{}{"title": "test 1"}, false},
|
||||
{[]byte("a = [1, 2, 3]"), map[string]interface{}{"a": []interface{}{int64(1), int64(2), int64(3)}}, false},
|
||||
{[]byte("b = [\n[1, 2],\n[3, 4]\n]"), map[string]interface{}{"b": []interface{}{[]interface{}{int64(1), int64(2)}, []interface{}{int64(3), int64(4)}}}, false},
|
||||
// errors
|
||||
{[]byte("z = [\n[1, 2]\n[3, 4]\n]"), nil, true},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
res, err := HandleTOMLMetaData(c.input)
|
||||
if err != nil {
|
||||
if c.isErr {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("[%d] unexpected error value: %v", i, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(res, c.want) {
|
||||
t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleYAMLMetaData(t *testing.T) {
|
||||
cases := []struct {
|
||||
input []byte
|
||||
want interface{}
|
||||
isErr bool
|
||||
}{
|
||||
{nil, map[string]interface{}{}, false},
|
||||
{[]byte("title: test 1"), map[string]interface{}{"title": "test 1"}, false},
|
||||
{[]byte("a: Easy!\nb:\n c: 2\n d: [3, 4]"), map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}, false},
|
||||
{[]byte("a:\n true: 1\n false: 2"), map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}, false},
|
||||
// errors
|
||||
{[]byte("z = not toml"), nil, true},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
res, err := HandleYAMLMetaData(c.input)
|
||||
if err != nil {
|
||||
if c.isErr {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("[%d] unexpected error value: %v", i, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(res, c.want) {
|
||||
t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleJSONMetaData(t *testing.T) {
|
||||
cases := []struct {
|
||||
input []byte
|
||||
want interface{}
|
||||
isErr bool
|
||||
}{
|
||||
{nil, map[string]interface{}{}, false},
|
||||
{[]byte("{\"title\": \"test 1\"}"), map[string]interface{}{"title": "test 1"}, false},
|
||||
// errors
|
||||
{[]byte("{noquotes}"), nil, true},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
res, err := HandleJSONMetaData(c.input)
|
||||
if err != nil {
|
||||
if c.isErr {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("[%d] unexpected error value: %v", i, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(res, c.want) {
|
||||
t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleOrgMetaData(t *testing.T) {
|
||||
cases := []struct {
|
||||
input []byte
|
||||
want interface{}
|
||||
isErr bool
|
||||
}{
|
||||
{nil, map[string]interface{}{}, false},
|
||||
{[]byte("#+title: test 1\n"), map[string]interface{}{"title": "test 1"}, false},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
res, err := HandleOrgMetaData(c.input)
|
||||
if err != nil {
|
||||
if c.isErr {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("[%d] unexpected error value: %v", i, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(res, c.want) {
|
||||
t.Errorf("[%d] not equal: given %q\nwant %#v,\n got %#v", i, c.input, c.want, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatToLeadRune(t *testing.T) {
|
||||
for i, this := range []struct {
|
||||
kind string
|
||||
|
@ -264,41 +154,6 @@ func TestFormatToLeadRune(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestDetectFrontMatter(t *testing.T) {
|
||||
cases := []struct {
|
||||
mark rune
|
||||
want *FrontmatterType
|
||||
}{
|
||||
// funcs are uncomparable, so we ignore FrontmatterType.Parse in these tests
|
||||
{'-', &FrontmatterType{nil, []byte(YAMLDelim), []byte(YAMLDelim), false}},
|
||||
{'+', &FrontmatterType{nil, []byte(TOMLDelim), []byte(TOMLDelim), false}},
|
||||
{'{', &FrontmatterType{nil, []byte("{"), []byte("}"), true}},
|
||||
{'#', &FrontmatterType{nil, []byte("#+"), []byte("\n"), false}},
|
||||
{'$', nil},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
res := DetectFrontMatter(c.mark)
|
||||
if res == nil {
|
||||
if c.want == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
t.Fatalf("want %v, got %v", *c.want, res)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(res.markstart, c.want.markstart) {
|
||||
t.Errorf("markstart mismatch: want %v, got %v", c.want.markstart, res.markstart)
|
||||
}
|
||||
if !reflect.DeepEqual(res.markend, c.want.markend) {
|
||||
t.Errorf("markend mismatch: want %v, got %v", c.want.markend, res.markend)
|
||||
}
|
||||
if !reflect.DeepEqual(res.includeMark, c.want.includeMark) {
|
||||
t.Errorf("includeMark mismatch: want %v, got %v", c.want.includeMark, res.includeMark)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveTOMLIdentifier(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
|
@ -321,64 +176,6 @@ func TestRemoveTOMLIdentifier(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestStringifyYAMLMapKeys(t *testing.T) {
|
||||
cases := []struct {
|
||||
input interface{}
|
||||
want interface{}
|
||||
replaced bool
|
||||
}{
|
||||
{
|
||||
map[interface{}]interface{}{"a": 1, "b": 2},
|
||||
map[string]interface{}{"a": 1, "b": 2},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},
|
||||
map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{true: 1, "b": false},
|
||||
map[string]interface{}{"true": 1, "b": false},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{1: "a", 2: "b"},
|
||||
map[string]interface{}{"1": "a", "2": "b"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},
|
||||
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
|
||||
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
|
||||
false,
|
||||
},
|
||||
{
|
||||
[]interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},
|
||||
[]interface{}{map[string]interface{}{"1": "a", "2": "b"}},
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
res, replaced := stringifyMapKeys(c.input)
|
||||
|
||||
if c.replaced != replaced {
|
||||
t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)
|
||||
}
|
||||
if !c.replaced {
|
||||
res = c.input
|
||||
}
|
||||
if !reflect.DeepEqual(res, c.want) {
|
||||
t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFrontmatterTags(b *testing.B) {
|
||||
|
||||
for _, frontmatter := range []string{"JSON", "YAML", "YAML2", "TOML"} {
|
||||
|
@ -388,69 +185,6 @@ func BenchmarkFrontmatterTags(b *testing.B) {
|
|||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {
|
||||
maps := make([]map[interface{}]interface{}, b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
maps[i] = map[interface{}]interface{}{
|
||||
"a": map[interface{}]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
"d": map[interface{}]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
},
|
||||
},
|
||||
"b": []interface{}{"a", "b"},
|
||||
"c": "d",
|
||||
}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
stringifyMapKeys(maps[i])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {
|
||||
m := map[string]interface{}{
|
||||
"a": map[string]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
"d": map[string]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
},
|
||||
},
|
||||
"b": []interface{}{"a", "b"},
|
||||
"c": "d",
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
stringifyMapKeys(m)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringifyMapKeysIntegers(b *testing.B) {
|
||||
maps := make([]map[interface{}]interface{}, b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
maps[i] = map[interface{}]interface{}{
|
||||
1: map[interface{}]interface{}{
|
||||
4: 32,
|
||||
5: 43,
|
||||
6: map[interface{}]interface{}{
|
||||
7: 32,
|
||||
8: 43,
|
||||
},
|
||||
},
|
||||
2: []interface{}{"a", "b"},
|
||||
3: "d",
|
||||
}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
stringifyMapKeys(maps[i])
|
||||
}
|
||||
}
|
||||
func doBenchmarkFrontmatter(b *testing.B, fileformat string, numTags int) {
|
||||
yamlTemplate := `---
|
||||
name: "Tags"
|
||||
|
|
|
@ -15,81 +15,139 @@ package metadecoders
|
|||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/chaseadamsio/goorgeous"
|
||||
"github.com/gohugoio/hugo/parser/pageparser"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/spf13/cast"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type Format string
|
||||
|
||||
const (
|
||||
// These are the supported metdata formats in Hugo. Most of these are also
|
||||
// supported as /data formats.
|
||||
ORG Format = "org"
|
||||
JSON Format = "json"
|
||||
TOML Format = "toml"
|
||||
YAML Format = "yaml"
|
||||
)
|
||||
|
||||
// FormatFromFrontMatterType will return empty if not supported.
|
||||
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
|
||||
switch typ {
|
||||
case pageparser.TypeFrontMatterJSON:
|
||||
return JSON
|
||||
case pageparser.TypeFrontMatterORG:
|
||||
return ORG
|
||||
case pageparser.TypeFrontMatterTOML:
|
||||
return TOML
|
||||
case pageparser.TypeFrontMatterYAML:
|
||||
return YAML
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarshalToMap will unmarshall data in format f into a new map. This is
|
||||
// what's needed for Hugo's front matter decoding.
|
||||
func UnmarshalToMap(data []byte, f Format) (map[string]interface{}, error) {
|
||||
m := make(map[string]interface{})
|
||||
|
||||
if data == nil {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
err := unmarshal(data, f, &m)
|
||||
|
||||
return m, err
|
||||
|
||||
}
|
||||
|
||||
// Unmarshal will unmarshall data in format f into an interface{}.
|
||||
// This is what's needed for Hugo's /data handling.
|
||||
func Unmarshal(data []byte, f Format) (interface{}, error) {
|
||||
if data == nil {
|
||||
return make(map[string]interface{}), nil
|
||||
}
|
||||
var v interface{}
|
||||
err := unmarshal(data, f, &v)
|
||||
|
||||
return v, err
|
||||
}
|
||||
|
||||
// unmarshal unmarshals data in format f into v.
|
||||
func unmarshal(data []byte, f Format, v interface{}) error {
|
||||
|
||||
var err error
|
||||
|
||||
switch f {
|
||||
case ORG:
|
||||
m, err = goorgeous.OrgHeaders(data)
|
||||
vv, err := goorgeous.OrgHeaders(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch v.(type) {
|
||||
case *map[string]interface{}:
|
||||
*v.(*map[string]interface{}) = vv
|
||||
default:
|
||||
*v.(*interface{}) = vv
|
||||
}
|
||||
case JSON:
|
||||
err = json.Unmarshal(data, &m)
|
||||
err = json.Unmarshal(data, v)
|
||||
case TOML:
|
||||
_, err = toml.Decode(string(data), &m)
|
||||
err = toml.Unmarshal(data, v)
|
||||
case YAML:
|
||||
err = yaml.Unmarshal(data, &m)
|
||||
err = yaml.Unmarshal(data, v)
|
||||
|
||||
// To support boolean keys, the `yaml` package unmarshals maps to
|
||||
// To support boolean keys, the YAML package unmarshals maps to
|
||||
// map[interface{}]interface{}. Here we recurse through the result
|
||||
// and change all maps to map[string]interface{} like we would've
|
||||
// gotten from `json`.
|
||||
if err == nil {
|
||||
for k, v := range m {
|
||||
if vv, changed := stringifyMapKeys(v); changed {
|
||||
m[k] = vv
|
||||
}
|
||||
var ptr interface{}
|
||||
switch v.(type) {
|
||||
case *map[string]interface{}:
|
||||
ptr = *v.(*map[string]interface{})
|
||||
case *interface{}:
|
||||
ptr = *v.(*interface{})
|
||||
default:
|
||||
return errors.Errorf("unknown type %T in YAML unmarshal", v)
|
||||
}
|
||||
|
||||
if mm, changed := stringifyMapKeys(ptr); changed {
|
||||
switch v.(type) {
|
||||
case *map[string]interface{}:
|
||||
*v.(*map[string]interface{}) = mm.(map[string]interface{})
|
||||
case *interface{}:
|
||||
*v.(*interface{}) = mm
|
||||
}
|
||||
}
|
||||
default:
|
||||
return nil, errors.Errorf("unmarshal of format %q is not supported", f)
|
||||
return errors.Errorf("unmarshal of format %q is not supported", f)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "unmarshal failed for format %q", f)
|
||||
}
|
||||
|
||||
return m, nil
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
// stringifyMapKeys recurses into in and changes all instances of
|
||||
// map[interface{}]interface{} to map[string]interface{}. This is useful to
|
||||
// work around the impedence mismatch between JSON and YAML unmarshaling that's
|
||||
// described here: https://github.com/go-yaml/yaml/issues/139
|
||||
//
|
||||
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
|
||||
func stringifyMapKeys(in interface{}) (interface{}, bool) {
|
||||
|
||||
switch in := in.(type) {
|
||||
case []interface{}:
|
||||
for i, v := range in {
|
||||
if vv, replaced := stringifyMapKeys(v); replaced {
|
||||
in[i] = vv
|
||||
}
|
||||
}
|
||||
case map[string]interface{}:
|
||||
for k, v := range in {
|
||||
if vv, changed := stringifyMapKeys(v); changed {
|
||||
in[k] = vv
|
||||
}
|
||||
}
|
||||
case map[interface{}]interface{}:
|
||||
res := make(map[string]interface{})
|
||||
var (
|
||||
ok bool
|
||||
err error
|
||||
)
|
||||
for k, v := range in {
|
||||
var ks string
|
||||
|
||||
if ks, ok = k.(string); !ok {
|
||||
ks, err = cast.ToStringE(k)
|
||||
if err != nil {
|
||||
ks = fmt.Sprintf("%v", k)
|
||||
}
|
||||
}
|
||||
if vv, replaced := stringifyMapKeys(v); replaced {
|
||||
res[ks] = vv
|
||||
} else {
|
||||
res[ks] = v
|
||||
}
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
|
207
parser/metadecoders/decoder_test.go
Normal file
207
parser/metadecoders/decoder_test.go
Normal file
|
@ -0,0 +1,207 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metadecoders
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUnmarshalToMap(t *testing.T) {
|
||||
assert := require.New(t)
|
||||
|
||||
expect := map[string]interface{}{"a": "b"}
|
||||
|
||||
for i, test := range []struct {
|
||||
data string
|
||||
format Format
|
||||
expect interface{}
|
||||
}{
|
||||
{`a = "b"`, TOML, expect},
|
||||
{`a: "b"`, YAML, expect},
|
||||
// Make sure we get all string keys, even for YAML
|
||||
{"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
|
||||
{"a:\n true: 1\n false: 2", YAML, map[string]interface{}{"a": map[string]interface{}{"true": 1, "false": 2}}},
|
||||
{`{ "a": "b" }`, JSON, expect},
|
||||
{`#+a: b`, ORG, expect},
|
||||
// errors
|
||||
{`a = b`, TOML, false},
|
||||
} {
|
||||
msg := fmt.Sprintf("%d: %s", i, test.format)
|
||||
m, err := UnmarshalToMap([]byte(test.data), test.format)
|
||||
if b, ok := test.expect.(bool); ok && !b {
|
||||
assert.Error(err, msg)
|
||||
} else {
|
||||
assert.NoError(err, msg)
|
||||
assert.Equal(test.expect, m, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnmarshalToInterface(t *testing.T) {
|
||||
assert := require.New(t)
|
||||
|
||||
expect := map[string]interface{}{"a": "b"}
|
||||
|
||||
for i, test := range []struct {
|
||||
data string
|
||||
format Format
|
||||
expect interface{}
|
||||
}{
|
||||
{`[ "Brecker", "Blake", "Redman" ]`, JSON, []interface{}{"Brecker", "Blake", "Redman"}},
|
||||
{`{ "a": "b" }`, JSON, expect},
|
||||
{`#+a: b`, ORG, expect},
|
||||
{`a = "b"`, TOML, expect},
|
||||
{`a: "b"`, YAML, expect},
|
||||
{"a: Easy!\nb:\n c: 2\n d: [3, 4]", YAML, map[string]interface{}{"a": "Easy!", "b": map[string]interface{}{"c": 2, "d": []interface{}{3, 4}}}},
|
||||
// errors
|
||||
{`a = "`, TOML, false},
|
||||
} {
|
||||
msg := fmt.Sprintf("%d: %s", i, test.format)
|
||||
m, err := Unmarshal([]byte(test.data), test.format)
|
||||
if b, ok := test.expect.(bool); ok && !b {
|
||||
assert.Error(err, msg)
|
||||
} else {
|
||||
assert.NoError(err, msg)
|
||||
assert.Equal(test.expect, m, msg)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestStringifyYAMLMapKeys(t *testing.T) {
|
||||
cases := []struct {
|
||||
input interface{}
|
||||
want interface{}
|
||||
replaced bool
|
||||
}{
|
||||
{
|
||||
map[interface{}]interface{}{"a": 1, "b": 2},
|
||||
map[string]interface{}{"a": 1, "b": 2},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{"a": []interface{}{1, map[interface{}]interface{}{"b": 2}}},
|
||||
map[string]interface{}{"a": []interface{}{1, map[string]interface{}{"b": 2}}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{true: 1, "b": false},
|
||||
map[string]interface{}{"true": 1, "b": false},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{1: "a", 2: "b"},
|
||||
map[string]interface{}{"1": "a", "2": "b"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[interface{}]interface{}{"a": map[interface{}]interface{}{"b": 1}},
|
||||
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
|
||||
true,
|
||||
},
|
||||
{
|
||||
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
|
||||
map[string]interface{}{"a": map[string]interface{}{"b": 1}},
|
||||
false,
|
||||
},
|
||||
{
|
||||
[]interface{}{map[interface{}]interface{}{1: "a", 2: "b"}},
|
||||
[]interface{}{map[string]interface{}{"1": "a", "2": "b"}},
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
res, replaced := stringifyMapKeys(c.input)
|
||||
|
||||
if c.replaced != replaced {
|
||||
t.Fatalf("[%d] Replaced mismatch: %t", i, replaced)
|
||||
}
|
||||
if !c.replaced {
|
||||
res = c.input
|
||||
}
|
||||
if !reflect.DeepEqual(res, c.want) {
|
||||
t.Errorf("[%d] given %q\nwant: %q\n got: %q", i, c.input, c.want, res)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringifyMapKeysStringsOnlyInterfaceMaps(b *testing.B) {
|
||||
maps := make([]map[interface{}]interface{}, b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
maps[i] = map[interface{}]interface{}{
|
||||
"a": map[interface{}]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
"d": map[interface{}]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
},
|
||||
},
|
||||
"b": []interface{}{"a", "b"},
|
||||
"c": "d",
|
||||
}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
stringifyMapKeys(maps[i])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringifyMapKeysStringsOnlyStringMaps(b *testing.B) {
|
||||
m := map[string]interface{}{
|
||||
"a": map[string]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
"d": map[string]interface{}{
|
||||
"b": 32,
|
||||
"c": 43,
|
||||
},
|
||||
},
|
||||
"b": []interface{}{"a", "b"},
|
||||
"c": "d",
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
stringifyMapKeys(m)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkStringifyMapKeysIntegers(b *testing.B) {
|
||||
maps := make([]map[interface{}]interface{}, b.N)
|
||||
for i := 0; i < b.N; i++ {
|
||||
maps[i] = map[interface{}]interface{}{
|
||||
1: map[interface{}]interface{}{
|
||||
4: 32,
|
||||
5: 43,
|
||||
6: map[interface{}]interface{}{
|
||||
7: 32,
|
||||
8: 43,
|
||||
},
|
||||
},
|
||||
2: []interface{}{"a", "b"},
|
||||
3: "d",
|
||||
}
|
||||
}
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
stringifyMapKeys(maps[i])
|
||||
}
|
||||
}
|
66
parser/metadecoders/format.go
Normal file
66
parser/metadecoders/format.go
Normal file
|
@ -0,0 +1,66 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metadecoders
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/parser/pageparser"
|
||||
)
|
||||
|
||||
type Format string
|
||||
|
||||
const (
|
||||
// These are the supported metdata formats in Hugo. Most of these are also
|
||||
// supported as /data formats.
|
||||
ORG Format = "org"
|
||||
JSON Format = "json"
|
||||
TOML Format = "toml"
|
||||
YAML Format = "yaml"
|
||||
)
|
||||
|
||||
// FormatFromString turns formatStr, typically a file extension without any ".",
|
||||
// into a Format. It returns an empty string for unknown formats.
|
||||
func FormatFromString(formatStr string) Format {
|
||||
formatStr = strings.ToLower(formatStr)
|
||||
switch formatStr {
|
||||
case "yaml", "yml":
|
||||
return YAML
|
||||
case "json":
|
||||
return JSON
|
||||
case "toml":
|
||||
return TOML
|
||||
case "org":
|
||||
return ORG
|
||||
}
|
||||
|
||||
return ""
|
||||
|
||||
}
|
||||
|
||||
// FormatFromFrontMatterType will return empty if not supported.
|
||||
func FormatFromFrontMatterType(typ pageparser.ItemType) Format {
|
||||
switch typ {
|
||||
case pageparser.TypeFrontMatterJSON:
|
||||
return JSON
|
||||
case pageparser.TypeFrontMatterORG:
|
||||
return ORG
|
||||
case pageparser.TypeFrontMatterTOML:
|
||||
return TOML
|
||||
case pageparser.TypeFrontMatterYAML:
|
||||
return YAML
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
57
parser/metadecoders/format_test.go
Normal file
57
parser/metadecoders/format_test.go
Normal file
|
@ -0,0 +1,57 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metadecoders
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/parser/pageparser"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFormatFromString(t *testing.T) {
|
||||
assert := require.New(t)
|
||||
for i, test := range []struct {
|
||||
s string
|
||||
expect Format
|
||||
}{
|
||||
{"json", JSON},
|
||||
{"yaml", YAML},
|
||||
{"yml", YAML},
|
||||
{"toml", TOML},
|
||||
{"tOMl", TOML},
|
||||
{"org", ORG},
|
||||
{"foo", ""},
|
||||
} {
|
||||
assert.Equal(test.expect, FormatFromString(test.s), fmt.Sprintf("t%d", i))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatFromFrontMatterType(t *testing.T) {
|
||||
assert := require.New(t)
|
||||
for i, test := range []struct {
|
||||
typ pageparser.ItemType
|
||||
expect Format
|
||||
}{
|
||||
{pageparser.TypeFrontMatterJSON, JSON},
|
||||
{pageparser.TypeFrontMatterTOML, TOML},
|
||||
{pageparser.TypeFrontMatterYAML, YAML},
|
||||
{pageparser.TypeFrontMatterORG, ORG},
|
||||
{pageparser.TypeIgnore, ""},
|
||||
} {
|
||||
assert.Equal(test.expect, FormatFromFrontMatterType(test.typ), fmt.Sprintf("t%d", i))
|
||||
}
|
||||
}
|
|
@ -1,31 +0,0 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metadecoders
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// HandleJSONData unmarshals JSON-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
func HandleJSONData(datum []byte) (interface{}, error) {
|
||||
if datum == nil {
|
||||
// Package json returns on error on nil input.
|
||||
// Return an empty map to be consistent with our other supported
|
||||
// formats.
|
||||
return make(map[string]interface{}), nil
|
||||
}
|
||||
|
||||
var f interface{}
|
||||
err := json.Unmarshal(datum, &f)
|
||||
return f, err
|
||||
}
|
|
@ -1,84 +0,0 @@
|
|||
// Copyright 2018 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The metadecoders package contains functions to decode metadata (e.g. page front matter)
|
||||
// from different formats: TOML, YAML, JSON.
|
||||
package metadecoders
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cast"
|
||||
yaml "gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
// HandleYAMLData unmarshals YAML-encoded datum and returns a Go interface
|
||||
// representing the encoded data structure.
|
||||
func HandleYAMLData(datum []byte) (interface{}, error) {
|
||||
var m interface{}
|
||||
err := yaml.Unmarshal(datum, &m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// To support boolean keys, the `yaml` package unmarshals maps to
|
||||
// map[interface{}]interface{}. Here we recurse through the result
|
||||
// and change all maps to map[string]interface{} like we would've
|
||||
// gotten from `json`.
|
||||
if mm, changed := stringifyMapKeys(m); changed {
|
||||
return mm, nil
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// stringifyMapKeys recurses into in and changes all instances of
|
||||
// map[interface{}]interface{} to map[string]interface{}. This is useful to
|
||||
// work around the impedence mismatch between JSON and YAML unmarshaling that's
|
||||
// described here: https://github.com/go-yaml/yaml/issues/139
|
||||
//
|
||||
// Inspired by https://github.com/stripe/stripe-mock, MIT licensed
|
||||
func stringifyMapKeys(in interface{}) (interface{}, bool) {
|
||||
switch in := in.(type) {
|
||||
case []interface{}:
|
||||
for i, v := range in {
|
||||
if vv, replaced := stringifyMapKeys(v); replaced {
|
||||
in[i] = vv
|
||||
}
|
||||
}
|
||||
case map[interface{}]interface{}:
|
||||
res := make(map[string]interface{})
|
||||
var (
|
||||
ok bool
|
||||
err error
|
||||
)
|
||||
for k, v := range in {
|
||||
var ks string
|
||||
|
||||
if ks, ok = k.(string); !ok {
|
||||
ks, err = cast.ToStringE(k)
|
||||
if err != nil {
|
||||
ks = fmt.Sprintf("%v", k)
|
||||
}
|
||||
}
|
||||
if vv, replaced := stringifyMapKeys(v); replaced {
|
||||
res[ks] = vv
|
||||
} else {
|
||||
res[ks] = v
|
||||
}
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
|
@ -101,15 +101,8 @@ func (p *page) IsRenderable() bool {
|
|||
|
||||
// Metadata returns the unmarshalled frontmatter data.
|
||||
func (p *page) Metadata() (meta map[string]interface{}, err error) {
|
||||
frontmatter := p.FrontMatter()
|
||||
|
||||
if len(frontmatter) != 0 {
|
||||
fm := DetectFrontMatter(rune(frontmatter[0]))
|
||||
if fm != nil {
|
||||
meta, err = fm.Parse(frontmatter)
|
||||
}
|
||||
}
|
||||
return
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// ReadFrom reads the content from an io.Reader and constructs a page.
|
||||
|
|
|
@ -1,130 +1 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestPage(t *testing.T) {
|
||||
cases := []struct {
|
||||
raw string
|
||||
|
||||
content string
|
||||
frontmatter string
|
||||
renderable bool
|
||||
metadata map[string]interface{}
|
||||
}{
|
||||
{
|
||||
testPageLeader + jsonPageFrontMatter + "\n" + testPageTrailer + jsonPageContent,
|
||||
jsonPageContent,
|
||||
jsonPageFrontMatter,
|
||||
true,
|
||||
map[string]interface{}{
|
||||
"title": "JSON Test 1",
|
||||
"social": []interface{}{
|
||||
[]interface{}{"a", "#"},
|
||||
[]interface{}{"b", "#"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testPageLeader + tomlPageFrontMatter + testPageTrailer + tomlPageContent,
|
||||
tomlPageContent,
|
||||
tomlPageFrontMatter,
|
||||
true,
|
||||
map[string]interface{}{
|
||||
"title": "TOML Test 1",
|
||||
"social": []interface{}{
|
||||
[]interface{}{"a", "#"},
|
||||
[]interface{}{"b", "#"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testPageLeader + yamlPageFrontMatter + testPageTrailer + yamlPageContent,
|
||||
yamlPageContent,
|
||||
yamlPageFrontMatter,
|
||||
true,
|
||||
map[string]interface{}{
|
||||
"title": "YAML Test 1",
|
||||
"social": []interface{}{
|
||||
[]interface{}{"a", "#"},
|
||||
[]interface{}{"b", "#"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
testPageLeader + orgPageFrontMatter + orgPageContent,
|
||||
orgPageContent,
|
||||
orgPageFrontMatter,
|
||||
true,
|
||||
map[string]interface{}{
|
||||
"TITLE": "Org Test 1",
|
||||
"categories": []string{"a", "b"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
p := pageMust(ReadFrom(strings.NewReader(c.raw)))
|
||||
meta, err := p.Metadata()
|
||||
|
||||
mesg := fmt.Sprintf("[%d]", i)
|
||||
|
||||
require.Nil(t, err, mesg)
|
||||
assert.Equal(t, c.content, string(p.Content()), mesg+" content")
|
||||
assert.Equal(t, c.frontmatter, string(p.FrontMatter()), mesg+" frontmatter")
|
||||
assert.Equal(t, c.renderable, p.IsRenderable(), mesg+" renderable")
|
||||
assert.Equal(t, c.metadata, meta, mesg+" metadata")
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
testWhitespace = "\t\t\n\n"
|
||||
testPageLeader = "\ufeff" + testWhitespace + "<!--[metadata]>\n"
|
||||
testPageTrailer = "\n<![end-metadata]-->\n"
|
||||
|
||||
jsonPageContent = "# JSON Test\n"
|
||||
jsonPageFrontMatter = `{
|
||||
"title": "JSON Test 1",
|
||||
"social": [
|
||||
["a", "#"],
|
||||
["b", "#"]
|
||||
]
|
||||
}`
|
||||
|
||||
tomlPageContent = "# TOML Test\n"
|
||||
tomlPageFrontMatter = `+++
|
||||
title = "TOML Test 1"
|
||||
social = [
|
||||
["a", "#"],
|
||||
["b", "#"],
|
||||
]
|
||||
+++
|
||||
`
|
||||
|
||||
yamlPageContent = "# YAML Test\n"
|
||||
yamlPageFrontMatter = `---
|
||||
title: YAML Test 1
|
||||
social:
|
||||
- - "a"
|
||||
- "#"
|
||||
- - "b"
|
||||
- "#"
|
||||
---
|
||||
`
|
||||
|
||||
orgPageContent = "* Org Test\n"
|
||||
orgPageFrontMatter = `#+TITLE: Org Test 1
|
||||
#+categories: a b
|
||||
`
|
||||
|
||||
pageHTMLComment = `<!--
|
||||
This is a sample comment.
|
||||
-->
|
||||
`
|
||||
)
|
||||
|
|
|
@ -20,7 +20,7 @@ import (
|
|||
|
||||
type Item struct {
|
||||
Type ItemType
|
||||
pos pos
|
||||
Pos Pos
|
||||
Val []byte
|
||||
}
|
||||
|
||||
|
|
|
@ -25,7 +25,7 @@ import (
|
|||
)
|
||||
|
||||
// position (in bytes)
|
||||
type pos int
|
||||
type Pos int
|
||||
|
||||
const eof = -1
|
||||
|
||||
|
@ -47,9 +47,9 @@ type pageLexer struct {
|
|||
input []byte
|
||||
stateStart stateFunc
|
||||
state stateFunc
|
||||
pos pos // input position
|
||||
start pos // item start position
|
||||
width pos // width of last element
|
||||
pos Pos // input position
|
||||
start Pos // item start position
|
||||
width Pos // width of last element
|
||||
|
||||
// Set when we have parsed any summary divider
|
||||
summaryDividerChecked bool
|
||||
|
@ -73,7 +73,7 @@ func (l *pageLexer) Input() []byte {
|
|||
// note: the input position here is normally 0 (start), but
|
||||
// can be set if position of first shortcode is known
|
||||
// TODO(bep) 2errors byte
|
||||
func newPageLexer(input []byte, inputPosition pos, stateStart stateFunc) *pageLexer {
|
||||
func newPageLexer(input []byte, inputPosition Pos, stateStart stateFunc) *pageLexer {
|
||||
lexer := &pageLexer{
|
||||
input: input,
|
||||
pos: inputPosition,
|
||||
|
@ -131,7 +131,7 @@ func (l *pageLexer) next() rune {
|
|||
}
|
||||
|
||||
runeValue, runeWidth := utf8.DecodeRune(l.input[l.pos:])
|
||||
l.width = pos(runeWidth)
|
||||
l.width = Pos(runeWidth)
|
||||
l.pos += l.width
|
||||
return runeValue
|
||||
}
|
||||
|
@ -210,7 +210,7 @@ func lexMainSection(l *pageLexer) stateFunc {
|
|||
l3 = l.index(leftDelimSc)
|
||||
skip := minPositiveIndex(l1, l2, l3)
|
||||
if skip > 0 {
|
||||
l.pos += pos(skip)
|
||||
l.pos += Pos(skip)
|
||||
}
|
||||
|
||||
for {
|
||||
|
@ -234,7 +234,7 @@ func lexMainSection(l *pageLexer) stateFunc {
|
|||
l.emit(tText)
|
||||
}
|
||||
l.summaryDividerChecked = true
|
||||
l.pos += pos(len(summaryDivider))
|
||||
l.pos += Pos(len(summaryDivider))
|
||||
//l.consumeCRLF()
|
||||
l.emit(TypeLeadSummaryDivider)
|
||||
} else if l.hasPrefix(summaryDividerOrg) {
|
||||
|
@ -242,7 +242,7 @@ func lexMainSection(l *pageLexer) stateFunc {
|
|||
l.emit(tText)
|
||||
}
|
||||
l.summaryDividerChecked = true
|
||||
l.pos += pos(len(summaryDividerOrg))
|
||||
l.pos += Pos(len(summaryDividerOrg))
|
||||
//l.consumeCRLF()
|
||||
l.emit(TypeSummaryDividerOrg)
|
||||
}
|
||||
|
@ -291,12 +291,12 @@ LOOP:
|
|||
if right == -1 {
|
||||
return l.errorf("starting HTML comment with no end")
|
||||
}
|
||||
l.pos += pos(right) + pos(len(htmlCOmmentEnd))
|
||||
l.pos += Pos(right) + Pos(len(htmlCOmmentEnd))
|
||||
l.emit(TypeHTMLComment)
|
||||
} else {
|
||||
// Not need to look further. Hugo treats this as plain HTML,
|
||||
// no front matter, no shortcodes, no nothing.
|
||||
l.pos = pos(len(l.input))
|
||||
l.pos = Pos(len(l.input))
|
||||
l.emit(TypeHTMLDocument)
|
||||
}
|
||||
}
|
||||
|
@ -434,7 +434,7 @@ func (l *pageLexer) lexFrontMatterSection(tp ItemType, delimr rune, name string,
|
|||
}
|
||||
|
||||
func lexShortcodeLeftDelim(l *pageLexer) stateFunc {
|
||||
l.pos += pos(len(l.currentLeftShortcodeDelim()))
|
||||
l.pos += Pos(len(l.currentLeftShortcodeDelim()))
|
||||
if l.hasPrefix(leftComment) {
|
||||
return lexShortcodeComment
|
||||
}
|
||||
|
@ -451,20 +451,20 @@ func lexShortcodeComment(l *pageLexer) stateFunc {
|
|||
}
|
||||
// we emit all as text, except the comment markers
|
||||
l.emit(tText)
|
||||
l.pos += pos(len(leftComment))
|
||||
l.pos += Pos(len(leftComment))
|
||||
l.ignore()
|
||||
l.pos += pos(posRightComment - len(leftComment))
|
||||
l.pos += Pos(posRightComment - len(leftComment))
|
||||
l.emit(tText)
|
||||
l.pos += pos(len(rightComment))
|
||||
l.pos += Pos(len(rightComment))
|
||||
l.ignore()
|
||||
l.pos += pos(len(l.currentRightShortcodeDelim()))
|
||||
l.pos += Pos(len(l.currentRightShortcodeDelim()))
|
||||
l.emit(tText)
|
||||
return lexMainSection
|
||||
}
|
||||
|
||||
func lexShortcodeRightDelim(l *pageLexer) stateFunc {
|
||||
l.closingState = 0
|
||||
l.pos += pos(len(l.currentRightShortcodeDelim()))
|
||||
l.pos += Pos(len(l.currentRightShortcodeDelim()))
|
||||
l.emit(l.currentRightShortcodeDelimItem())
|
||||
return lexMainSection
|
||||
}
|
||||
|
|
|
@ -48,7 +48,7 @@ func Parse(r io.Reader) (Result, error) {
|
|||
}
|
||||
|
||||
func parseMainSection(input []byte, from int) Result {
|
||||
lexer := newPageLexer(input, pos(from), lexMainSection) // TODO(bep) 2errors
|
||||
lexer := newPageLexer(input, Pos(from), lexMainSection) // TODO(bep) 2errors
|
||||
lexer.run()
|
||||
return lexer
|
||||
}
|
||||
|
@ -57,7 +57,7 @@ func parseMainSection(input []byte, from int) Result {
|
|||
// if needed.
|
||||
type Iterator struct {
|
||||
l *pageLexer
|
||||
lastPos pos // position of the last item returned by nextItem
|
||||
lastPos Pos // position of the last item returned by nextItem
|
||||
}
|
||||
|
||||
// consumes and returns the next item
|
||||
|
@ -69,7 +69,7 @@ func (t *Iterator) Next() Item {
|
|||
var errIndexOutOfBounds = Item{tError, 0, []byte("no more tokens")}
|
||||
|
||||
func (t *Iterator) current() Item {
|
||||
if t.lastPos >= pos(len(t.l.items)) {
|
||||
if t.lastPos >= Pos(len(t.l.items)) {
|
||||
return errIndexOutOfBounds
|
||||
}
|
||||
return t.l.items[t.lastPos]
|
||||
|
@ -98,7 +98,7 @@ func (t *Iterator) Peek() Item {
|
|||
// PeekWalk will feed the next items in the iterator to walkFn
|
||||
// until it returns false.
|
||||
func (t *Iterator) PeekWalk(walkFn func(item Item) bool) {
|
||||
for i := t.lastPos + 1; i < pos(len(t.l.items)); i++ {
|
||||
for i := t.lastPos + 1; i < Pos(len(t.l.items)); i++ {
|
||||
item := t.l.items[i]
|
||||
if !walkFn(item) {
|
||||
break
|
||||
|
@ -120,5 +120,5 @@ func (t *Iterator) Consume(cnt int) {
|
|||
|
||||
// LineNumber returns the current line number. Used for logging.
|
||||
func (t *Iterator) LineNumber() int {
|
||||
return bytes.Count(t.l.input[:t.current().pos], lf) + 1
|
||||
return bytes.Count(t.l.input[:t.current().Pos], lf) + 1
|
||||
}
|
||||
|
|
|
@ -59,9 +59,7 @@ var frontMatterTests = []lexerTest{
|
|||
{"No front matter", "\nSome text.\n", []Item{tstSomeText, tstEOF}},
|
||||
{"YAML front matter", "---\nfoo: \"bar\"\n---\n\nSome text.\n", []Item{tstFrontMatterYAML, tstSomeText, tstEOF}},
|
||||
{"YAML empty front matter", "---\n---\n\nSome text.\n", []Item{nti(TypeFrontMatterYAML, "\n"), tstSomeText, tstEOF}},
|
||||
|
||||
{"YAML commented out front matter", "<!--\n---\nfoo: \"bar\"\n---\n-->\nSome text.\n", []Item{nti(TypeHTMLComment, "<!--\n---\nfoo: \"bar\"\n---\n-->"), tstSomeText, tstEOF}},
|
||||
|
||||
// Note that we keep all bytes as they are, but we need to handle CRLF
|
||||
{"YAML front matter CRLF", "---\r\nfoo: \"bar\"\r\n---\n\nSome text.\n", []Item{tstFrontMatterYAMLCRLF, tstSomeText, tstEOF}},
|
||||
{"TOML front matter", "+++\nfoo = \"bar\"\n+++\n\nSome text.\n", []Item{tstFrontMatterTOML, tstSomeText, tstEOF}},
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/parser"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
"github.com/spf13/cast"
|
||||
)
|
||||
|
||||
|
@ -38,21 +39,7 @@ func (ns *Namespace) Remarshal(format string, data interface{}) (string, error)
|
|||
return "", err
|
||||
}
|
||||
|
||||
var metaHandler func(d []byte) (map[string]interface{}, error)
|
||||
|
||||
switch fromFormat {
|
||||
case "yaml":
|
||||
metaHandler = parser.HandleYAMLMetaData
|
||||
case "toml":
|
||||
metaHandler = parser.HandleTOMLMetaData
|
||||
case "json":
|
||||
metaHandler = parser.HandleJSONMetaData
|
||||
}
|
||||
|
||||
meta, err := metaHandler([]byte(from))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
meta, err := metadecoders.UnmarshalToMap([]byte(from), fromFormat)
|
||||
|
||||
var result bytes.Buffer
|
||||
if err := parser.InterfaceToConfig(meta, mark, &result); err != nil {
|
||||
|
@ -76,21 +63,21 @@ func toFormatMark(format string) (rune, error) {
|
|||
return 0, errors.New("failed to detect target data serialization format")
|
||||
}
|
||||
|
||||
func detectFormat(data string) (string, error) {
|
||||
func detectFormat(data string) (metadecoders.Format, error) {
|
||||
jsonIdx := strings.Index(data, "{")
|
||||
yamlIdx := strings.Index(data, ":")
|
||||
tomlIdx := strings.Index(data, "=")
|
||||
|
||||
if jsonIdx != -1 && (yamlIdx == -1 || jsonIdx < yamlIdx) && (tomlIdx == -1 || jsonIdx < tomlIdx) {
|
||||
return "json", nil
|
||||
return metadecoders.JSON, nil
|
||||
}
|
||||
|
||||
if yamlIdx != -1 && (tomlIdx == -1 || yamlIdx < tomlIdx) {
|
||||
return "yaml", nil
|
||||
return metadecoders.YAML, nil
|
||||
}
|
||||
|
||||
if tomlIdx != -1 {
|
||||
return "toml", nil
|
||||
return metadecoders.TOML, nil
|
||||
}
|
||||
|
||||
return "", errors.New("failed to detect data serialization format")
|
||||
|
|
|
@ -18,6 +18,7 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/parser/metadecoders"
|
||||
"github.com/spf13/viper"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
@ -179,12 +180,12 @@ func TestRemarshalDetectFormat(t *testing.T) {
|
|||
data string
|
||||
expect interface{}
|
||||
}{
|
||||
{`foo = "bar"`, "toml"},
|
||||
{` foo = "bar"`, "toml"},
|
||||
{`foo="bar"`, "toml"},
|
||||
{`foo: "bar"`, "yaml"},
|
||||
{`foo:"bar"`, "yaml"},
|
||||
{`{ "foo": "bar"`, "json"},
|
||||
{`foo = "bar"`, metadecoders.TOML},
|
||||
{` foo = "bar"`, metadecoders.TOML},
|
||||
{`foo="bar"`, metadecoders.TOML},
|
||||
{`foo: "bar"`, metadecoders.YAML},
|
||||
{`foo:"bar"`, metadecoders.YAML},
|
||||
{`{ "foo": "bar"`, metadecoders.JSON},
|
||||
{`asdfasdf`, false},
|
||||
{``, false},
|
||||
} {
|
||||
|
@ -198,6 +199,6 @@ func TestRemarshalDetectFormat(t *testing.T) {
|
|||
}
|
||||
|
||||
assert.NoError(err, errMsg)
|
||||
assert.Equal(test.expect, result, errMsg)
|
||||
assert.Equal(test.expect, result)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue