From a3701e09313695d4a0f6fb0eb7844c1a4befc07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Tue, 27 Jul 2021 19:07:10 +0200 Subject: [PATCH] Switch to go-toml v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have been using `go-toml` for language files only. This commit makes it the only TOML library. It's spec compliant and very fast. A benchark building a site with 200 pages with TOML front matter: ```bash name old time/op new time/op delta SiteNew/Regular_TOML_front_matter-16 48.5ms ± 1% 47.1ms ± 1% -2.85% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Regular_TOML_front_matter-16 16.9MB ± 0% 16.7MB ± 0% -1.56% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Regular_TOML_front_matter-16 302k ± 0% 296k ± 0% -2.20% (p=0.029 n=4+4) ``` Note that the front matter unmarshaling is only a small part of building a site, so the above is very good. Fixes #8801 --- commands/commands_test.go | 2 +- common/herrors/file_error_test.go | 1 - common/herrors/line_number_extractors.go | 15 ++++++++++++-- go.mod | 3 +-- go.sum | 5 ++++- hugolib/config_test.go | 24 +++++++++++----------- hugolib/configdir_test.go | 4 ++-- langs/i18n/translationProvider.go | 2 +- parser/frontmatter.go | 2 +- parser/frontmatter_test.go | 4 ++-- parser/metadecoders/decoder.go | 2 +- tpl/transform/remarshal_test.go | 26 +++++++++++------------- 12 files changed, 50 insertions(+), 40 deletions(-) diff --git a/commands/commands_test.go b/commands/commands_test.go index 99ffea48c..ef61dc7a5 100644 --- a/commands/commands_test.go +++ b/commands/commands_test.go @@ -121,7 +121,7 @@ func TestExecute(t *testing.T) { resp := Execute([]string{"new", "site", siteDir, "-e=staging"}) c.Assert(resp.Err, qt.IsNil) config := readFileFrom(c, filepath.Join(siteDir, "config.toml")) - c.Assert(config, qt.Contains, "baseURL = \"http://example.org/\"") + c.Assert(config, qt.Contains, "baseURL = 'http://example.org/'") checkNewSiteInited(c, siteDir) }) } diff --git a/common/herrors/file_error_test.go b/common/herrors/file_error_test.go index 4b42f8f39..675be94e8 100644 --- a/common/herrors/file_error_test.go +++ b/common/herrors/file_error_test.go @@ -36,7 +36,6 @@ func TestToLineNumberError(t *testing.T) { {errors.New(`template: _default/single.html:4:15: executing "_default/single.html" at <.Titles>: can't evaluate field Titles in type *hugolib.PageOutput`), 0, 4, 15}, {errors.New("parse failed: template: _default/bundle-resource-meta.html:11: unexpected in operand"), 0, 11, 1}, {errors.New(`failed:: template: _default/bundle-resource-meta.html:2:7: executing "main" at <.Titles>`), 0, 2, 7}, - {errors.New("error in front matter: Near line 32 (last key parsed 'title')"), 0, 32, 1}, {errors.New(`failed to load translations: (6, 7): was expecting token =, but got "g" instead`), 0, 6, 7}, } { diff --git a/common/herrors/line_number_extractors.go b/common/herrors/line_number_extractors.go index 13e94614d..3df62bdfe 100644 --- a/common/herrors/line_number_extractors.go +++ b/common/herrors/line_number_extractors.go @@ -16,6 +16,10 @@ package herrors import ( "regexp" "strconv" + + "github.com/pkg/errors" + + "github.com/pelletier/go-toml/v2" ) var lineNumberExtractors = []lineNumberExtractor{ @@ -24,8 +28,7 @@ var lineNumberExtractors = []lineNumberExtractor{ newLineNumberErrHandlerFromRegexp(".*:(\\d+):"), // TOML parse errors - newLineNumberErrHandlerFromRegexp(".*Near line (\\d+)(\\s.*)"), - + tomlLineNumberExtractor, // YAML parse errors newLineNumberErrHandlerFromRegexp("line (\\d+):"), @@ -35,6 +38,14 @@ var lineNumberExtractors = []lineNumberExtractor{ type lineNumberExtractor func(e error) (int, int) +var tomlLineNumberExtractor = func(e error) (int, int) { + e = errors.Cause(e) + if terr, ok := e.(*toml.DecodeError); ok { + return terr.Position() + } + return -1, -1 +} + func newLineNumberErrHandlerFromRegexp(expression string) lineNumberExtractor { re := regexp.MustCompile(expression) return extractLineNo(re) diff --git a/go.mod b/go.mod index 219e9d64d..a04655eec 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/gohugoio/hugo require ( github.com/BurntSushi/locker v0.0.0-20171006230638-a6e239ea1c69 - github.com/BurntSushi/toml v0.3.1 github.com/PuerkitoBio/purell v1.1.1 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alecthomas/chroma v0.9.2 @@ -44,7 +43,7 @@ require ( github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect github.com/niklasfasching/go-org v1.5.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/pelletier/go-toml v1.9.3 + github.com/pelletier/go-toml/v2 v2.0.0-beta.3 github.com/pkg/errors v0.9.1 github.com/rogpeppe/go-internal v1.8.0 github.com/russross/blackfriday v1.5.3-0.20200218234912-41c5fccfd6f6 diff --git a/go.sum b/go.sum index 3eb5bd4c3..b38926d9d 100644 --- a/go.sum +++ b/go.sum @@ -433,6 +433,8 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.0-beta.3 h1:PNCTU4naEJ8mKal97P3A2qDU74QRQGlv4FXiL1XDqi4= +github.com/pelletier/go-toml/v2 v2.0.0-beta.3/go.mod h1:aNseLYu/uKskg0zpr/kbr2z8yGuWtotWf/0BpGIAL2Y= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -490,8 +492,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU= +github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tdewolff/minify/v2 v2.9.20 h1:Fut7w3T7nWfDOb/bOgyEvshQRRMt+xzi1T7spEEKXDw= diff --git a/hugolib/config_test.go b/hugolib/config_test.go index 575931975..bda1c7fb5 100644 --- a/hugolib/config_test.go +++ b/hugolib/config_test.go @@ -287,13 +287,13 @@ name = "menu-theme" "pl1": "p1-en-main", }, "menus": maps.Params{ - "main": []map[string]interface{}{ - { + "main": []interface{}{ + map[string]interface{}{ "name": "menu-lang-en-main", }, }, - "theme": []map[string]interface{}{ - { + "theme": []interface{}{ + map[string]interface{}{ "name": "menu-lang-en-theme", }, }, @@ -307,18 +307,18 @@ name = "menu-theme" "pl2": "p2-nb-theme", }, "menus": maps.Params{ - "main": []map[string]interface{}{ - { + "main": []interface{}{ + map[string]interface{}{ "name": "menu-lang-nb-main", }, }, - "theme": []map[string]interface{}{ - { + "theme": []interface{}{ + map[string]interface{}{ "name": "menu-lang-nb-theme", }, }, - "top": []map[string]interface{}{ - { + "top": []interface{}{ + map[string]interface{}{ "name": "menu-lang-nb-top", }, }, @@ -393,8 +393,8 @@ name = "menu-theme" "en": maps.Params{ "languagename": "English", "menus": maps.Params{ - "main": []map[string]interface{}{ - { + "main": []interface{}{ + map[string]interface{}{ "name": "menu-theme", }, }, diff --git a/hugolib/configdir_test.go b/hugolib/configdir_test.go index aa1013ec0..b3fc71e58 100644 --- a/hugolib/configdir_test.go +++ b/hugolib/configdir_test.go @@ -119,10 +119,10 @@ p3 = "p3params_no_production" c.Assert(cfg.GetString("params.p3"), qt.Equals, "p3params_development") c.Assert(cfg.GetString("languages.no.params.p3"), qt.Equals, "p3params_no_development") - c.Assert(len(cfg.Get("menus.docs").(([]map[string]interface{}))), qt.Equals, 2) + c.Assert(len(cfg.Get("menus.docs").([]interface{})), qt.Equals, 2) noMenus := cfg.Get("languages.no.menus.docs") c.Assert(noMenus, qt.Not(qt.IsNil)) - c.Assert(len(noMenus.(([]map[string]interface{}))), qt.Equals, 1) + c.Assert(len(noMenus.([]interface{})), qt.Equals, 1) } func TestLoadConfigDirError(t *testing.T) { diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go index c66505817..e8bfbf6f1 100644 --- a/langs/i18n/translationProvider.go +++ b/langs/i18n/translationProvider.go @@ -25,7 +25,7 @@ import ( "github.com/gohugoio/go-i18n/v2/i18n" "github.com/gohugoio/hugo/helpers" - toml "github.com/pelletier/go-toml" + toml "github.com/pelletier/go-toml/v2" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" diff --git a/parser/frontmatter.go b/parser/frontmatter.go index 4965d3fe8..e7492745e 100644 --- a/parser/frontmatter.go +++ b/parser/frontmatter.go @@ -20,7 +20,7 @@ import ( "github.com/gohugoio/hugo/parser/metadecoders" - "github.com/BurntSushi/toml" + toml "github.com/pelletier/go-toml/v2" yaml "gopkg.in/yaml.v2" ) diff --git a/parser/frontmatter_test.go b/parser/frontmatter_test.go index 9d9b7c3b8..dfbc695d8 100644 --- a/parser/frontmatter_test.go +++ b/parser/frontmatter_test.go @@ -31,9 +31,9 @@ func TestInterfaceToConfig(t *testing.T) { // TOML {map[string]interface{}{}, metadecoders.TOML, nil, false}, { - map[string]interface{}{"title": "test 1"}, + map[string]interface{}{"title": "test' 1"}, metadecoders.TOML, - []byte("title = \"test 1\"\n"), + []byte("title = \"test' 1\"\n"), false, }, diff --git a/parser/metadecoders/decoder.go b/parser/metadecoders/decoder.go index 131d6f39b..168c130ed 100644 --- a/parser/metadecoders/decoder.go +++ b/parser/metadecoders/decoder.go @@ -24,7 +24,7 @@ import ( "github.com/gohugoio/hugo/common/herrors" "github.com/niklasfasching/go-org/org" - "github.com/BurntSushi/toml" + toml "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" "github.com/spf13/afero" "github.com/spf13/cast" diff --git a/tpl/transform/remarshal_test.go b/tpl/transform/remarshal_test.go index 9f3e05e61..c34122a2a 100644 --- a/tpl/transform/remarshal_test.go +++ b/tpl/transform/remarshal_test.go @@ -30,20 +30,20 @@ func TestRemarshal(t *testing.T) { ns := New(newDeps(v)) c := qt.New(t) - tomlExample := `title = "Test Metadata" + tomlExample := `title = 'Test Metadata' [[resources]] - src = "**image-4.png" - title = "The Fourth Image!" + src = '**image-4.png' + title = 'The Fourth Image!' [resources.params] - byline = "picasso" + byline = 'picasso' [[resources]] - name = "my-cool-image-:counter" - src = "**.png" - title = "TOML: The Image #:counter" + name = 'my-cool-image-:counter' + src = '**.png' + title = 'TOML: The Image #:counter' [resources.params] - byline = "bep" + byline = 'bep' ` yamlExample := `resources: @@ -129,11 +129,9 @@ a = "b" ` - expected := ` -Hugo = "Rules" - + expected := `Hugo = 'Rules' [m] - a = "b" +a = 'b' ` for _, format := range []string{"json", "yaml", "toml"} { @@ -149,7 +147,7 @@ Hugo = "Rules" diff := htesting.DiffStrings(expected, converted) if len(diff) > 0 { - t.Fatalf("[%s] Expected \n%v\ngot\n%v\ndiff:\n%v\n", fromTo, expected, converted, diff) + t.Fatalf("[%s] Expected \n%v\ngot\n>>%v\ndiff:\n%v\n", fromTo, expected, converted, diff) } } } @@ -182,5 +180,5 @@ func TestTestRemarshalMapInput(t *testing.T) { output, err := ns.Remarshal("toml", input) c.Assert(err, qt.IsNil) - c.Assert(output, qt.Equals, "hello = \"world\"\n") + c.Assert(output, qt.Equals, "hello = 'world'\n") }