From 574c2959b8d3338764fa1db102a5e0fd6ed322d9 Mon Sep 17 00:00:00 2001 From: SatowTakeshi Date: Sat, 29 Feb 2020 18:44:05 +0900 Subject: [PATCH] Add minify config Fixes #6750 Updates #6892 --- docs/data/docs.json | 48 ++++++++ docshelper/docs.go | 16 ++- hugolib/hugo_sites.go | 7 +- markup/markup_config/config.go | 1 - minifiers/config.go | 111 ++++++++++++++++++ minifiers/config_test.go | 52 ++++++++ minifiers/minifiers.go | 60 +++++----- minifiers/minifiers_test.go | 44 ++++++- publisher/publisher.go | 13 +- .../resource_transformers/minifier/minify.go | 12 +- .../minifier/minify_test.go | 22 +++- tpl/resources/resources.go | 7 +- 12 files changed, 346 insertions(+), 47 deletions(-) create mode 100644 minifiers/config.go create mode 100644 minifiers/config_test.go diff --git a/docs/data/docs.json b/docs/data/docs.json index 942f04fb0..2af976318 100644 --- a/docs/data/docs.json +++ b/docs/data/docs.json @@ -1451,6 +1451,35 @@ "footnoteAnchorPrefix": "", "footnoteReturnLinkContents": "" } + }, + "minifiers": { + "tdewolff": { + "enableHtml": true, + "enableCss": true, + "enableJs": true, + "enableJson": true, + "enableSvg": true, + "enableXml": true, + "html": { + "keepConditionalComments": true, + "keepDefaultAttrVals": true, + "keepDocumentTags": true, + "keepEndTags": true, + "keepWhitespace": false + }, + "css": { + "decimals": -1, + "keepCSS2": true + }, + "js": {}, + "json": {}, + "svg": { + "decimals": -1 + }, + "xml": { + "keepWhitespace": false + } + } } }, "media": { @@ -3192,6 +3221,12 @@ "Aliases": null, "Examples": null }, + "IsProduction": { + "Description": "", + "Args": null, + "Aliases": null, + "Examples": null + }, "Version": { "Description": "", "Args": null, @@ -3544,6 +3579,19 @@ ] ] }, + "Sqrt": { + "Description": "Sqrt returns the square root of a number.\nNOTE: will return for NaN for negative values of a", + "Args": [ + "a" + ], + "Aliases": null, + "Examples": [ + [ + "{{math.Sqrt 81}}", + "9" + ] + ] + }, "Sub": { "Description": "Sub subtracts two numbers.", "Args": [ diff --git a/docshelper/docs.go b/docshelper/docs.go index 94cb70dec..17e0ccd91 100644 --- a/docshelper/docs.go +++ b/docshelper/docs.go @@ -24,7 +24,11 @@ var DocProviders = make(map[string]DocProvider) // AddDocProvider adds or updates the DocProvider for a given name. func AddDocProvider(name string, provider DocProvider) { - DocProviders[name] = provider + if prev, ok := DocProviders[name]; !ok { + DocProviders[name] = provider + } else { + DocProviders[name] = merge(prev, provider) + } } // DocProvider is used to save arbitrary JSON data @@ -35,3 +39,13 @@ type DocProvider func() map[string]interface{} func (d DocProvider) MarshalJSON() ([]byte, error) { return json.MarshalIndent(d(), "", " ") } + +func merge(a, b DocProvider) DocProvider { + next := a() + for k, v := range b() { + next[k] = v + } + return func() map[string]interface{} { + return next + } +} diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 04a231fdf..dca9e4968 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -408,7 +408,12 @@ func applyDeps(cfg deps.DepsCfg, sites ...*Site) error { s.Deps = d // Set up the main publishing chain. - s.publisher = publisher.NewDestinationPublisher(d.PathSpec.BaseFs.PublishFs, s.outputFormatsConfig, s.mediaTypesConfig, cfg.Cfg.GetBool("minify")) + pub, err := publisher.NewDestinationPublisher(d.PathSpec.BaseFs.PublishFs, s.outputFormatsConfig, s.mediaTypesConfig, cfg.Cfg) + + if err != nil { + return err + } + s.publisher = pub if err := s.initializeSiteInfo(); err != nil { return err diff --git a/markup/markup_config/config.go b/markup/markup_config/config.go index 529553cb5..34c8807ac 100644 --- a/markup/markup_config/config.go +++ b/markup/markup_config/config.go @@ -100,6 +100,5 @@ func init() { return docs } - // TODO(bep) merge maps docshelper.AddDocProvider("config", docsProvider) } diff --git a/minifiers/config.go b/minifiers/config.go new file mode 100644 index 000000000..20d122e9b --- /dev/null +++ b/minifiers/config.go @@ -0,0 +1,111 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minifiers + +import ( + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/docshelper" + "github.com/gohugoio/hugo/parser" + + "github.com/mitchellh/mapstructure" + "github.com/tdewolff/minify/v2/css" + "github.com/tdewolff/minify/v2/html" + "github.com/tdewolff/minify/v2/js" + "github.com/tdewolff/minify/v2/json" + "github.com/tdewolff/minify/v2/svg" + "github.com/tdewolff/minify/v2/xml" +) + +var defaultTdewolffConfig = tdewolffConfig{ + HTML: html.Minifier{ + KeepDocumentTags: true, + KeepConditionalComments: true, + KeepEndTags: true, + KeepDefaultAttrVals: true, + KeepWhitespace: false, + // KeepQuotes: false, >= v2.6.2 + }, + CSS: css.Minifier{ + Decimals: -1, // will be deprecated + // Precision: 0, // use Precision with >= v2.7.0 + KeepCSS2: true, + }, + JS: js.Minifier{}, + JSON: json.Minifier{}, + SVG: svg.Minifier{ + Decimals: -1, // will be deprecated + // Precision: 0, // use Precision with >= v2.7.0 + }, + XML: xml.Minifier{ + KeepWhitespace: false, + }, +} + +type tdewolffConfig struct { + HTML html.Minifier + CSS css.Minifier + JS js.Minifier + JSON json.Minifier + SVG svg.Minifier + XML xml.Minifier +} + +type minifiersConfig struct { + EnableHTML bool + EnableCSS bool + EnableJS bool + EnableJSON bool + EnableSVG bool + EnableXML bool + + Tdewolff tdewolffConfig +} + +var defaultConfig = minifiersConfig{ + EnableHTML: true, + EnableCSS: true, + EnableJS: true, + EnableJSON: true, + EnableSVG: true, + EnableXML: true, + + Tdewolff: defaultTdewolffConfig, +} + +func decodeConfig(cfg config.Provider) (conf minifiersConfig, err error) { + conf = defaultConfig + + m := cfg.GetStringMap("minifiers") + if m == nil { + return + } + + err = mapstructure.WeakDecode(m, &conf) + + if err != nil { + return + } + + return +} + +func init() { + docsProvider := func() map[string]interface{} { + docs := make(map[string]interface{}) + docs["minifiers"] = parser.LowerCaseCamelJSONMarshaller{Value: defaultConfig} + return docs + + } + docshelper.AddDocProvider("config", docsProvider) +} diff --git a/minifiers/config_test.go b/minifiers/config_test.go new file mode 100644 index 000000000..b4f68f8c7 --- /dev/null +++ b/minifiers/config_test.go @@ -0,0 +1,52 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package minifiers + +import ( + "fmt" + "testing" + + "github.com/spf13/viper" + + qt "github.com/frankban/quicktest" +) + +func TestConfig(t *testing.T) { + c := qt.New(t) + v := viper.New() + + v.Set("minifiers", map[string]interface{}{ + "enablexml": false, + "tdewolff": map[string]interface{}{ + "html": map[string]interface{}{ + "keepwhitespace": false, + }, + }, + }) + + conf, err := decodeConfig(v) + fmt.Println(conf) + + c.Assert(err, qt.IsNil) + + // explicitly set value + c.Assert(conf.Tdewolff.HTML.KeepWhitespace, qt.Equals, false) + // default value + c.Assert(conf.Tdewolff.HTML.KeepEndTags, qt.Equals, true) + c.Assert(conf.Tdewolff.CSS.KeepCSS2, qt.Equals, true) + + // `enable` flags + c.Assert(conf.EnableHTML, qt.Equals, true) + c.Assert(conf.EnableXML, qt.Equals, false) +} diff --git a/minifiers/minifiers.go b/minifiers/minifiers.go index 9533ebb69..3feb81601 100644 --- a/minifiers/minifiers.go +++ b/minifiers/minifiers.go @@ -20,17 +20,12 @@ import ( "io" "regexp" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/output" "github.com/gohugoio/hugo/transform" "github.com/gohugoio/hugo/media" "github.com/tdewolff/minify/v2" - "github.com/tdewolff/minify/v2/css" - "github.com/tdewolff/minify/v2/html" - "github.com/tdewolff/minify/v2/js" - "github.com/tdewolff/minify/v2/json" - "github.com/tdewolff/minify/v2/svg" - "github.com/tdewolff/minify/v2/xml" ) // Client wraps a minifier. @@ -62,39 +57,44 @@ func (m Client) Minify(mediatype media.Type, dst io.Writer, src io.Reader) error // New creates a new Client with the provided MIME types as the mapping foundation. // The HTML minifier is also registered for additional HTML types (AMP etc.) in the // provided list of output formats. -func New(mediaTypes media.Types, outputFormats output.Formats) Client { - m := minify.New() - htmlMin := &html.Minifier{ - KeepDocumentTags: true, - KeepConditionalComments: true, - KeepEndTags: true, - KeepDefaultAttrVals: true, - } +func New(mediaTypes media.Types, outputFormats output.Formats, cfg config.Provider) (Client, error) { + conf, err := decodeConfig(cfg) - cssMin := &css.Minifier{ - Decimals: -1, - KeepCSS2: true, + m := minify.New() + if err != nil { + return Client{m: m}, err } // We use the Type definition of the media types defined in the site if found. - addMinifier(m, mediaTypes, "css", cssMin) - addMinifierFunc(m, mediaTypes, "js", js.Minify) - m.AddFuncRegexp(regexp.MustCompile("^(application|text)/(x-)?(java|ecma)script$"), js.Minify) - m.AddFuncRegexp(regexp.MustCompile(`^(application|text)/(x-|ld\+)?json$`), json.Minify) - addMinifierFunc(m, mediaTypes, "json", json.Minify) - addMinifierFunc(m, mediaTypes, "svg", svg.Minify) - addMinifierFunc(m, mediaTypes, "xml", xml.Minify) + if conf.EnableCSS { + addMinifier(m, mediaTypes, "css", &conf.Tdewolff.CSS) + } + if conf.EnableJS { + addMinifier(m, mediaTypes, "js", &conf.Tdewolff.JS) + m.AddRegexp(regexp.MustCompile("^(application|text)/(x-)?(java|ecma)script$"), &conf.Tdewolff.JS) + } + if conf.EnableJSON { + addMinifier(m, mediaTypes, "json", &conf.Tdewolff.JSON) + m.AddRegexp(regexp.MustCompile(`^(application|text)/(x-|ld\+)?json$`), &conf.Tdewolff.JSON) + } + if conf.EnableSVG { + addMinifier(m, mediaTypes, "svg", &conf.Tdewolff.SVG) + } + if conf.EnableXML { + addMinifier(m, mediaTypes, "xml", &conf.Tdewolff.XML) + } // HTML - addMinifier(m, mediaTypes, "html", htmlMin) - for _, of := range outputFormats { - if of.IsHTML { - m.Add(of.MediaType.Type(), htmlMin) + if conf.EnableHTML { + addMinifier(m, mediaTypes, "html", &conf.Tdewolff.HTML) + for _, of := range outputFormats { + if of.IsHTML { + m.Add(of.MediaType.Type(), &conf.Tdewolff.HTML) + } } } - return Client{m: m} - + return Client{m: m}, nil } func addMinifier(m *minify.M, mt media.Types, suffix string, min minify.Minifier) { diff --git a/minifiers/minifiers_test.go b/minifiers/minifiers_test.go index 87e706320..64588f83c 100644 --- a/minifiers/minifiers_test.go +++ b/minifiers/minifiers_test.go @@ -23,11 +23,13 @@ import ( qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/output" + "github.com/spf13/viper" ) func TestNew(t *testing.T) { c := qt.New(t) - m := New(media.DefaultTypes, output.DefaultFormats) + v := viper.New() + m, _ := New(media.DefaultTypes, output.DefaultFormats, v) var rawJS string var minJS string @@ -73,9 +75,44 @@ func TestNew(t *testing.T) { } +func TestConfiguredMinify(t *testing.T) { + c := qt.New(t) + v := viper.New() + v.Set("minifiers", map[string]interface{}{ + "enablexml": false, + "tdewolff": map[string]interface{}{ + "html": map[string]interface{}{ + "keepwhitespace": true, + }, + }, + }) + m, _ := New(media.DefaultTypes, output.DefaultFormats, v) + + for _, test := range []struct { + tp media.Type + rawString string + expectedMinString string + errorExpected bool + }{ + {media.HTMLType, " Hugo! ", " Hugo! ", false}, // configured minifier + {media.CSSType, " body { color: blue; } ", "body{color:blue}", false}, // default minifier + {media.XMLType, " Hugo! ", "", true}, // disable Xml minificatin + } { + var b bytes.Buffer + if !test.errorExpected { + c.Assert(m.Minify(test.tp, &b, strings.NewReader(test.rawString)), qt.IsNil) + c.Assert(b.String(), qt.Equals, test.expectedMinString) + } else { + err := m.Minify(test.tp, &b, strings.NewReader(test.rawString)) + c.Assert(err, qt.ErrorMatches, "minifier does not exist for mimetype") + } + } +} + func TestJSONRoundTrip(t *testing.T) { c := qt.New(t) - m := New(media.DefaultTypes, output.DefaultFormats) + v := viper.New() + m, _ := New(media.DefaultTypes, output.DefaultFormats, v) for _, test := range []string{`{ "glossary": { @@ -113,7 +150,8 @@ func TestJSONRoundTrip(t *testing.T) { func TestBugs(t *testing.T) { c := qt.New(t) - m := New(media.DefaultTypes, output.DefaultFormats) + v := viper.New() + m, _ := New(media.DefaultTypes, output.DefaultFormats, v) for _, test := range []struct { tp media.Type diff --git a/publisher/publisher.go b/publisher/publisher.go index 119be356b..e1179572b 100644 --- a/publisher/publisher.go +++ b/publisher/publisher.go @@ -18,6 +18,7 @@ import ( "io" "sync/atomic" + "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/minifiers" @@ -73,13 +74,17 @@ type DestinationPublisher struct { } // NewDestinationPublisher creates a new DestinationPublisher. -func NewDestinationPublisher(fs afero.Fs, outputFormats output.Formats, mediaTypes media.Types, minify bool) DestinationPublisher { - pub := DestinationPublisher{fs: fs} +func NewDestinationPublisher(fs afero.Fs, outputFormats output.Formats, mediaTypes media.Types, cfg config.Provider) (pub DestinationPublisher, err error) { + pub = DestinationPublisher{fs: fs} + minify := cfg.GetBool("minify") if minify { - pub.min = minifiers.New(mediaTypes, outputFormats) + pub.min, err = minifiers.New(mediaTypes, outputFormats, cfg) + if err != nil { + return + } pub.minify = true } - return pub + return } // Publish applies any relevant transformations and writes the file diff --git a/resources/resource_transformers/minifier/minify.go b/resources/resource_transformers/minifier/minify.go index 38e3fc93a..060485e80 100644 --- a/resources/resource_transformers/minifier/minify.go +++ b/resources/resource_transformers/minifier/minify.go @@ -29,8 +29,12 @@ type Client struct { // New creates a new Client given a specification. Note that it is the media types // configured for the site that is used to match files to the correct minifier. -func New(rs *resources.Spec) *Client { - return &Client{rs: rs, m: minifiers.New(rs.MediaTypes, rs.OutputFormats)} +func New(rs *resources.Spec) (*Client, error) { + m, err := minifiers.New(rs.MediaTypes, rs.OutputFormats, rs.Cfg) + if err != nil { + return nil, err + } + return &Client{rs: rs, m: m}, nil } type minifyTransformation struct { @@ -43,9 +47,7 @@ func (t *minifyTransformation) Key() internal.ResourceTransformationKey { } func (t *minifyTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - if err := t.m.Minify(ctx.InMediaType, ctx.To, ctx.From); err != nil { - return err - } + _ = t.m.Minify(ctx.InMediaType, ctx.To, ctx.From) ctx.AddOutPathIdentifier(".min") return nil } diff --git a/resources/resource_transformers/minifier/minify_test.go b/resources/resource_transformers/minifier/minify_test.go index 3f8853520..a1b22f109 100644 --- a/resources/resource_transformers/minifier/minify_test.go +++ b/resources/resource_transformers/minifier/minify_test.go @@ -27,7 +27,7 @@ func TestTransform(t *testing.T) { spec, err := htesting.NewTestResourceSpec() c.Assert(err, qt.IsNil) - client := New(spec) + client, _ := New(spec) r, err := htesting.NewResourceTransformerForSpec(spec, "hugo.html", "

Hugo Rocks!

") c.Assert(err, qt.IsNil) @@ -41,3 +41,23 @@ func TestTransform(t *testing.T) { c.Assert(content, qt.Equals, "

Hugo Rocks!

") } + +func TestNoMinifier(t *testing.T) { + c := qt.New(t) + + spec, _ := htesting.NewTestResourceSpec() + spec.Cfg.Set("minifiers.enableXML", false) + client, _ := New(spec) + + original := " Hugo Rocks! " + r, err := htesting.NewResourceTransformerForSpec(spec, "hugo.xml", original) + c.Assert(err, qt.IsNil) + + transformed, err := client.Minify(r) + c.Assert(err, qt.IsNil) + + content, err := transformed.(resource.ContentProvider).Content() + // error should be ignored because general users cannot control codes under `theme`s + c.Assert(err, qt.IsNil) + c.Assert(content, qt.Equals, original) +} diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index be10510f2..a1055632c 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -46,13 +46,18 @@ func New(deps *deps.Deps) (*Namespace, error) { return nil, err } + minifyClient, err := minifier.New(deps.ResourceSpec) + if err != nil { + return nil, err + } + return &Namespace{ deps: deps, scssClient: scssClient, createClient: create.New(deps.ResourceSpec), bundlerClient: bundler.New(deps.ResourceSpec), integrityClient: integrity.New(deps.ResourceSpec), - minifyClient: minifier.New(deps.ResourceSpec), + minifyClient: minifyClient, postcssClient: postcss.New(deps.ResourceSpec), templatesClient: templates.New(deps.ResourceSpec, deps), }, nil