From f4389e48ce0a70807362772d66c12ab5cd9e15f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sun, 12 Dec 2021 12:11:11 +0100 Subject: [PATCH] Add some basic security policies with sensible defaults This ommmit contains some security hardening measures for the Hugo build runtime. There are some rarely used features in Hugo that would be good to have disabled by default. One example would be the "external helpers". For `asciidoctor` and some others we use Go's `os/exec` package to start a new process. These are a predefined set of binary names, all loaded from `PATH` and with a predefined set of arguments. Still, if you don't use `asciidoctor` in your project, you might as well have it turned off. You can configure your own in the new `security` configuration section, but the defaults are configured to create a minimal amount of site breakage. And if that do happen, you will get clear instructions in the loa about what to do. The default configuration is listed below. Note that almost all of these options are regular expression _whitelists_ (a string or a slice); the value `none` will block all. ```toml [security] enableInlineShortcodes = false [security.exec] allow = ['^dart-sass-embedded$', '^go$', '^npx$', '^postcss$'] osEnv = ['(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$'] [security.funcs] getenv = ['^HUGO_'] [security.http] methods = ['(?i)GET|POST'] urls = ['.*'] ``` --- common/collections/slice.go | 10 + common/herrors/errors.go | 7 + common/hexec/exec.go | 276 ++++++++++++++++++ common/hexec/safeCommand.go | 45 --- common/hugo/hugo.go | 19 +- config/defaultConfigProvider.go | 2 + config/security/docshelper.go | 26 ++ config/security/securityConfig.go | 227 ++++++++++++++ config/security/securityonfig_test.go | 166 +++++++++++ config/security/whitelist.go | 102 +++++++ config/security/whitelist_test.go | 47 +++ create/content.go | 14 +- deps/deps.go | 19 +- docs/config/_default/security.toml | 13 + docs/content/en/about/security-model/index.md | 21 +- .../en/getting-started/configuration.md | 4 + docs/data/docs.json | 37 ++- helpers/content.go | 4 +- helpers/content_test.go | 2 +- helpers/general_test.go | 2 +- helpers/testhelpers_test.go | 2 +- htesting/test_helpers.go | 2 +- hugolib/config.go | 9 + hugolib/js_test.go | 10 +- hugolib/page_test.go | 93 +++--- hugolib/resource_chain_babel_test.go | 14 +- hugolib/resource_chain_test.go | 8 +- hugolib/securitypolicies_test.go | 202 +++++++++++++ hugolib/shortcode.go | 4 +- hugolib/shortcode_test.go | 6 + hugolib/site.go | 42 ++- hugolib/testdata/cities.csv | 130 +++++++++ hugolib/testdata/fruits.json | 5 + hugolib/testhelpers_test.go | 12 + markup/asciidocext/convert.go | 36 ++- markup/asciidocext/convert_test.go | 54 ++-- markup/converter/converter.go | 2 + markup/internal/external.go | 54 ++-- markup/pandoc/convert.go | 36 ++- markup/pandoc/convert_test.go | 6 +- markup/rst/convert.go | 58 ++-- markup/rst/convert_test.go | 11 +- modules/client.go | 29 +- modules/client_test.go | 4 + resources/resource_factories/create/create.go | 16 + resources/resource_spec.go | 5 + .../resource_transformers/babel/babel.go | 46 +-- .../htesting/testhelpers.go | 2 +- .../resource_transformers/postcss/postcss.go | 47 ++- .../tocss/dartsass/client.go | 19 +- .../tocss/dartsass/transform.go | 13 +- resources/testhelpers_test.go | 4 +- tpl/collections/collections_test.go | 3 +- tpl/data/data.go | 7 + tpl/data/resources.go | 7 + tpl/data/resources_test.go | 7 +- tpl/os/os.go | 4 + tpl/transform/transform_test.go | 2 +- 58 files changed, 1713 insertions(+), 341 deletions(-) create mode 100644 common/hexec/exec.go delete mode 100644 common/hexec/safeCommand.go create mode 100644 config/security/docshelper.go create mode 100644 config/security/securityConfig.go create mode 100644 config/security/securityonfig_test.go create mode 100644 config/security/whitelist.go create mode 100644 config/security/whitelist_test.go create mode 100644 docs/config/_default/security.toml create mode 100644 hugolib/securitypolicies_test.go create mode 100644 hugolib/testdata/cities.csv create mode 100644 hugolib/testdata/fruits.json diff --git a/common/collections/slice.go b/common/collections/slice.go index 38ca86b08..07ad48eb3 100644 --- a/common/collections/slice.go +++ b/common/collections/slice.go @@ -64,3 +64,13 @@ func Slice(args ...interface{}) interface{} { } return slice.Interface() } + +// StringSliceToInterfaceSlice converts ss to []interface{}. +func StringSliceToInterfaceSlice(ss []string) []interface{} { + result := make([]interface{}, len(ss)) + for i, s := range ss { + result[i] = s + } + return result + +} diff --git a/common/herrors/errors.go b/common/herrors/errors.go index fded30b1a..00aed1eb6 100644 --- a/common/herrors/errors.go +++ b/common/herrors/errors.go @@ -88,3 +88,10 @@ func GetGID() uint64 { // We will, at least to begin with, make some Hugo features (SCSS with libsass) optional, // and this error is used to signal those situations. var ErrFeatureNotAvailable = errors.New("this feature is not available in your current Hugo version, see https://goo.gl/YMrWcn for more information") + +// Must panics if err != nil. +func Must(err error) { + if err != nil { + panic(err) + } +} diff --git a/common/hexec/exec.go b/common/hexec/exec.go new file mode 100644 index 000000000..a8bdd1bb7 --- /dev/null +++ b/common/hexec/exec.go @@ -0,0 +1,276 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hexec + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "os" + "os/exec" + + "github.com/cli/safeexec" + "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/security" +) + +var WithDir = func(dir string) func(c *commandeer) { + return func(c *commandeer) { + c.dir = dir + } +} + +var WithContext = func(ctx context.Context) func(c *commandeer) { + return func(c *commandeer) { + c.ctx = ctx + } +} + +var WithStdout = func(w io.Writer) func(c *commandeer) { + return func(c *commandeer) { + c.stdout = w + } +} + +var WithStderr = func(w io.Writer) func(c *commandeer) { + return func(c *commandeer) { + c.stderr = w + } +} + +var WithStdin = func(r io.Reader) func(c *commandeer) { + return func(c *commandeer) { + c.stdin = r + } +} + +var WithEnviron = func(env []string) func(c *commandeer) { + return func(c *commandeer) { + setOrAppend := func(s string) { + k1, _ := config.SplitEnvVar(s) + var found bool + for i, v := range c.env { + k2, _ := config.SplitEnvVar(v) + if k1 == k2 { + found = true + c.env[i] = s + } + } + + if !found { + c.env = append(c.env, s) + } + } + + for _, s := range env { + setOrAppend(s) + } + } +} + +// New creates a new Exec using the provided security config. +func New(cfg security.Config) *Exec { + var baseEnviron []string + for _, v := range os.Environ() { + k, _ := config.SplitEnvVar(v) + if cfg.Exec.OsEnv.Accept(k) { + baseEnviron = append(baseEnviron, v) + } + } + + return &Exec{ + sc: cfg, + baseEnviron: baseEnviron, + } +} + +// IsNotFound reports whether this is an error about a binary not found. +func IsNotFound(err error) bool { + var notFoundErr *NotFoundError + return errors.As(err, ¬FoundErr) +} + +// SafeCommand is a wrapper around os/exec Command which uses a LookPath +// implementation that does not search in current directory before looking in PATH. +// See https://github.com/cli/safeexec and the linked issues. +func SafeCommand(name string, arg ...string) (*exec.Cmd, error) { + bin, err := safeexec.LookPath(name) + if err != nil { + return nil, err + } + + return exec.Command(bin, arg...), nil +} + +// Exec encorces a security policy for commands run via os/exec. +type Exec struct { + sc security.Config + + // os.Environ filtered by the Exec.OsEnviron whitelist filter. + baseEnviron []string +} + +// New will fail if name is not allowed according to the configured security policy. +// Else a configured Runner will be returned ready to be Run. +func (e *Exec) New(name string, arg ...interface{}) (Runner, error) { + if err := e.sc.CheckAllowedExec(name); err != nil { + return nil, err + } + + env := make([]string, len(e.baseEnviron)) + copy(env, e.baseEnviron) + + cm := &commandeer{ + name: name, + env: env, + } + + return cm.command(arg...) + +} + +// Npx is a convenience method to create a Runner running npx --no-install }} + +Hugo has a built-in security policy that restricts access to [os/exec](https://pkg.go.dev/os/exec), remote communication and similar. + +The defdault configuration is listed below. And build using features not whitelisted in the security policy will faill with a detailed message about what needs to be done. Most of these settings are whitelists (string or slice, [Regular Expressions](https://pkg.go.dev/regexp) or `none` which matches nothing). + +{{< code-toggle config="security" />}} + +Note that these and other config settings in Hugo can be overridden by the OS environment. If you want to block all remote HTTP fetching of data: + +``` +HUGO_SECURITY_HTTP_URLS=none hugo +``` ## Dependency Security diff --git a/docs/content/en/getting-started/configuration.md b/docs/content/en/getting-started/configuration.md index 0f48c39e3..2123558d9 100644 --- a/docs/content/en/getting-started/configuration.md +++ b/docs/content/en/getting-started/configuration.md @@ -381,6 +381,10 @@ Maximum number of items in the RSS feed. ### sectionPagesMenu See ["Section Menu for Lazy Bloggers"](/templates/menu-templates/#section-menu-for-lazy-bloggers). +### security + +See [Security Policy](/about/security-model/#security-policy) + ### sitemap Default [sitemap configuration](/templates/sitemap-template/#configure-sitemapxml). diff --git a/docs/data/docs.json b/docs/data/docs.json index 70a2eafb4..8f8950dc4 100644 --- a/docs/data/docs.json +++ b/docs/data/docs.json @@ -1775,9 +1775,15 @@ "permalinks": { "_merge": "none" }, + "privacy": { + "_merge": "none" + }, "related": { "_merge": "none" }, + "security": { + "_merge": "none" + }, "sitemap": { "_merge": "none" }, @@ -1822,6 +1828,32 @@ "keepWhitespace": false } } + }, + "security": { + "enableInlineShortcodes": false, + "exec": { + "allow": [ + "^go$", + "^npx$", + "^postcss$" + ], + "osEnv": [ + "(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$" + ] + }, + "funcs": { + "getenv": [ + "^HUGO_" + ] + }, + "http": { + "methods": [ + "(?i)GET|POST" + ], + "urls": [ + ".*" + ] + } } }, "media": { @@ -1966,7 +1998,10 @@ "string": "image/jpeg", "suffixes": [ "jpg", - "jpeg" + "jpeg", + "jpe", + "jif", + "jfif" ] }, { diff --git a/helpers/content.go b/helpers/content.go index 161b14e76..2d26a0c48 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -24,6 +24,7 @@ import ( "unicode" "unicode/utf8" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/spf13/afero" @@ -64,7 +65,7 @@ type ContentSpec struct { // NewContentSpec returns a ContentSpec initialized // with the appropriate fields from the given config.Provider. -func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) { +func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs, ex *hexec.Exec) (*ContentSpec, error) { spec := &ContentSpec{ summaryLength: cfg.GetInt("summaryLength"), BuildFuture: cfg.GetBool("buildFuture"), @@ -78,6 +79,7 @@ func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero. Cfg: cfg, ContentFs: contentFs, Logger: logger, + Exec: ex, }) if err != nil { return nil, err diff --git a/helpers/content_test.go b/helpers/content_test.go index 515b788f1..c1ff5c1d2 100644 --- a/helpers/content_test.go +++ b/helpers/content_test.go @@ -110,7 +110,7 @@ func TestNewContentSpec(t *testing.T) { cfg.Set("buildExpired", true) cfg.Set("buildDrafts", true) - spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs()) + spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) c.Assert(err, qt.IsNil) c.Assert(spec.summaryLength, qt.Equals, 32) diff --git a/helpers/general_test.go b/helpers/general_test.go index bfabcbef4..db8cb30a8 100644 --- a/helpers/general_test.go +++ b/helpers/general_test.go @@ -30,7 +30,7 @@ import ( func TestResolveMarkup(t *testing.T) { c := qt.New(t) cfg := config.New() - spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs()) + spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) c.Assert(err, qt.IsNil) for i, this := range []struct { diff --git a/helpers/testhelpers_test.go b/helpers/testhelpers_test.go index 7d63e4d88..e9502acc0 100644 --- a/helpers/testhelpers_test.go +++ b/helpers/testhelpers_test.go @@ -50,7 +50,7 @@ func newTestCfg() config.Provider { func newTestContentSpec() *ContentSpec { v := config.New() - spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs()) + spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) if err != nil { panic(err) } diff --git a/htesting/test_helpers.go b/htesting/test_helpers.go index 20722f092..fa3f29c44 100644 --- a/htesting/test_helpers.go +++ b/htesting/test_helpers.go @@ -115,7 +115,7 @@ func IsGitHubAction() bool { // SupportsAll reports whether the running system supports all Hugo features, // e.g. Asciidoc, Pandoc etc. func SupportsAll() bool { - return IsGitHubAction() + return IsGitHubAction() || os.Getenv("CI_LOCAL") != "" } // GoMinorVersion returns the minor version of the current Go version, diff --git a/hugolib/config.go b/hugolib/config.go index 3b5ade598..e79899b94 100644 --- a/hugolib/config.go +++ b/hugolib/config.go @@ -18,6 +18,7 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/types" "github.com/gohugoio/hugo/common/maps" @@ -41,6 +42,7 @@ import ( "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/config/privacy" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/config/services" "github.com/gohugoio/hugo/helpers" "github.com/spf13/afero" @@ -377,6 +379,12 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide return nil, nil, err } + secConfig, err := security.DecodeConfig(v1) + if err != nil { + return nil, nil, err + } + ex := hexec.New(secConfig) + v1.Set("filecacheConfigs", filecacheConfigs) var configFilenames []string @@ -405,6 +413,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide modulesClient := modules.NewClient(modules.ClientConfig{ Fs: l.Fs, Logger: l.Logger, + Exec: ex, HookBeforeFinalize: hook, WorkingDir: workingDir, ThemesDir: themesDir, diff --git a/hugolib/js_test.go b/hugolib/js_test.go index 66c284d8b..69f528758 100644 --- a/hugolib/js_test.go +++ b/hugolib/js_test.go @@ -20,7 +20,6 @@ import ( "runtime" "testing" - "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/htesting" @@ -123,10 +122,9 @@ TS2: {{ template "print" $ts2 }} b.WithSourceFile("assets/js/included.js", includedJS) - cmd, err := hexec.SafeCommand("npm", "install") + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) - out, err := cmd.CombinedOutput() - b.Assert(err, qt.IsNil, qt.Commentf(string(out))) b.Build(BuildCfg{}) @@ -195,8 +193,8 @@ require github.com/gohugoio/hugoTestProjectJSModImports v0.9.0 // indirect }`) b.Assert(os.Chdir(workDir), qt.IsNil) - cmd, _ := hexec.SafeCommand("npm", "install") - _, err = cmd.CombinedOutput() + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) b.Build(BuildCfg{}) diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 7a1ff6c4e..50263d483 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -24,8 +24,6 @@ import ( "github.com/gohugoio/hugo/htesting" - "github.com/gohugoio/hugo/markup/rst" - "github.com/gohugoio/hugo/markup/asciidocext" "github.com/gohugoio/hugo/config" @@ -370,6 +368,7 @@ func normalizeExpected(ext, str string) string { func testAllMarkdownEnginesForPages(t *testing.T, assertFunc func(t *testing.T, ext string, pages page.Pages), settings map[string]interface{}, pageSources ...string) { + engines := []struct { ext string shouldExecute func() bool @@ -377,7 +376,7 @@ func testAllMarkdownEnginesForPages(t *testing.T, {"md", func() bool { return true }}, {"mmark", func() bool { return true }}, {"ad", func() bool { return asciidocext.Supports() }}, - {"rst", func() bool { return rst.Supports() }}, + {"rst", func() bool { return true }}, } for _, e := range engines { @@ -385,48 +384,58 @@ func testAllMarkdownEnginesForPages(t *testing.T, continue } - cfg, fs := newTestCfg(func(cfg config.Provider) error { - for k, v := range settings { - cfg.Set(k, v) + t.Run(e.ext, func(t *testing.T) { + + cfg, fs := newTestCfg(func(cfg config.Provider) error { + for k, v := range settings { + cfg.Set(k, v) + } + return nil + }) + + contentDir := "content" + + if s := cfg.GetString("contentDir"); s != "" { + contentDir = s } - return nil + + cfg.Set("security", map[string]interface{}{ + "exec": map[string]interface{}{ + "allow": []string{"^python$", "^rst2html.*", "^asciidoctor$"}, + }, + }) + + var fileSourcePairs []string + + for i, source := range pageSources { + fileSourcePairs = append(fileSourcePairs, fmt.Sprintf("p%d.%s", i, e.ext), source) + } + + for i := 0; i < len(fileSourcePairs); i += 2 { + writeSource(t, fs, filepath.Join(contentDir, fileSourcePairs[i]), fileSourcePairs[i+1]) + } + + // Add a content page for the home page + homePath := fmt.Sprintf("_index.%s", e.ext) + writeSource(t, fs, filepath.Join(contentDir, homePath), homePage) + + b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() + b.Build(BuildCfg{}) + + s := b.H.Sites[0] + + b.Assert(len(s.RegularPages()), qt.Equals, len(pageSources)) + + assertFunc(t, e.ext, s.RegularPages()) + + home, err := s.Info.Home() + b.Assert(err, qt.IsNil) + b.Assert(home, qt.Not(qt.IsNil)) + b.Assert(home.File().Path(), qt.Equals, homePath) + b.Assert(content(home), qt.Contains, "Home Page Content") + }) - contentDir := "content" - - if s := cfg.GetString("contentDir"); s != "" { - contentDir = s - } - - var fileSourcePairs []string - - for i, source := range pageSources { - fileSourcePairs = append(fileSourcePairs, fmt.Sprintf("p%d.%s", i, e.ext), source) - } - - for i := 0; i < len(fileSourcePairs); i += 2 { - writeSource(t, fs, filepath.Join(contentDir, fileSourcePairs[i]), fileSourcePairs[i+1]) - } - - // Add a content page for the home page - homePath := fmt.Sprintf("_index.%s", e.ext) - writeSource(t, fs, filepath.Join(contentDir, homePath), homePage) - - b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded() - b.Build(BuildCfg{SkipRender: true}) - - s := b.H.Sites[0] - - b.Assert(len(s.RegularPages()), qt.Equals, len(pageSources)) - - assertFunc(t, e.ext, s.RegularPages()) - - home, err := s.Info.Home() - b.Assert(err, qt.IsNil) - b.Assert(home, qt.Not(qt.IsNil)) - b.Assert(home.File().Path(), qt.Equals, homePath) - b.Assert(content(home), qt.Contains, "Home Page Content") - } } diff --git a/hugolib/resource_chain_babel_test.go b/hugolib/resource_chain_babel_test.go index 5cca22ba1..7a97e820a 100644 --- a/hugolib/resource_chain_babel_test.go +++ b/hugolib/resource_chain_babel_test.go @@ -21,8 +21,6 @@ import ( "github.com/gohugoio/hugo/config" - "github.com/gohugoio/hugo/common/hexec" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/htesting" @@ -51,7 +49,7 @@ func TestResourceChainBabel(t *testing.T) { "devDependencies": { "@babel/cli": "7.8.4", - "@babel/core": "7.9.0", + "@babel/core": "7.9.0", "@babel/preset-env": "7.9.5" } } @@ -94,6 +92,12 @@ class Car2 { v := config.New() v.Set("workingDir", workDir) v.Set("disableKinds", []string{"taxonomy", "term", "page"}) + v.Set("security", map[string]interface{}{ + "exec": map[string]interface{}{ + "allow": []string{"^npx$", "^babel$"}, + }, + }) + b := newTestSitesBuilder(t).WithLogger(logger) // Need to use OS fs for this. @@ -123,8 +127,8 @@ Transpiled3: {{ $transpiled.Permalink }} b.WithSourceFile("babel.config.js", babelConfig) b.Assert(os.Chdir(workDir), qt.IsNil) - cmd, _ := hexec.SafeCommand("npm", "install") - _, err = cmd.CombinedOutput() + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) b.Build(BuildCfg{}) diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 214dda216..0a5b9177c 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -32,8 +32,6 @@ import ( "testing" "time" - "github.com/gohugoio/hugo/common/hexec" - jww "github.com/spf13/jwalterweatherman" "github.com/gohugoio/hugo/common/herrors" @@ -387,8 +385,6 @@ T1: {{ $r.Content }} } func TestResourceChainBasic(t *testing.T) { - t.Parallel() - ts := httptest.NewServer(http.FileServer(http.Dir("testdata/"))) t.Cleanup(func() { ts.Close() @@ -1184,8 +1180,8 @@ class-in-b { b.WithSourceFile("postcss.config.js", postcssConfig) b.Assert(os.Chdir(workDir), qt.IsNil) - cmd, err := hexec.SafeCommand("npm", "install") - _, err = cmd.CombinedOutput() + cmd := b.NpmInstall() + err = cmd.Run() b.Assert(err, qt.IsNil) b.Build(BuildCfg{}) diff --git a/hugolib/securitypolicies_test.go b/hugolib/securitypolicies_test.go new file mode 100644 index 000000000..297f49479 --- /dev/null +++ b/hugolib/securitypolicies_test.go @@ -0,0 +1,202 @@ +// Copyright 2019 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "testing" + + qt "github.com/frankban/quicktest" + "github.com/gohugoio/hugo/markup/asciidocext" + "github.com/gohugoio/hugo/markup/pandoc" + "github.com/gohugoio/hugo/markup/rst" + "github.com/gohugoio/hugo/resources/resource_transformers/tocss/dartsass" +) + +func TestSecurityPolicies(t *testing.T) { + c := qt.New(t) + + testVariant := func(c *qt.C, withBuilder func(b *sitesBuilder), expectErr string) { + c.Helper() + b := newTestSitesBuilder(c) + withBuilder(b) + + if expectErr != "" { + err := b.BuildE(BuildCfg{}) + b.Assert(err, qt.IsNotNil) + b.Assert(err, qt.ErrorMatches, expectErr) + } else { + b.Build(BuildCfg{}) + } + + } + + httpTestVariant := func(c *qt.C, templ, expectErr string, withBuilder func(b *sitesBuilder)) { + ts := httptest.NewServer(http.FileServer(http.Dir("testdata/"))) + c.Cleanup(func() { + ts.Close() + }) + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", fmt.Sprintf(templ, ts.URL)) + if withBuilder != nil { + withBuilder(b) + } + } + testVariant(c, cb, expectErr) + } + + c.Run("os.GetEnv, denied", func(c *qt.C) { + c.Parallel() + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", `{{ os.Getenv "FOOBAR" }}`) + } + testVariant(c, cb, `(?s).*"FOOBAR" is not whitelisted in policy "security\.funcs\.getenv".*`) + }) + + c.Run("os.GetEnv, OK", func(c *qt.C) { + c.Parallel() + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", `{{ os.Getenv "HUGO_FOO" }}`) + } + testVariant(c, cb, "") + }) + + c.Run("Asciidoc, denied", func(c *qt.C) { + c.Parallel() + if !asciidocext.Supports() { + c.Skip() + } + + cb := func(b *sitesBuilder) { + b.WithContent("page.ad", "foo") + } + + testVariant(c, cb, `(?s).*"asciidoctor" is not whitelisted in policy "security\.exec\.allow".*`) + }) + + c.Run("RST, denied", func(c *qt.C) { + c.Parallel() + if !rst.Supports() { + c.Skip() + } + + cb := func(b *sitesBuilder) { + b.WithContent("page.rst", "foo") + } + + if runtime.GOOS == "windows" { + testVariant(c, cb, `(?s).*python(\.exe)?" is not whitelisted in policy "security\.exec\.allow".*`) + } else { + testVariant(c, cb, `(?s).*"rst2html(\.py)?" is not whitelisted in policy "security\.exec\.allow".*`) + + } + + }) + + c.Run("Pandoc, denied", func(c *qt.C) { + c.Parallel() + if !pandoc.Supports() { + c.Skip() + } + + cb := func(b *sitesBuilder) { + b.WithContent("page.pdc", "foo") + } + + testVariant(c, cb, `"(?s).*pandoc" is not whitelisted in policy "security\.exec\.allow".*`) + }) + + c.Run("Dart SASS, OK", func(c *qt.C) { + c.Parallel() + if !dartsass.Supports() { + c.Skip() + } + cb := func(b *sitesBuilder) { + b.WithTemplatesAdded("index.html", `{{ $scss := "body { color: #333; }" | resources.FromString "foo.scss" | resources.ToCSS (dict "transpiler" "dartsass") }}`) + } + testVariant(c, cb, "") + }) + + c.Run("Dart SASS, denied", func(c *qt.C) { + c.Parallel() + if !dartsass.Supports() { + c.Skip() + } + cb := func(b *sitesBuilder) { + b.WithConfigFile("toml", ` + [security] + [security.exec] + allow="none" + + `) + b.WithTemplatesAdded("index.html", `{{ $scss := "body { color: #333; }" | resources.FromString "foo.scss" | resources.ToCSS (dict "transpiler" "dartsass") }}`) + } + testVariant(c, cb, `(?s).*"dart-sass-embedded" is not whitelisted in policy "security\.exec\.allow".*`) + }) + + c.Run("resources.Get, OK", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil) + }) + + c.Run("resources.Get, denied method", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil) + }) + + c.Run("resources.Get, denied URL", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`, + func(b *sitesBuilder) { + b.WithConfigFile("toml", ` +[security] +[security.http] +urls="none" +`) + }) + }) + + c.Run("getJSON, OK", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := getJSON "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil) + }) + + c.Run("getJSON, denied URL", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $json := getJSON "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`, + func(b *sitesBuilder) { + b.WithConfigFile("toml", ` +[security] +[security.http] +urls="none" +`) + }) + }) + + c.Run("getCSV, denied URL", func(c *qt.C) { + c.Parallel() + httpTestVariant(c, `{{ $d := getCSV ";" "%[1]s/cities.csv" }}{{ $d.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`, + func(b *sitesBuilder) { + b.WithConfigFile("toml", ` +[security] +[security.http] +urls="none" +`) + }) + }) + +} diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 21d65de32..ec3a4a01b 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -257,7 +257,7 @@ func newShortcodeHandler(p *pageState, s *Site, placeholderFunc func() string) * sh := &shortcodeHandler{ p: p, s: s, - enableInlineShortcodes: s.enableInlineShortcodes, + enableInlineShortcodes: s.ExecHelper.Sec().EnableInlineShortcodes, shortcodes: make([]*shortcode, 0, 4), nameSet: make(map[string]bool), } @@ -287,7 +287,7 @@ func renderShortcode( var hasVariants bool if sc.isInline { - if !p.s.enableInlineShortcodes { + if !p.s.ExecHelper.Sec().EnableInlineShortcodes { return "", false, nil } templName := path.Join("_inline_shortcode", p.File().Path(), sc.name) diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 6ef110c9b..6316afc98 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -619,6 +619,12 @@ title: "Foo" cfg.Set("uglyURLs", false) cfg.Set("verbose", true) + cfg.Set("security", map[string]interface{}{ + "exec": map[string]interface{}{ + "allow": []string{"^python$", "^rst2html.*", "^asciidoctor$"}, + }, + }) + cfg.Set("markup.highlight.noClasses", false) cfg.Set("markup.highlight.codeFences", true) cfg.Set("markup", map[string]interface{}{ diff --git a/hugolib/site.go b/hugolib/site.go index 96cf0b93c..dce4b8d25 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -120,8 +120,6 @@ type Site struct { disabledKinds map[string]bool - enableInlineShortcodes bool - // Output formats defined in site config per Page Kind, or some defaults // if not set. // Output formats defined in Page front matter will override these. @@ -378,25 +376,24 @@ func (s *Site) isEnabled(kind string) bool { // reset returns a new Site prepared for rebuild. func (s *Site) reset() *Site { return &Site{ - Deps: s.Deps, - disabledKinds: s.disabledKinds, - titleFunc: s.titleFunc, - relatedDocsHandler: s.relatedDocsHandler.Clone(), - siteRefLinker: s.siteRefLinker, - outputFormats: s.outputFormats, - rc: s.rc, - outputFormatsConfig: s.outputFormatsConfig, - frontmatterHandler: s.frontmatterHandler, - mediaTypesConfig: s.mediaTypesConfig, - language: s.language, - siteBucket: s.siteBucket, - h: s.h, - publisher: s.publisher, - siteConfigConfig: s.siteConfigConfig, - enableInlineShortcodes: s.enableInlineShortcodes, - init: s.init, - PageCollections: s.PageCollections, - siteCfg: s.siteCfg, + Deps: s.Deps, + disabledKinds: s.disabledKinds, + titleFunc: s.titleFunc, + relatedDocsHandler: s.relatedDocsHandler.Clone(), + siteRefLinker: s.siteRefLinker, + outputFormats: s.outputFormats, + rc: s.rc, + outputFormatsConfig: s.outputFormatsConfig, + frontmatterHandler: s.frontmatterHandler, + mediaTypesConfig: s.mediaTypesConfig, + language: s.language, + siteBucket: s.siteBucket, + h: s.h, + publisher: s.publisher, + siteConfigConfig: s.siteConfigConfig, + init: s.init, + PageCollections: s.PageCollections, + siteCfg: s.siteCfg, } } @@ -564,8 +561,7 @@ But this also means that your site configuration may not do what you expect. If outputFormatsConfig: siteOutputFormatsConfig, mediaTypesConfig: siteMediaTypesConfig, - enableInlineShortcodes: cfg.Language.GetBool("enableInlineShortcodes"), - siteCfg: siteConfig, + siteCfg: siteConfig, titleFunc: titleFunc, diff --git a/hugolib/testdata/cities.csv b/hugolib/testdata/cities.csv new file mode 100644 index 000000000..ee6b058b6 --- /dev/null +++ b/hugolib/testdata/cities.csv @@ -0,0 +1,130 @@ +"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State" + 41, 5, 59, "N", 80, 39, 0, "W", "Youngstown", OH + 42, 52, 48, "N", 97, 23, 23, "W", "Yankton", SD + 46, 35, 59, "N", 120, 30, 36, "W", "Yakima", WA + 42, 16, 12, "N", 71, 48, 0, "W", "Worcester", MA + 43, 37, 48, "N", 89, 46, 11, "W", "Wisconsin Dells", WI + 36, 5, 59, "N", 80, 15, 0, "W", "Winston-Salem", NC + 49, 52, 48, "N", 97, 9, 0, "W", "Winnipeg", MB + 39, 11, 23, "N", 78, 9, 36, "W", "Winchester", VA + 34, 14, 24, "N", 77, 55, 11, "W", "Wilmington", NC + 39, 45, 0, "N", 75, 33, 0, "W", "Wilmington", DE + 48, 9, 0, "N", 103, 37, 12, "W", "Williston", ND + 41, 15, 0, "N", 77, 0, 0, "W", "Williamsport", PA + 37, 40, 48, "N", 82, 16, 47, "W", "Williamson", WV + 33, 54, 0, "N", 98, 29, 23, "W", "Wichita Falls", TX + 37, 41, 23, "N", 97, 20, 23, "W", "Wichita", KS + 40, 4, 11, "N", 80, 43, 12, "W", "Wheeling", WV + 26, 43, 11, "N", 80, 3, 0, "W", "West Palm Beach", FL + 47, 25, 11, "N", 120, 19, 11, "W", "Wenatchee", WA + 41, 25, 11, "N", 122, 23, 23, "W", "Weed", CA + 31, 13, 11, "N", 82, 20, 59, "W", "Waycross", GA + 44, 57, 35, "N", 89, 38, 23, "W", "Wausau", WI + 42, 21, 36, "N", 87, 49, 48, "W", "Waukegan", IL + 44, 54, 0, "N", 97, 6, 36, "W", "Watertown", SD + 43, 58, 47, "N", 75, 55, 11, "W", "Watertown", NY + 42, 30, 0, "N", 92, 20, 23, "W", "Waterloo", IA + 41, 32, 59, "N", 73, 3, 0, "W", "Waterbury", CT + 38, 53, 23, "N", 77, 1, 47, "W", "Washington", DC + 41, 50, 59, "N", 79, 8, 23, "W", "Warren", PA + 46, 4, 11, "N", 118, 19, 48, "W", "Walla Walla", WA + 31, 32, 59, "N", 97, 8, 23, "W", "Waco", TX + 38, 40, 48, "N", 87, 31, 47, "W", "Vincennes", IN + 28, 48, 35, "N", 97, 0, 36, "W", "Victoria", TX + 32, 20, 59, "N", 90, 52, 47, "W", "Vicksburg", MS + 49, 16, 12, "N", 123, 7, 12, "W", "Vancouver", BC + 46, 55, 11, "N", 98, 0, 36, "W", "Valley City", ND + 30, 49, 47, "N", 83, 16, 47, "W", "Valdosta", GA + 43, 6, 36, "N", 75, 13, 48, "W", "Utica", NY + 39, 54, 0, "N", 79, 43, 48, "W", "Uniontown", PA + 32, 20, 59, "N", 95, 18, 0, "W", "Tyler", TX + 42, 33, 36, "N", 114, 28, 12, "W", "Twin Falls", ID + 33, 12, 35, "N", 87, 34, 11, "W", "Tuscaloosa", AL + 34, 15, 35, "N", 88, 42, 35, "W", "Tupelo", MS + 36, 9, 35, "N", 95, 54, 36, "W", "Tulsa", OK + 32, 13, 12, "N", 110, 58, 12, "W", "Tucson", AZ + 37, 10, 11, "N", 104, 30, 36, "W", "Trinidad", CO + 40, 13, 47, "N", 74, 46, 11, "W", "Trenton", NJ + 44, 45, 35, "N", 85, 37, 47, "W", "Traverse City", MI + 43, 39, 0, "N", 79, 22, 47, "W", "Toronto", ON + 39, 2, 59, "N", 95, 40, 11, "W", "Topeka", KS + 41, 39, 0, "N", 83, 32, 24, "W", "Toledo", OH + 33, 25, 48, "N", 94, 3, 0, "W", "Texarkana", TX + 39, 28, 12, "N", 87, 24, 36, "W", "Terre Haute", IN + 27, 57, 0, "N", 82, 26, 59, "W", "Tampa", FL + 30, 27, 0, "N", 84, 16, 47, "W", "Tallahassee", FL + 47, 14, 24, "N", 122, 25, 48, "W", "Tacoma", WA + 43, 2, 59, "N", 76, 9, 0, "W", "Syracuse", NY + 32, 35, 59, "N", 82, 20, 23, "W", "Swainsboro", GA + 33, 55, 11, "N", 80, 20, 59, "W", "Sumter", SC + 40, 59, 24, "N", 75, 11, 24, "W", "Stroudsburg", PA + 37, 57, 35, "N", 121, 17, 24, "W", "Stockton", CA + 44, 31, 12, "N", 89, 34, 11, "W", "Stevens Point", WI + 40, 21, 36, "N", 80, 37, 12, "W", "Steubenville", OH + 40, 37, 11, "N", 103, 13, 12, "W", "Sterling", CO + 38, 9, 0, "N", 79, 4, 11, "W", "Staunton", VA + 39, 55, 11, "N", 83, 48, 35, "W", "Springfield", OH + 37, 13, 12, "N", 93, 17, 24, "W", "Springfield", MO + 42, 5, 59, "N", 72, 35, 23, "W", "Springfield", MA + 39, 47, 59, "N", 89, 39, 0, "W", "Springfield", IL + 47, 40, 11, "N", 117, 24, 36, "W", "Spokane", WA + 41, 40, 48, "N", 86, 15, 0, "W", "South Bend", IN + 43, 32, 24, "N", 96, 43, 48, "W", "Sioux Falls", SD + 42, 29, 24, "N", 96, 23, 23, "W", "Sioux City", IA + 32, 30, 35, "N", 93, 45, 0, "W", "Shreveport", LA + 33, 38, 23, "N", 96, 36, 36, "W", "Sherman", TX + 44, 47, 59, "N", 106, 57, 35, "W", "Sheridan", WY + 35, 13, 47, "N", 96, 40, 48, "W", "Seminole", OK + 32, 25, 11, "N", 87, 1, 11, "W", "Selma", AL + 38, 42, 35, "N", 93, 13, 48, "W", "Sedalia", MO + 47, 35, 59, "N", 122, 19, 48, "W", "Seattle", WA + 41, 24, 35, "N", 75, 40, 11, "W", "Scranton", PA + 41, 52, 11, "N", 103, 39, 36, "W", "Scottsbluff", NB + 42, 49, 11, "N", 73, 56, 59, "W", "Schenectady", NY + 32, 4, 48, "N", 81, 5, 23, "W", "Savannah", GA + 46, 29, 24, "N", 84, 20, 59, "W", "Sault Sainte Marie", MI + 27, 20, 24, "N", 82, 31, 47, "W", "Sarasota", FL + 38, 26, 23, "N", 122, 43, 12, "W", "Santa Rosa", CA + 35, 40, 48, "N", 105, 56, 59, "W", "Santa Fe", NM + 34, 25, 11, "N", 119, 41, 59, "W", "Santa Barbara", CA + 33, 45, 35, "N", 117, 52, 12, "W", "Santa Ana", CA + 37, 20, 24, "N", 121, 52, 47, "W", "San Jose", CA + 37, 46, 47, "N", 122, 25, 11, "W", "San Francisco", CA + 41, 27, 0, "N", 82, 42, 35, "W", "Sandusky", OH + 32, 42, 35, "N", 117, 9, 0, "W", "San Diego", CA + 34, 6, 36, "N", 117, 18, 35, "W", "San Bernardino", CA + 29, 25, 12, "N", 98, 30, 0, "W", "San Antonio", TX + 31, 27, 35, "N", 100, 26, 24, "W", "San Angelo", TX + 40, 45, 35, "N", 111, 52, 47, "W", "Salt Lake City", UT + 38, 22, 11, "N", 75, 35, 59, "W", "Salisbury", MD + 36, 40, 11, "N", 121, 39, 0, "W", "Salinas", CA + 38, 50, 24, "N", 97, 36, 36, "W", "Salina", KS + 38, 31, 47, "N", 106, 0, 0, "W", "Salida", CO + 44, 56, 23, "N", 123, 1, 47, "W", "Salem", OR + 44, 57, 0, "N", 93, 5, 59, "W", "Saint Paul", MN + 38, 37, 11, "N", 90, 11, 24, "W", "Saint Louis", MO + 39, 46, 12, "N", 94, 50, 23, "W", "Saint Joseph", MO + 42, 5, 59, "N", 86, 28, 48, "W", "Saint Joseph", MI + 44, 25, 11, "N", 72, 1, 11, "W", "Saint Johnsbury", VT + 45, 34, 11, "N", 94, 10, 11, "W", "Saint Cloud", MN + 29, 53, 23, "N", 81, 19, 11, "W", "Saint Augustine", FL + 43, 25, 48, "N", 83, 56, 24, "W", "Saginaw", MI + 38, 35, 24, "N", 121, 29, 23, "W", "Sacramento", CA + 43, 36, 36, "N", 72, 58, 12, "W", "Rutland", VT + 33, 24, 0, "N", 104, 31, 47, "W", "Roswell", NM + 35, 56, 23, "N", 77, 48, 0, "W", "Rocky Mount", NC + 41, 35, 24, "N", 109, 13, 48, "W", "Rock Springs", WY + 42, 16, 12, "N", 89, 5, 59, "W", "Rockford", IL + 43, 9, 35, "N", 77, 36, 36, "W", "Rochester", NY + 44, 1, 12, "N", 92, 27, 35, "W", "Rochester", MN + 37, 16, 12, "N", 79, 56, 24, "W", "Roanoke", VA + 37, 32, 24, "N", 77, 26, 59, "W", "Richmond", VA + 39, 49, 48, "N", 84, 53, 23, "W", "Richmond", IN + 38, 46, 12, "N", 112, 5, 23, "W", "Richfield", UT + 45, 38, 23, "N", 89, 25, 11, "W", "Rhinelander", WI + 39, 31, 12, "N", 119, 48, 35, "W", "Reno", NV + 50, 25, 11, "N", 104, 39, 0, "W", "Regina", SA + 40, 10, 48, "N", 122, 14, 23, "W", "Red Bluff", CA + 40, 19, 48, "N", 75, 55, 48, "W", "Reading", PA + 41, 9, 35, "N", 81, 14, 23, "W", "Ravenna", OH + diff --git a/hugolib/testdata/fruits.json b/hugolib/testdata/fruits.json new file mode 100644 index 000000000..3bb802a16 --- /dev/null +++ b/hugolib/testdata/fruits.json @@ -0,0 +1,5 @@ +{ + "fruit": "Apple", + "size": "Large", + "color": "Red" +} diff --git a/hugolib/testhelpers_test.go b/hugolib/testhelpers_test.go index ba3965675..72e22ed1d 100644 --- a/hugolib/testhelpers_test.go +++ b/hugolib/testhelpers_test.go @@ -18,6 +18,7 @@ import ( "time" "unicode/utf8" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/output" @@ -30,6 +31,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" @@ -791,6 +793,16 @@ func (s *sitesBuilder) GetPageRel(p page.Page, ref string) page.Page { return p } +func (s *sitesBuilder) NpmInstall() hexec.Runner { + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("npm") + ex := hexec.New(sc) + command, err := ex.New("npm", "install") + s.Assert(err, qt.IsNil) + return command + +} + func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper { return testHelper{ Cfg: cfg, diff --git a/markup/asciidocext/convert.go b/markup/asciidocext/convert.go index ff843cb6e..4c83e0e95 100644 --- a/markup/asciidocext/convert.go +++ b/markup/asciidocext/convert.go @@ -21,10 +21,9 @@ import ( "path/filepath" "strings" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" - "github.com/cli/safeexec" - "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/asciidocext/asciidocext_config" "github.com/gohugoio/hugo/markup/converter" @@ -67,7 +66,11 @@ type asciidocConverter struct { } func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { - content, toc, err := a.extractTOC(a.getAsciidocContent(ctx.Src, a.ctx)) + b, err := a.getAsciidocContent(ctx.Src, a.ctx) + if err != nil { + return nil, err + } + content, toc, err := a.extractTOC(b) if err != nil { return nil, err } @@ -83,20 +86,19 @@ func (a *asciidocConverter) Supports(_ identity.Identity) bool { // getAsciidocContent calls asciidoctor as an external helper // to convert AsciiDoc content to HTML. -func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte { - path := getAsciidoctorExecPath() - if path == "" { +func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { + if !hasAsciiDoc() { a.cfg.Logger.Errorln("asciidoctor not found in $PATH: Please install.\n", " Leaving AsciiDoc content unrendered.") - return src + return src, nil } args := a.parseArgs(ctx) args = append(args, "-") - a.cfg.Logger.Infoln("Rendering", ctx.DocumentName, "with", path, "using asciidoctor args", args, "...") + a.cfg.Logger.Infoln("Rendering", ctx.DocumentName, " using asciidoctor args", args, "...") - return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args) + return internal.ExternallyRenderContent(a.cfg, ctx, src, asciiDocBinaryName, args) } func (a *asciidocConverter) parseArgs(ctx converter.DocumentContext) []string { @@ -195,12 +197,10 @@ func (a *asciidocConverter) appendArg(args []string, option, value, defaultValue return args } -func getAsciidoctorExecPath() string { - path, err := safeexec.LookPath("asciidoctor") - if err != nil { - return "" - } - return path +const asciiDocBinaryName = "asciidoctor" + +func hasAsciiDoc() bool { + return hexec.InPath(asciiDocBinaryName) } // extractTOC extracts the toc from the given src html. @@ -311,8 +311,12 @@ func nodeContent(node *html.Node) string { // Supports returns whether Asciidoctor is installed on this computer. func Supports() bool { + hasBin := hasAsciiDoc() if htesting.SupportsAll() { + if !hasBin { + panic("asciidoctor not installed") + } return true } - return getAsciidoctorExecPath() != "" + return hasBin } diff --git a/markup/asciidocext/convert_test.go b/markup/asciidocext/convert_test.go index acc525c3b..3a350c5ce 100644 --- a/markup/asciidocext/convert_test.go +++ b/markup/asciidocext/convert_test.go @@ -21,8 +21,10 @@ import ( "path/filepath" "testing" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/markup_config" "github.com/gohugoio/hugo/markup/tableofcontents" @@ -280,20 +282,28 @@ func TestAsciidoctorAttributes(t *testing.T) { c.Assert(args[4], qt.Equals, "--no-header-footer") } +func getProvider(c *qt.C, mconf markup_config.Config) converter.Provider { + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("asciidoctor") + + p, err := Provider.New( + converter.ProviderConfig{ + MarkupConfig: mconf, + Logger: loggers.NewErrorLogger(), + Exec: hexec.New(sc), + }, + ) + c.Assert(err, qt.IsNil) + return p +} + func TestConvert(t *testing.T) { if !Supports() { t.Skip("asciidoctor not installed") } c := qt.New(t) - mconf := markup_config.Default - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, markup_config.Default) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) @@ -308,14 +318,8 @@ func TestTableOfContents(t *testing.T) { t.Skip("asciidoctor not installed") } c := qt.New(t) - mconf := markup_config.Default - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, markup_config.Default) + conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: macro @@ -390,14 +394,7 @@ func TestTableOfContentsWithCode(t *testing.T) { t.Skip("asciidoctor not installed") } c := qt.New(t) - mconf := markup_config.Default - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, markup_config.Default) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: auto @@ -433,13 +430,8 @@ func TestTableOfContentsPreserveTOC(t *testing.T) { c := qt.New(t) mconf := markup_config.Default mconf.AsciidocExt.PreserveTOC = true - p, err := Provider.New( - converter.ProviderConfig{ - MarkupConfig: mconf, - Logger: loggers.NewErrorLogger(), - }, - ) - c.Assert(err, qt.IsNil) + p := getProvider(c, mconf) + conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: diff --git a/markup/converter/converter.go b/markup/converter/converter.go index 3fa3bea39..180208a7b 100644 --- a/markup/converter/converter.go +++ b/markup/converter/converter.go @@ -16,6 +16,7 @@ package converter import ( "bytes" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/identity" @@ -32,6 +33,7 @@ type ProviderConfig struct { Cfg config.Provider // Site config ContentFs afero.Fs Logger loggers.Logger + Exec *hexec.Exec Highlight func(code, lang, optsStr string) (string, error) } diff --git a/markup/internal/external.go b/markup/internal/external.go index 0937afa34..97cf5cc7d 100644 --- a/markup/internal/external.go +++ b/markup/internal/external.go @@ -2,42 +2,56 @@ package internal import ( "bytes" + "fmt" "strings" - "github.com/cli/safeexec" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" - "github.com/gohugoio/hugo/markup/converter" ) func ExternallyRenderContent( cfg converter.ProviderConfig, ctx converter.DocumentContext, - content []byte, path string, args []string) []byte { + content []byte, binaryName string, args []string) ([]byte, error) { logger := cfg.Logger - cmd, err := hexec.SafeCommand(path, args...) - if err != nil { - logger.Errorf("%s rendering %s: %v", path, ctx.DocumentName, err) - return nil + + if strings.Contains(binaryName, "/") { + panic(fmt.Sprintf("should be no slash in %q", binaryName)) } - cmd.Stdin = bytes.NewReader(content) + + argsv := collections.StringSliceToInterfaceSlice(args) + var out, cmderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &cmderr + argsv = append(argsv, hexec.WithStdout(&out)) + argsv = append(argsv, hexec.WithStderr(&cmderr)) + argsv = append(argsv, hexec.WithStdin(bytes.NewReader(content))) + + cmd, err := cfg.Exec.New(binaryName, argsv...) + if err != nil { + return nil, err + } + err = cmd.Run() + // Most external helpers exit w/ non-zero exit code only if severe, i.e. // halting errors occurred. -> log stderr output regardless of state of err for _, item := range strings.Split(cmderr.String(), "\n") { item := strings.TrimSpace(item) if item != "" { - logger.Errorf("%s: %s", ctx.DocumentName, item) + if err == nil { + logger.Warnf("%s: %s", ctx.DocumentName, item) + } else { + logger.Errorf("%s: %s", ctx.DocumentName, item) + } } } + if err != nil { - logger.Errorf("%s rendering %s: %v", path, ctx.DocumentName, err) + logger.Errorf("%s rendering %s: %v", binaryName, ctx.DocumentName, err) } - return normalizeExternalHelperLineFeeds(out.Bytes()) + return normalizeExternalHelperLineFeeds(out.Bytes()), nil } // Strips carriage returns from third-party / external processes (useful for Windows) @@ -45,13 +59,13 @@ func normalizeExternalHelperLineFeeds(content []byte) []byte { return bytes.Replace(content, []byte("\r"), []byte(""), -1) } -func GetPythonExecPath() string { - path, err := safeexec.LookPath("python") - if err != nil { - path, err = safeexec.LookPath("python.exe") - if err != nil { - return "" +var pythonBinaryCandidates = []string{"python", "python.exe"} + +func GetPythonBinaryAndExecPath() (string, string) { + for _, p := range pythonBinaryCandidates { + if pth := hexec.LookPath(p); pth != "" { + return p, pth } } - return path + return "", "" } diff --git a/markup/pandoc/convert.go b/markup/pandoc/convert.go index 1c25e41d2..ae90cf417 100644 --- a/markup/pandoc/convert.go +++ b/markup/pandoc/convert.go @@ -15,7 +15,7 @@ package pandoc import ( - "github.com/cli/safeexec" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/markup/internal" @@ -44,7 +44,11 @@ type pandocConverter struct { } func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { - return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil + b, err := c.getPandocContent(ctx.Src, c.ctx) + if err != nil { + return nil, err + } + return converter.Bytes(b), nil } func (c *pandocConverter) Supports(feature identity.Identity) bool { @@ -52,31 +56,35 @@ func (c *pandocConverter) Supports(feature identity.Identity) bool { } // getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML. -func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte { +func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { logger := c.cfg.Logger - path := getPandocExecPath() - if path == "" { + binaryName := getPandocBinaryName() + if binaryName == "" { logger.Println("pandoc not found in $PATH: Please install.\n", " Leaving pandoc content unrendered.") - return src + return src, nil } args := []string{"--mathjax"} - return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args) + return internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args) } -func getPandocExecPath() string { - path, err := safeexec.LookPath("pandoc") - if err != nil { - return "" - } +const pandocBinary = "pandoc" - return path +func getPandocBinaryName() string { + if hexec.InPath(pandocBinary) { + return pandocBinary + } + return "" } // Supports returns whether Pandoc is installed on this computer. func Supports() bool { + hasBin := getPandocBinaryName() != "" if htesting.SupportsAll() { + if !hasBin { + panic("pandoc not installed") + } return true } - return getPandocExecPath() != "" + return hasBin } diff --git a/markup/pandoc/convert_test.go b/markup/pandoc/convert_test.go index bd6ca19e6..f549d5f4f 100644 --- a/markup/pandoc/convert_test.go +++ b/markup/pandoc/convert_test.go @@ -16,7 +16,9 @@ package pandoc import ( "testing" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" @@ -28,7 +30,9 @@ func TestConvert(t *testing.T) { t.Skip("pandoc not installed") } c := qt.New(t) - p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("pandoc") + p, err := Provider.New(converter.ProviderConfig{Exec: hexec.New(sc), Logger: loggers.NewErrorLogger()}) c.Assert(err, qt.IsNil) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) diff --git a/markup/rst/convert.go b/markup/rst/convert.go index 4c11c4be8..b86b35f1b 100644 --- a/markup/rst/convert.go +++ b/markup/rst/convert.go @@ -18,7 +18,7 @@ import ( "bytes" "runtime" - "github.com/cli/safeexec" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/identity" @@ -48,7 +48,11 @@ type rstConverter struct { } func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) { - return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil + b, err := c.getRstContent(ctx.Src, c.ctx) + if err != nil { + return nil, err + } + return converter.Bytes(b), nil } func (c *rstConverter) Supports(feature identity.Identity) bool { @@ -57,31 +61,38 @@ func (c *rstConverter) Supports(feature identity.Identity) bool { // getRstContent calls the Python script rst2html as an external helper // to convert reStructuredText content to HTML. -func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte { +func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) ([]byte, error) { logger := c.cfg.Logger - path := getRstExecPath() + binaryName, binaryPath := getRstBinaryNameAndPath() - if path == "" { + if binaryName == "" { logger.Println("rst2html / rst2html.py not found in $PATH: Please install.\n", " Leaving reStructuredText content unrendered.") - return src + return src, nil } - logger.Infoln("Rendering", ctx.DocumentName, "with", path, "...") + logger.Infoln("Rendering", ctx.DocumentName, "with", binaryName, "...") var result []byte + var err error + // certain *nix based OSs wrap executables in scripted launchers // invoking binaries on these OSs via python interpreter causes SyntaxError // invoke directly so that shebangs work as expected // handle Windows manually because it doesn't do shebangs if runtime.GOOS == "windows" { - python := internal.GetPythonExecPath() - args := []string{path, "--leave-comments", "--initial-header-level=2"} - result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args) + pythonBinary, _ := internal.GetPythonBinaryAndExecPath() + args := []string{binaryPath, "--leave-comments", "--initial-header-level=2"} + result, err = internal.ExternallyRenderContent(c.cfg, ctx, src, pythonBinary, args) } else { args := []string{"--leave-comments", "--initial-header-level=2"} - result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args) + result, err = internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args) } + + if err != nil { + return nil, err + } + // TODO(bep) check if rst2html has a body only option. bodyStart := bytes.Index(result, []byte("\n")) if bodyStart < 0 { @@ -96,24 +107,29 @@ func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) } } - return result[bodyStart+7 : bodyEnd] + return result[bodyStart+7 : bodyEnd], err } -func getRstExecPath() string { - path, err := safeexec.LookPath("rst2html") - if err != nil { - path, err = safeexec.LookPath("rst2html.py") - if err != nil { - return "" +var rst2Binaries = []string{"rst2html", "rst2html.py"} + +func getRstBinaryNameAndPath() (string, string) { + for _, candidate := range rst2Binaries { + if pth := hexec.LookPath(candidate); pth != "" { + return candidate, pth } } - return path + return "", "" } -// Supports returns whether rst is installed on this computer. +// Supports returns whether rst is (or should be) installed on this computer. func Supports() bool { + name, _ := getRstBinaryNameAndPath() + hasBin := name != "" if htesting.SupportsAll() { + if !hasBin { + panic("rst not installed") + } return true } - return getRstExecPath() != "" + return hasBin } diff --git a/markup/rst/convert_test.go b/markup/rst/convert_test.go index 269d92caa..5d2882de1 100644 --- a/markup/rst/convert_test.go +++ b/markup/rst/convert_test.go @@ -16,7 +16,9 @@ package rst import ( "testing" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/markup/converter" @@ -28,7 +30,14 @@ func TestConvert(t *testing.T) { t.Skip("rst not installed") } c := qt.New(t) - p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()}) + sc := security.DefaultConfig + sc.Exec.Allow = security.NewWhitelist("rst", "python") + + p, err := Provider.New( + converter.ProviderConfig{ + Logger: loggers.NewErrorLogger(), + Exec: hexec.New(sc), + }) c.Assert(err, qt.IsNil) conv, err := p.New(converter.DocumentContext{}) c.Assert(err, qt.IsNil) diff --git a/modules/client.go b/modules/client.go index fcb5957c3..1924cd5b4 100644 --- a/modules/client.go +++ b/modules/client.go @@ -28,6 +28,7 @@ import ( "strings" "time" + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" hglob "github.com/gohugoio/hugo/hugofs/glob" @@ -79,7 +80,7 @@ func NewClient(cfg ClientConfig) *Client { goModFilename = n } - env := os.Environ() + var env []string mcfg := cfg.ModuleConfig config.SetEnvVars(&env, @@ -87,12 +88,9 @@ func NewClient(cfg ClientConfig) *Client { "GO111MODULE", "on", "GOPROXY", mcfg.Proxy, "GOPRIVATE", mcfg.Private, - "GONOPROXY", mcfg.NoProxy) - - if cfg.CacheDir != "" { - // Module cache stored below $GOPATH/pkg - config.SetEnvVars(&env, "GOPATH", cfg.CacheDir) - } + "GONOPROXY", mcfg.NoProxy, + "GOPATH", cfg.CacheDir, + ) logger := cfg.Logger if logger == nil { @@ -609,16 +607,19 @@ func (c *Client) runGo( } stderr := new(bytes.Buffer) - cmd, err := hexec.SafeCommandContext(ctx, "go", args...) + + argsv := collections.StringSliceToInterfaceSlice(args) + argsv = append(argsv, hexec.WithEnviron(c.environ)) + argsv = append(argsv, hexec.WithStderr(io.MultiWriter(stderr, os.Stderr))) + argsv = append(argsv, hexec.WithStdout(stdout)) + argsv = append(argsv, hexec.WithDir(c.ccfg.WorkingDir)) + argsv = append(argsv, hexec.WithContext(ctx)) + + cmd, err := c.ccfg.Exec.New("go", argsv...) if err != nil { return err } - cmd.Env = c.environ - cmd.Dir = c.ccfg.WorkingDir - cmd.Stdout = stdout - cmd.Stderr = io.MultiWriter(stderr, os.Stderr) - if err := cmd.Run(); err != nil { if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound { c.goBinaryStatus = goBinaryStatusNotFound @@ -727,6 +728,8 @@ type ClientConfig struct { // Eg. "production" Environment string + Exec *hexec.Exec + CacheDir string // Module cache ModuleConfig Config } diff --git a/modules/client_test.go b/modules/client_test.go index f801af07d..75e3c2b08 100644 --- a/modules/client_test.go +++ b/modules/client_test.go @@ -21,6 +21,8 @@ import ( "sync/atomic" "testing" + "github.com/gohugoio/hugo/common/hexec" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/htesting" @@ -53,7 +55,9 @@ github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/h ccfg := ClientConfig{ Fs: hugofs.Os, WorkingDir: workingDir, + CacheDir: filepath.Join(workingDir, "modcache"), ThemesDir: themesDir, + Exec: hexec.New(security.DefaultConfig), } withConfig(&ccfg) diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 64e2f95a6..f7d0efe64 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -154,6 +154,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro // FromRemote expects one or n-parts of a URL to a resource // If you provide multiple parts they will be joined together to the final URL. func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) { + if err := c.validateFromRemoteArgs(uri, options); err != nil { + return nil, err + } rURL, err := url.Parse(uri) if err != nil { return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri) @@ -262,6 +265,19 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc } +func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error { + if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil { + return err + } + + if method, ok := options["method"].(string); ok { + if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil { + return err + } + } + return nil +} + func addDefaultHeaders(req *http.Request, accepts ...string) { for _, accept := range accepts { if !hasHeaderValue(req.Header, "Accept", accept) { diff --git a/resources/resource_spec.go b/resources/resource_spec.go index 156def363..897c1bbaa 100644 --- a/resources/resource_spec.go +++ b/resources/resource_spec.go @@ -26,6 +26,7 @@ import ( "github.com/gohugoio/hugo/resources/jsconfig" "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/identity" @@ -51,6 +52,7 @@ func NewSpec( incr identity.Incrementer, logger loggers.Logger, errorHandler herrors.ErrorSender, + execHelper *hexec.Exec, outputFormats output.Formats, mimeTypes media.Types) (*Spec, error) { imgConfig, err := images.DecodeConfig(s.Cfg.GetStringMap("imaging")) @@ -81,6 +83,7 @@ func NewSpec( Logger: logger, ErrorSender: errorHandler, imaging: imaging, + ExecHelper: execHelper, incr: incr, MediaTypes: mimeTypes, OutputFormats: outputFormats, @@ -120,6 +123,8 @@ type Spec struct { // Holds default filter settings etc. imaging *images.ImageProcessor + ExecHelper *hexec.Exec + incr identity.Incrementer imageCache *imageCache ResourceCache *ResourceCache diff --git a/resources/resource_transformers/babel/babel.go b/resources/resource_transformers/babel/babel.go index e291b210b..c20a131f6 100644 --- a/resources/resource_transformers/babel/babel.go +++ b/resources/resource_transformers/babel/babel.go @@ -23,7 +23,6 @@ import ( "regexp" "strconv" - "github.com/cli/safeexec" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" @@ -59,8 +58,8 @@ func DecodeOptions(m map[string]interface{}) (opts Options, err error) { return } -func (opts Options) toArgs() []string { - var args []string +func (opts Options) toArgs() []interface{} { + var args []interface{} // external is not a known constant on the babel command line // .sourceMaps must be a boolean, "inline", "both", or undefined @@ -115,21 +114,12 @@ func (t *babelTransformation) Key() internal.ResourceTransformationKey { // npm install -g @babel/preset-env // Instead of installing globally, you can also install everything as a dev-dependency (--save-dev instead of -g) func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - const localBabelPath = "node_modules/.bin/" const binaryName = "babel" - // Try first in the project's node_modules. - csiBinPath := filepath.Join(t.rs.WorkingDir, localBabelPath, binaryName) + ex := t.rs.ExecHelper - binary := csiBinPath - - if _, err := safeexec.LookPath(binary); err != nil { - // Try PATH - binary = binaryName - if _, err := safeexec.LookPath(binary); err != nil { - // This may be on a CI server etc. Will fall back to pre-built assets. - return herrors.ErrFeatureNotAvailable - } + if err := ex.Sec().CheckAllowedExec(binaryName); err != nil { + return err } var configFile string @@ -157,11 +147,11 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx ctx.ReplaceOutPathExtension(".js") - var cmdArgs []string + var cmdArgs []interface{} if configFile != "" { logger.Infoln("babel: use config file", configFile) - cmdArgs = []string{"--config-file", configFile} + cmdArgs = []interface{}{"--config-file", configFile} } if optArgs := t.options.toArgs(); len(optArgs) > 0 { @@ -178,18 +168,27 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx } cmdArgs = append(cmdArgs, "--out-file="+compileOutput.Name()) + stderr := io.MultiWriter(infoW, &errBuf) + cmdArgs = append(cmdArgs, hexec.WithStderr(stderr)) + cmdArgs = append(cmdArgs, hexec.WithStdout(stderr)) + cmdArgs = append(cmdArgs, hexec.WithEnviron(hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs))) + defer os.Remove(compileOutput.Name()) - cmd, err := hexec.SafeCommand(binary, cmdArgs...) + // ARGA [--no-install babel --config-file /private/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/hugo-test-babel812882892/babel.config.js --source-maps --filename=js/main2.js --out-file=/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/compileOut-2237820197.js] + // [--no-install babel --config-file /private/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/hugo-test-babel332846848/babel.config.js --filename=js/main.js --out-file=/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/compileOut-1451390834.js 0x10304ee60 0x10304ed60 0x10304f060] + cmd, err := ex.Npx(binaryName, cmdArgs...) + if err != nil { + if hexec.IsNotFound(err) { + // This may be on a CI server etc. Will fall back to pre-built assets. + return herrors.ErrFeatureNotAvailable + } return err } - cmd.Stderr = io.MultiWriter(infoW, &errBuf) - cmd.Stdout = cmd.Stderr - cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs) - stdin, err := cmd.StdinPipe() + if err != nil { return err } @@ -201,6 +200,9 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx err = cmd.Run() if err != nil { + if hexec.IsNotFound(err) { + return herrors.ErrFeatureNotAvailable + } return errors.Wrap(err, errBuf.String()) } diff --git a/resources/resource_transformers/htesting/testhelpers.go b/resources/resource_transformers/htesting/testhelpers.go index 21333eccb..674101f03 100644 --- a/resources/resource_transformers/htesting/testhelpers.go +++ b/resources/resource_transformers/htesting/testhelpers.go @@ -51,7 +51,7 @@ func NewTestResourceSpec() (*resources.Spec, error) { return nil, err } - spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) return spec, err } diff --git a/resources/resource_transformers/postcss/postcss.go b/resources/resource_transformers/postcss/postcss.go index 8104d0336..56cbea156 100644 --- a/resources/resource_transformers/postcss/postcss.go +++ b/resources/resource_transformers/postcss/postcss.go @@ -25,8 +25,7 @@ import ( "strconv" "strings" - "github.com/cli/safeexec" - + "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/hugo" @@ -142,22 +141,9 @@ func (t *postcssTransformation) Key() internal.ResourceTransformationKey { // npm install -g postcss-cli // npm install -g autoprefixer func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationCtx) error { - const localPostCSSPath = "node_modules/.bin/" const binaryName = "postcss" - // Try first in the project's node_modules. - csiBinPath := filepath.Join(t.rs.WorkingDir, localPostCSSPath, binaryName) - - binary := csiBinPath - - if _, err := safeexec.LookPath(binary); err != nil { - // Try PATH - binary = binaryName - if _, err := safeexec.LookPath(binary); err != nil { - // This may be on a CI server etc. Will fall back to pre-built assets. - return herrors.ErrFeatureNotAvailable - } - } + ex := t.rs.ExecHelper var configFile string logger := t.rs.Logger @@ -179,29 +165,33 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC } } - var cmdArgs []string + var cmdArgs []interface{} if configFile != "" { logger.Infoln("postcss: use config file", configFile) - cmdArgs = []string{"--config", configFile} + cmdArgs = []interface{}{"--config", configFile} } if optArgs := t.options.toArgs(); len(optArgs) > 0 { - cmdArgs = append(cmdArgs, optArgs...) - } - - cmd, err := hexec.SafeCommand(binary, cmdArgs...) - if err != nil { - return err + cmdArgs = append(cmdArgs, collections.StringSliceToInterfaceSlice(optArgs)...) } var errBuf bytes.Buffer infoW := loggers.LoggerToWriterWithPrefix(logger.Info(), "postcss") - cmd.Stdout = ctx.To - cmd.Stderr = io.MultiWriter(infoW, &errBuf) + stderr := io.MultiWriter(infoW, &errBuf) + cmdArgs = append(cmdArgs, hexec.WithStderr(stderr)) + cmdArgs = append(cmdArgs, hexec.WithStdout(ctx.To)) + cmdArgs = append(cmdArgs, hexec.WithEnviron(hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs))) - cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs) + cmd, err := ex.Npx(binaryName, cmdArgs...) + if err != nil { + if hexec.IsNotFound(err) { + // This may be on a CI server etc. Will fall back to pre-built assets. + return herrors.ErrFeatureNotAvailable + } + return err + } stdin, err := cmd.StdinPipe() if err != nil { @@ -231,6 +221,9 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC err = cmd.Run() if err != nil { + if hexec.IsNotFound(err) { + return herrors.ErrFeatureNotAvailable + } return imp.toFileError(errBuf.String()) } diff --git a/resources/resource_transformers/tocss/dartsass/client.go b/resources/resource_transformers/tocss/dartsass/client.go index 1d8250dc5..c2a572d9b 100644 --- a/resources/resource_transformers/tocss/dartsass/client.go +++ b/resources/resource_transformers/tocss/dartsass/client.go @@ -33,8 +33,13 @@ const transformationName = "tocss-dart" func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) { if !Supports() { - return &Client{dartSassNoAvailable: true}, nil + return &Client{dartSassNotAvailable: true}, nil } + + if err := rs.ExecHelper.Sec().CheckAllowedExec(dartSassEmbeddedBinaryName); err != nil { + return nil, err + } + transpiler, err := godartsass.Start(godartsass.Options{}) if err != nil { return nil, err @@ -43,15 +48,15 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) } type Client struct { - dartSassNoAvailable bool - rs *resources.Spec - sfs *filesystems.SourceFilesystem - workFs afero.Fs - transpiler *godartsass.Transpiler + dartSassNotAvailable bool + rs *resources.Spec + sfs *filesystems.SourceFilesystem + workFs afero.Fs + transpiler *godartsass.Transpiler } func (c *Client) ToCSS(res resources.ResourceTransformer, args map[string]interface{}) (resource.Resource, error) { - if c.dartSassNoAvailable { + if c.dartSassNotAvailable { return res.Transform(resources.NewFeatureNotAvailableTransformer(transformationName, args)) } return res.Transform(&transform{c: c, optsm: args}) diff --git a/resources/resource_transformers/tocss/dartsass/transform.go b/resources/resource_transformers/tocss/dartsass/transform.go index d70d2e6e0..57d9feadb 100644 --- a/resources/resource_transformers/tocss/dartsass/transform.go +++ b/resources/resource_transformers/tocss/dartsass/transform.go @@ -21,9 +21,8 @@ import ( "path/filepath" "strings" - "github.com/cli/safeexec" - "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/htesting" "github.com/gohugoio/hugo/media" @@ -38,16 +37,18 @@ import ( "github.com/bep/godartsass" ) -// See https://github.com/sass/dart-sass-embedded/issues/24 -const stdinPlaceholder = "HUGOSTDIN" +const ( + // See https://github.com/sass/dart-sass-embedded/issues/24 + stdinPlaceholder = "HUGOSTDIN" + dartSassEmbeddedBinaryName = "dart-sass-embedded" +) // Supports returns whether dart-sass-embedded is found in $PATH. func Supports() bool { if htesting.SupportsAll() { return true } - p, err := safeexec.LookPath("dart-sass-embedded") - return err == nil && p != "" + return hexec.InPath(dartSassEmbeddedBinaryName) } type transform struct { diff --git a/resources/testhelpers_test.go b/resources/testhelpers_test.go index 12dc8efe8..14d431644 100644 --- a/resources/testhelpers_test.go +++ b/resources/testhelpers_test.go @@ -87,7 +87,7 @@ func newTestResourceSpec(desc specDescriptor) *Spec { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec } @@ -126,7 +126,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) { filecaches, err := filecache.NewCaches(s) c.Assert(err, qt.IsNil) - spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) + spec, err := NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes) c.Assert(err, qt.IsNil) return spec, workDir diff --git a/tpl/collections/collections_test.go b/tpl/collections/collections_test.go index 3faf46930..8cced6fe5 100644 --- a/tpl/collections/collections_test.go +++ b/tpl/collections/collections_test.go @@ -32,7 +32,6 @@ import ( "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" - ) type tstNoStringer struct{} @@ -973,7 +972,7 @@ func ToTstXIs(slice interface{}) []TstXI { func newDeps(cfg config.Provider) *deps.Deps { l := langs.NewLanguage("en", cfg) l.Set("i18nDir", "i18n") - cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs()) + cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) if err != nil { panic(err) } diff --git a/tpl/data/data.go b/tpl/data/data.go index e993ed140..cfd847474 100644 --- a/tpl/data/data.go +++ b/tpl/data/data.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/common/types" @@ -88,6 +89,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err err = ns.getResource(cache, unmarshal, req) if err != nil { + if security.IsAccessDenied(err) { + return nil, err + } ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetCSV, "Failed to get CSV resource %q: %s", url, err) return nil, nil } @@ -121,6 +125,9 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) { err = ns.getResource(cache, unmarshal, req) if err != nil { + if security.IsAccessDenied(err) { + return nil, err + } ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetJSON, "Failed to get JSON resource %q: %s", url, err) return nil, nil } diff --git a/tpl/data/resources.go b/tpl/data/resources.go index b38b2784a..b4b310bcc 100644 --- a/tpl/data/resources.go +++ b/tpl/data/resources.go @@ -38,6 +38,13 @@ var ( // getRemote loads the content of a remote file. This method is thread safe. func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error { url := req.URL.String() + if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPURL(url); err != nil { + return err + } + if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPMethod("GET"); err != nil { + return err + } + var headers bytes.Buffer req.Header.Write(&headers) id := helpers.MD5String(url + headers.String()) diff --git a/tpl/data/resources_test.go b/tpl/data/resources_test.go index 8425bf87a..e825c2be1 100644 --- a/tpl/data/resources_test.go +++ b/tpl/data/resources_test.go @@ -22,12 +22,14 @@ import ( "testing" "time" + "github.com/gohugoio/hugo/config/security" "github.com/gohugoio/hugo/modules" "github.com/gohugoio/hugo/helpers" qt "github.com/frankban/quicktest" "github.com/gohugoio/hugo/cache/filecache" + "github.com/gohugoio/hugo/common/hexec" "github.com/gohugoio/hugo/common/loggers" "github.com/gohugoio/hugo/config" "github.com/gohugoio/hugo/deps" @@ -193,8 +195,10 @@ func newDeps(cfg config.Provider) *deps.Deps { } cfg.Set("allModules", modules.Modules{mod}) + ex := hexec.New(security.DefaultConfig) + logger := loggers.NewIgnorableLogger(loggers.NewErrorLogger(), "none") - cs, err := helpers.NewContentSpec(cfg, logger, afero.NewMemMapFs()) + cs, err := helpers.NewContentSpec(cfg, logger, afero.NewMemMapFs(), ex) if err != nil { panic(err) } @@ -215,6 +219,7 @@ func newDeps(cfg config.Provider) *deps.Deps { Cfg: cfg, Fs: fs, FileCaches: fileCaches, + ExecHelper: ex, ContentSpec: cs, Log: logger, LogDistinct: helpers.NewDistinctLogger(logger), diff --git a/tpl/os/os.go b/tpl/os/os.go index e729b810b..43c42f5e1 100644 --- a/tpl/os/os.go +++ b/tpl/os/os.go @@ -56,6 +56,10 @@ func (ns *Namespace) Getenv(key interface{}) (string, error) { return "", nil } + if err = ns.deps.ExecHelper.Sec().CheckAllowedGetEnv(skey); err != nil { + return "", err + } + return _os.Getenv(skey), nil } diff --git a/tpl/transform/transform_test.go b/tpl/transform/transform_test.go index 2b0c69d09..260de5f83 100644 --- a/tpl/transform/transform_test.go +++ b/tpl/transform/transform_test.go @@ -241,7 +241,7 @@ func newDeps(cfg config.Provider) *deps.Deps { l := langs.NewLanguage("en", cfg) - cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs()) + cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil) if err != nil { panic(err) }