github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-11-07 20:30:36 -05:00
Code Issues Projects Releases Packages Wiki Activity

Add some basic security policies with sensible defaults

Browse source 
This ommmit contains some security hardening measures for the Hugo build runtime.

There are some rarely used features in Hugo that would be good to have disabled by default. One example would be the "external helpers".

For `asciidoctor` and some others we use Go's `os/exec` package to start a new process.

These are a predefined set of binary names, all loaded from `PATH` and with a predefined set of arguments. Still, if you don't use `asciidoctor` in your project, you might as well have it turned off.

You can configure your own in the new `security` configuration section, but the defaults are configured to create a minimal amount of site breakage. And if that do happen, you will get clear instructions in the loa about what to do.

The default configuration is listed below. Note that almost all of these options are regular expression _whitelists_ (a string or a slice); the value `none` will block all.

```toml
[security]
  enableInlineShortcodes = false
  [security.exec]
    allow = ['^dart-sass-embedded$', '^go$', '^npx$', '^postcss$']
    osEnv = ['(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$']

  [security.funcs]
    getenv = ['^HUGO_']

  [security.http]
    methods = ['(?i)GET|POST']
    urls = ['.*']
```
This commit is contained in:
Bjørn Erik Pedersen 2021-12-12 12:11:11 +01:00
parent 803f572e66
commit f4389e48ce
No known key found for this signature in database
GPG key ID: 330E6E2BD4859D8F
58 changed files with 1713 additions and 341 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
common/collections/slice.go
View file

@ -64,3 +64,13 @@ func Slice(args ...interface{}) interface{} {
}
return slice.Interface()
}
// StringSliceToInterfaceSlice converts ss to []interface{}.
func StringSliceToInterfaceSlice(ss []string) []interface{} {
result := make([]interface{}, len(ss))
for i, s := range ss {
result[i] = s
}
return result
}

7
common/herrors/errors.go
View file

@ -88,3 +88,10 @@ func GetGID() uint64 {
// We will, at least to begin with, make some Hugo features (SCSS with libsass) optional,
// and this error is used to signal those situations.
var ErrFeatureNotAvailable = errors.New("this feature is not available in your current Hugo version, see https://goo.gl/YMrWcn for more information")
// Must panics if err != nil.
func Must(err error) {
if err != nil {
panic(err)
}
}

276
common/hexec/exec.go Normal file
View file

@ -0,0 +1,276 @@
// Copyright 2020 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hexec
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"regexp"
"strings"
"os"
"os/exec"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/security"
)
var WithDir = func(dir string) func(c *commandeer) {
return func(c *commandeer) {
c.dir = dir
}
}
var WithContext = func(ctx context.Context) func(c *commandeer) {
return func(c *commandeer) {
c.ctx = ctx
}
}
var WithStdout = func(w io.Writer) func(c *commandeer) {
return func(c *commandeer) {
c.stdout = w
}
}
var WithStderr = func(w io.Writer) func(c *commandeer) {
return func(c *commandeer) {
c.stderr = w
}
}
var WithStdin = func(r io.Reader) func(c *commandeer) {
return func(c *commandeer) {
c.stdin = r
}
}
var WithEnviron = func(env []string) func(c *commandeer) {
return func(c *commandeer) {
setOrAppend := func(s string) {
k1, _ := config.SplitEnvVar(s)
var found bool
for i, v := range c.env {
k2, _ := config.SplitEnvVar(v)
if k1 == k2 {
found = true
c.env[i] = s
}
}
if !found {
c.env = append(c.env, s)
}
}
for _, s := range env {
setOrAppend(s)
}
}
}
// New creates a new Exec using the provided security config.
func New(cfg security.Config) *Exec {
var baseEnviron []string
for _, v := range os.Environ() {
k, _ := config.SplitEnvVar(v)
if cfg.Exec.OsEnv.Accept(k) {
baseEnviron = append(baseEnviron, v)
}
}
return &Exec{
sc: cfg,
baseEnviron: baseEnviron,
}
}
// IsNotFound reports whether this is an error about a binary not found.
func IsNotFound(err error) bool {
var notFoundErr *NotFoundError
return errors.As(err, &notFoundErr)
}
// SafeCommand is a wrapper around os/exec Command which uses a LookPath
// implementation that does not search in current directory before looking in PATH.
// See https://github.com/cli/safeexec and the linked issues.
func SafeCommand(name string, arg ...string) (*exec.Cmd, error) {
bin, err := safeexec.LookPath(name)
if err != nil {
return nil, err
}
return exec.Command(bin, arg...), nil
}
// Exec encorces a security policy for commands run via os/exec.
type Exec struct {
sc security.Config
// os.Environ filtered by the Exec.OsEnviron whitelist filter.
baseEnviron []string
}
// New will fail if name is not allowed according to the configured security policy.
// Else a configured Runner will be returned ready to be Run.
func (e *Exec) New(name string, arg ...interface{}) (Runner, error) {
if err := e.sc.CheckAllowedExec(name); err != nil {
return nil, err
}
env := make([]string, len(e.baseEnviron))
copy(env, e.baseEnviron)
cm := &commandeer{
name: name,
env: env,
}
return cm.command(arg...)
}
// Npx is a convenience method to create a Runner running npx --no-install <name> <args.
func (e *Exec) Npx(name string, arg ...interface{}) (Runner, error) {
arg = append(arg[:0], append([]interface{}{"--no-install", name}, arg[0:]...)...)
return e.New("npx", arg...)
}
// Sec returns the security policies this Exec is configured with.
func (e *Exec) Sec() security.Config {
return e.sc
}
type NotFoundError struct {
name string
}
func (e *NotFoundError) Error() string {
return fmt.Sprintf("binary with name %q not found", e.name)
}
// Runner wraps a *os.Cmd.
type Runner interface {
Run() error
StdinPipe() (io.WriteCloser, error)
}
type cmdWrapper struct {
name string
c *exec.Cmd
outerr *bytes.Buffer
}
var notFoundRe = regexp.MustCompile(`(?s)not found:|could not determine executable`)
func (c *cmdWrapper) Run() error {
err := c.c.Run()
if err == nil {
return nil
}
if notFoundRe.MatchString(c.outerr.String()) {
return &NotFoundError{name: c.name}
}
return fmt.Errorf("failed to execute binary %q with args %v: %s", c.name, c.c.Args[1:], c.outerr.String())
}
func (c *cmdWrapper) StdinPipe() (io.WriteCloser, error) {
return c.c.StdinPipe()
}
type commandeer struct {
stdout io.Writer
stderr io.Writer
stdin io.Reader
dir string
ctx context.Context
name string
env []string
}
func (c *commandeer) command(arg ...interface{}) (*cmdWrapper, error) {
if c == nil {
return nil, nil
}
var args []string
for _, a := range arg {
switch v := a.(type) {
case string:
args = append(args, v)
case func(*commandeer):
v(c)
default:
return nil, fmt.Errorf("invalid argument to command: %T", a)
}
}
bin, err := safeexec.LookPath(c.name)
if err != nil {
return nil, &NotFoundError{
name: c.name,
}
}
outerr := &bytes.Buffer{}
if c.stderr == nil {
c.stderr = outerr
} else {
c.stderr = io.MultiWriter(c.stderr, outerr)
}
var cmd *exec.Cmd
if c.ctx != nil {
cmd = exec.CommandContext(c.ctx, bin, args...)
} else {
cmd = exec.Command(bin, args...)
}
cmd.Stdin = c.stdin
cmd.Stderr = c.stderr
cmd.Stdout = c.stdout
cmd.Env = c.env
cmd.Dir = c.dir
return &cmdWrapper{outerr: outerr, c: cmd, name: c.name}, nil
}
// InPath reports whether binaryName is in $PATH.
func InPath(binaryName string) bool {
if strings.Contains(binaryName, "/") {
panic("binary name should not contain any slash")
}
_, err := safeexec.LookPath(binaryName)
return err == nil
}
// LookPath finds the path to binaryName in $PATH.
// Returns "" if not found.
func LookPath(binaryName string) string {
if strings.Contains(binaryName, "/") {
panic("binary name should not contain any slash")
}
s, err := safeexec.LookPath(binaryName)
if err != nil {
return ""
}
return s
}

45
common/hexec/safeCommand.go
View file

@ -1,45 +0,0 @@
// Copyright 2020 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hexec
import (
"context"
"os/exec"
"github.com/cli/safeexec"
)
// SafeCommand is a wrapper around os/exec Command which uses a LookPath
// implementation that does not search in current directory before looking in PATH.
// See https://github.com/cli/safeexec and the linked issues.
func SafeCommand(name string, arg ...string) (*exec.Cmd, error) {
bin, err := safeexec.LookPath(name)
if err != nil {
return nil, err
}
return exec.Command(bin, arg...), nil
}
// SafeCommandContext wraps CommandContext
// See SafeCommand for more context.
func SafeCommandContext(ctx context.Context, name string, arg ...string) (*exec.Cmd, error) {
bin, err := safeexec.LookPath(name)
if err != nil {
return nil, err
}
return exec.CommandContext(ctx, bin, arg...), nil
}

19
common/hugo/hugo.go
View file

@ -89,8 +89,10 @@ func NewInfo(environment string) Info {
}
}
// GetExecEnviron creates and gets the common os/exec environment used in the
// external programs we interact with via os/exec, e.g. postcss.
func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string {
env := os.Environ()
var env []string
nodepath := filepath.Join(workDir, "node_modules")
if np := os.Getenv("NODE_PATH"); np != "" {
nodepath = workDir + string(os.PathListSeparator) + np
@ -98,12 +100,15 @@ func GetExecEnviron(workDir string, cfg config.Provider, fs afero.Fs) []string {
config.SetEnvVars(&env, "NODE_PATH", nodepath)
config.SetEnvVars(&env, "PWD", workDir)
config.SetEnvVars(&env, "HUGO_ENVIRONMENT", cfg.GetString("environment"))
fis, err := afero.ReadDir(fs, files.FolderJSConfig)
if err == nil {
for _, fi := range fis {
key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_"))
value := fi.(hugofs.FileMetaInfo).Meta().Filename
config.SetEnvVars(&env, key, value)
if fs != nil {
fis, err := afero.ReadDir(fs, files.FolderJSConfig)
if err == nil {
for _, fi := range fis {
key := fmt.Sprintf("HUGO_FILE_%s", strings.ReplaceAll(strings.ToUpper(fi.Name()), ".", "_"))
value := fi.(hugofs.FileMetaInfo).Meta().Filename
config.SetEnvVars(&env, key, value)
}
}
}

2
config/defaultConfigProvider.go
View file

@ -44,6 +44,8 @@ var (
"permalinks": true,
"related": true,
"sitemap": true,
"privacy": true,
"security": true,
"taxonomies": true,
}

26
config/security/docshelper.go Normal file
View file

@ -0,0 +1,26 @@
// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package security
import (
"github.com/gohugoio/hugo/docshelper"
)
func init() {
docsProvider := func() docshelper.DocProvider {
return docshelper.DocProvider{"config": DefaultConfig.ToSecurityMap()}
}
docshelper.AddDocProviderFunc(docsProvider)
}

227
config/security/securityConfig.go Normal file
View file

@ -0,0 +1,227 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package security
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/parser"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/mitchellh/mapstructure"
)
const securityConfigKey = "security"
// DefaultConfig holds the default security policy.
var DefaultConfig = Config{
Exec: Exec{
Allow: NewWhitelist(
"^dart-sass-embedded$",
"^go$", // for Go Modules
"^npx$", // used by all Node tools (Babel, PostCSS).
"^postcss$",
),
// These have been tested to work with Hugo's external programs
// on Windows, Linux and MacOS.
OsEnv: NewWhitelist("(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$"),
},
Funcs: Funcs{
Getenv: NewWhitelist("^HUGO_"),
},
HTTP: HTTP{
URLs: NewWhitelist(".*"),
Methods: NewWhitelist("(?i)GET|POST"),
},
}
// Config is the top level security config.
type Config struct {
// Restricts access to os.Exec.
Exec Exec `json:"exec"`
// Restricts access to certain template funcs.
Funcs Funcs `json:"funcs"`
// Restricts access to resources.Get, getJSON, getCSV.
HTTP HTTP `json:"http"`
// Allow inline shortcodes
EnableInlineShortcodes bool `json:"enableInlineShortcodes"`
}
// Exec holds os/exec policies.
type Exec struct {
Allow Whitelist `json:"allow"`
OsEnv Whitelist `json:"osEnv"`
}
// Funcs holds template funcs policies.
type Funcs struct {
// OS env keys allowed to query in os.Getenv.
Getenv Whitelist `json:"getenv"`
}
type HTTP struct {
// URLs to allow in remote HTTP (resources.Get, getJSON, getCSV).
URLs Whitelist `json:"urls"`
// HTTP methods to allow.
Methods Whitelist `json:"methods"`
}
// ToTOML converts c to TOML with [security] as the root.
func (c Config) ToTOML() string {
sec := c.ToSecurityMap()
var b bytes.Buffer
if err := parser.InterfaceToConfig(sec, metadecoders.TOML, &b); err != nil {
panic(err)
}
return strings.TrimSpace(b.String())
}
func (c Config) CheckAllowedExec(name string) error {
if !c.Exec.Allow.Accept(name) {
return &AccessDeniedError{
name: name,
path: "security.exec.allow",
policies: c.ToTOML(),
}
}
return nil
}
func (c Config) CheckAllowedGetEnv(name string) error {
if !c.Funcs.Getenv.Accept(name) {
return &AccessDeniedError{
name: name,
path: "security.funcs.getenv",
policies: c.ToTOML(),
}
}
return nil
}
func (c Config) CheckAllowedHTTPURL(url string) error {
if !c.HTTP.URLs.Accept(url) {
return &AccessDeniedError{
name: url,
path: "security.http.urls",
policies: c.ToTOML(),
}
}
return nil
}
func (c Config) CheckAllowedHTTPMethod(method string) error {
if !c.HTTP.Methods.Accept(method) {
return &AccessDeniedError{
name: method,
path: "security.http.method",
policies: c.ToTOML(),
}
}
return nil
}
// ToSecurityMap converts c to a map with 'security' as the root key.
func (c Config) ToSecurityMap() map[string]interface{} {
// Take it to JSON and back to get proper casing etc.
asJson, err := json.Marshal(c)
herrors.Must(err)
m := make(map[string]interface{})
herrors.Must(json.Unmarshal(asJson, &m))
// Add the root
sec := map[string]interface{}{
"security": m,
}
return sec
}
// DecodeConfig creates a privacy Config from a given Hugo configuration.
func DecodeConfig(cfg config.Provider) (Config, error) {
sc := DefaultConfig
if cfg.IsSet(securityConfigKey) {
m := cfg.GetStringMap(securityConfigKey)
dec, err := mapstructure.NewDecoder(
&mapstructure.DecoderConfig{
WeaklyTypedInput: true,
Result: &sc,
DecodeHook: stringSliceToWhitelistHook(),
},
)
if err != nil {
return sc, err
}
if err = dec.Decode(m); err != nil {
return sc, err
}
}
if !sc.EnableInlineShortcodes {
// Legacy
sc.EnableInlineShortcodes = cfg.GetBool("enableInlineShortcodes")
}
return sc, nil
}
func stringSliceToWhitelistHook() mapstructure.DecodeHookFuncType {
return func(
f reflect.Type,
t reflect.Type,
data interface{}) (interface{}, error) {
if t != reflect.TypeOf(Whitelist{}) {
return data, nil
}
wl := types.ToStringSlicePreserveString(data)
return NewWhitelist(wl...), nil
}
}
// AccessDeniedError represents a security policy conflict.
type AccessDeniedError struct {
path string
name string
policies string
}
func (e *AccessDeniedError) Error() string {
return fmt.Sprintf("access denied: %q is not whitelisted in policy %q; the current security configuration is:\n\n%s\n\n", e.name, e.path, e.policies)
}
// IsAccessDenied reports whether err is an AccessDeniedError
func IsAccessDenied(err error) bool {
var notFoundErr *AccessDeniedError
return errors.As(err, &notFoundErr)
}

166
config/security/securityonfig_test.go Normal file
View file

@ -0,0 +1,166 @@
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package security
import (
"testing"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/config"
)
func TestDecodeConfigFromTOML(t *testing.T) {
c := qt.New(t)
c.Run("Slice whitelist", func(c *qt.C) {
c.Parallel()
tomlConfig := `
someOtherValue = "bar"
[security]
enableInlineShortcodes=true
[security.exec]
allow=["a", "b"]
osEnv=["a", "b", "c"]
[security.funcs]
getEnv=["a", "b"]
`
cfg, err := config.FromConfigString(tomlConfig, "toml")
c.Assert(err, qt.IsNil)
pc, err := DecodeConfig(cfg)
c.Assert(err, qt.IsNil)
c.Assert(pc, qt.Not(qt.IsNil))
c.Assert(pc.EnableInlineShortcodes, qt.IsTrue)
c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue)
c.Assert(pc.Exec.Allow.Accept("d"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("a"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
c.Assert(pc.Funcs.Getenv.Accept("a"), qt.IsTrue)
c.Assert(pc.Funcs.Getenv.Accept("c"), qt.IsFalse)
})
c.Run("String whitelist", func(c *qt.C) {
c.Parallel()
tomlConfig := `
someOtherValue = "bar"
[security]
[security.exec]
allow="a"
osEnv="b"
`
cfg, err := config.FromConfigString(tomlConfig, "toml")
c.Assert(err, qt.IsNil)
pc, err := DecodeConfig(cfg)
c.Assert(err, qt.IsNil)
c.Assert(pc, qt.Not(qt.IsNil))
c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue)
c.Assert(pc.Exec.Allow.Accept("d"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("b"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
})
c.Run("Default exec.osEnv", func(c *qt.C) {
c.Parallel()
tomlConfig := `
someOtherValue = "bar"
[security]
[security.exec]
allow="a"
`
cfg, err := config.FromConfigString(tomlConfig, "toml")
c.Assert(err, qt.IsNil)
pc, err := DecodeConfig(cfg)
c.Assert(err, qt.IsNil)
c.Assert(pc, qt.Not(qt.IsNil))
c.Assert(pc.Exec.Allow.Accept("a"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("PATH"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
})
c.Run("Enable inline shortcodes, legacy", func(c *qt.C) {
c.Parallel()
tomlConfig := `
someOtherValue = "bar"
enableInlineShortcodes=true
[security]
[security.exec]
allow="a"
osEnv="b"
`
cfg, err := config.FromConfigString(tomlConfig, "toml")
c.Assert(err, qt.IsNil)
pc, err := DecodeConfig(cfg)
c.Assert(err, qt.IsNil)
c.Assert(pc.EnableInlineShortcodes, qt.IsTrue)
})
}
func TestToTOML(t *testing.T) {
c := qt.New(t)
got := DefaultConfig.ToTOML()
c.Assert(got, qt.Equals,
"[security]\n enableInlineShortcodes = false\n [security.exec]\n allow = ['^dart-sass-embedded$', '^go$', '^npx$', '^postcss$']\n osEnv = ['(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$']\n\n [security.funcs]\n getenv = ['^HUGO_']\n\n [security.http]\n methods = ['(?i)GET|POST']\n urls = ['.*']",
)
}
func TestDecodeConfigDefault(t *testing.T) {
t.Parallel()
c := qt.New(t)
pc, err := DecodeConfig(config.New())
c.Assert(err, qt.IsNil)
c.Assert(pc, qt.Not(qt.IsNil))
c.Assert(pc.Exec.Allow.Accept("a"), qt.IsFalse)
c.Assert(pc.Exec.Allow.Accept("npx"), qt.IsTrue)
c.Assert(pc.Exec.Allow.Accept("Npx"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("a"), qt.IsFalse)
c.Assert(pc.Exec.OsEnv.Accept("PATH"), qt.IsTrue)
c.Assert(pc.Exec.OsEnv.Accept("e"), qt.IsFalse)
c.Assert(pc.HTTP.URLs.Accept("https://example.org"), qt.IsTrue)
c.Assert(pc.HTTP.Methods.Accept("POST"), qt.IsTrue)
c.Assert(pc.HTTP.Methods.Accept("GET"), qt.IsTrue)
c.Assert(pc.HTTP.Methods.Accept("get"), qt.IsTrue)
c.Assert(pc.HTTP.Methods.Accept("DELETE"), qt.IsFalse)
}

102
config/security/whitelist.go Normal file
View file

@ -0,0 +1,102 @@
// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package security
import (
"encoding/json"
"fmt"
"regexp"
"strings"
)
const (
acceptNoneKeyword = "none"
)
// Whitelist holds a whitelist.
type Whitelist struct {
acceptNone bool
patterns []*regexp.Regexp
// Store this for debugging/error reporting
patternsStrings []string
}
func (w Whitelist) MarshalJSON() ([]byte, error) {
if w.acceptNone {
return json.Marshal(acceptNoneKeyword)
}
return json.Marshal(w.patternsStrings)
}
// NewWhitelist creates a new Whitelist from zero or more patterns.
// An empty patterns list or a pattern with the value 'none' will create
// a whitelist that will Accept noone.
func NewWhitelist(patterns ...string) Whitelist {
if len(patterns) == 0 {
return Whitelist{acceptNone: true}
}
var acceptSome bool
var patternsStrings []string
for _, p := range patterns {
if p == acceptNoneKeyword {
acceptSome = false
break
}
if ps := strings.TrimSpace(p); ps != "" {
acceptSome = true
patternsStrings = append(patternsStrings, ps)
}
}
if !acceptSome {
return Whitelist{
acceptNone: true,
}
}
var patternsr []*regexp.Regexp
for i := 0; i < len(patterns); i++ {
p := strings.TrimSpace(patterns[i])
if p == "" {
continue
}
patternsr = append(patternsr, regexp.MustCompile(p))
}
return Whitelist{patterns: patternsr, patternsStrings: patternsStrings}
}
// Accepted reports whether name is whitelisted.
func (w Whitelist) Accept(name string) bool {
if w.acceptNone {
return false
}
for _, p := range w.patterns {
if p.MatchString(name) {
return true
}
}
return false
}
func (w Whitelist) String() string {
return fmt.Sprint(w.patternsStrings)
}

47
config/security/whitelist_test.go Normal file
View file

@ -0,0 +1,47 @@
// Copyright 2021 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package security
import (
"testing"
qt "github.com/frankban/quicktest"
)
func TestWhitelist(t *testing.T) {
t.Parallel()
c := qt.New(t)
c.Run("none", func(c *qt.C) {
c.Assert(NewWhitelist("none", "foo").Accept("foo"), qt.IsFalse)
c.Assert(NewWhitelist().Accept("foo"), qt.IsFalse)
c.Assert(NewWhitelist("").Accept("foo"), qt.IsFalse)
c.Assert(NewWhitelist(" ", " ").Accept("foo"), qt.IsFalse)
c.Assert(Whitelist{}.Accept("foo"), qt.IsFalse)
})
c.Run("One", func(c *qt.C) {
w := NewWhitelist("^foo.*")
c.Assert(w.Accept("foo"), qt.IsTrue)
c.Assert(w.Accept("mfoo"), qt.IsFalse)
})
c.Run("Multiple", func(c *qt.C) {
w := NewWhitelist("^foo.*", "^bar.*")
c.Assert(w.Accept("foo"), qt.IsTrue)
c.Assert(w.Accept("bar"), qt.IsTrue)
c.Assert(w.Accept("mbar"), qt.IsFalse)
})
}

14
create/content.go
View file

@ -24,11 +24,11 @@ import (
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/paths"
"github.com/pkg/errors"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugofs"
@ -344,16 +344,18 @@ func (b *contentBuilder) openInEditorIfConfigured(filename string) error {
}
b.h.Log.Printf("Editing %q with %q ...\n", filename, editor)
cmd, err := b.h.Deps.ExecHelper.New(
editor,
filename,
hexec.WithStdin(os.Stdin),
hexec.WithStderr(os.Stderr),
hexec.WithStdout(os.Stdout),
)
cmd, err := hexec.SafeCommand(editor, filename)
if err != nil {
return err
}
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}

19
deps/deps.go vendored
View file

@ -8,8 +8,10 @@ import (
"github.com/pkg/errors"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/langs"
@ -36,6 +38,8 @@ type Deps struct {
// Used to log errors that may repeat itself many times.
LogDistinct loggers.Logger
ExecHelper *hexec.Exec
// The templates to use. This will usually implement the full tpl.TemplateManager.
tmpl tpl.TemplateHandler
@ -230,6 +234,12 @@ func New(cfg DepsCfg) (*Deps, error) {
cfg.OutputFormats = output.DefaultFormats
}
securityConfig, err := security.DecodeConfig(cfg.Cfg)
if err != nil {
return nil, errors.WithMessage(err, "failed to create security config from configuration")
}
execHelper := hexec.New(securityConfig)
ps, err := helpers.NewPathSpec(fs, cfg.Language, logger)
if err != nil {
return nil, errors.Wrap(err, "create PathSpec")
@ -243,12 +253,12 @@ func New(cfg DepsCfg) (*Deps, error) {
errorHandler := &globalErrHandler{}
buildState := &BuildState{}
resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, cfg.OutputFormats, cfg.MediaTypes)
resourceSpec, err := resources.NewSpec(ps, fileCaches, buildState, logger, errorHandler, execHelper, cfg.OutputFormats, cfg.MediaTypes)
if err != nil {
return nil, err
}
contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs)
contentSpec, err := helpers.NewContentSpec(cfg.Language, logger, ps.BaseFs.Content.Fs, execHelper)
if err != nil {
return nil, err
}
@ -269,6 +279,7 @@ func New(cfg DepsCfg) (*Deps, error) {
Fs: fs,
Log: ignorableLogger,
LogDistinct: logDistinct,
ExecHelper: execHelper,
templateProvider: cfg.TemplateProvider,
translationProvider: cfg.TranslationProvider,
WithTemplate: cfg.WithTemplate,
@ -311,7 +322,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er
return nil, err
}
d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs)
d.ContentSpec, err = helpers.NewContentSpec(l, d.Log, d.BaseFs.Content.Fs, d.ExecHelper)
if err != nil {
return nil, err
}
@ -322,7 +333,7 @@ func (d Deps) ForLanguage(cfg DepsCfg, onCreated func(d *Deps) error) (*Deps, er
// TODO(bep) clean up these inits.
resourceCache := d.ResourceSpec.ResourceCache
postBuildAssets := d.ResourceSpec.PostBuildAssets
d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, cfg.OutputFormats, cfg.MediaTypes)
d.ResourceSpec, err = resources.NewSpec(d.PathSpec, d.ResourceSpec.FileCaches, d.BuildState, d.Log, d.globalErrHandler, d.ExecHelper, cfg.OutputFormats, cfg.MediaTypes)
if err != nil {
return nil, err
}

13
docs/config/_default/security.toml Normal file
View file

@ -0,0 +1,13 @@
enableInlineShortcodes = false
[exec]
allow = ['^go$']
osEnv = ['^PATH$']
[funcs]
getenv = ['^HUGO_', '^REPOSITORY_URL$']
[http]
methods = ['(?i)GET|POST']
urls = ['.*']

21
docs/content/en/about/security-model/index.md
View file

@ -21,14 +21,29 @@ Hugo produces static output, so once built, the runtime is the browser (assuming
But when developing and building your site, the runtime is the `hugo` executable. Securing a runtime can be [a real challenge](https://blog.logrocket.com/how-to-protect-your-node-js-applications-from-malicious-dependencies-5f2e60ea08f9/).
**Hugo's main approach is that of sandboxing:**
**Hugo's main approach is that of sandboxing and a security policy with strict defaults:**
* Hugo has a virtual file system and only the main project (not third-party components) is allowed to mount directories or files outside the project root.
* Only the main project can walk symbolic links.
* User-defined components have only read-access to the filesystem.
* We shell out to some external binaries to support [Asciidoctor](/content-management/formats/#list-of-content-formats) and similar, but those binaries and their flags are predefined. General functions to run arbitrary external OS commands have been [discussed](https://github.com/gohugoio/hugo/issues/796), but not implemented because of security concerns.
* We shell out to some external binaries to support [Asciidoctor](/content-management/formats/#list-of-content-formats) and similar, but those binaries and their flags are predefined and disabled by default (see [Security Policy](#security-policy)). General functions to run arbitrary external OS commands have been [discussed](https://github.com/gohugoio/hugo/issues/796), but not implemented because of security concerns.
Hugo will soon introduce a concept of _Content Source Plugins_ (AKA _Pages from Data_), but the above will still hold true.
## Security Policy
{{< new-in "0.91.0" >}}
Hugo has a built-in security policy that restricts access to [os/exec](https://pkg.go.dev/os/exec), remote communication and similar.
The defdault configuration is listed below. And build using features not whitelisted in the security policy will faill with a detailed message about what needs to be done. Most of these settings are whitelists (string or slice, [Regular Expressions](https://pkg.go.dev/regexp) or `none` which matches nothing).
{{< code-toggle config="security" />}}
Note that these and other config settings in Hugo can be overridden by the OS environment. If you want to block all remote HTTP fetching of data:
```
HUGO_SECURITY_HTTP_URLS=none hugo
```
## Dependency Security

4
docs/content/en/getting-started/configuration.md
View file

@ -381,6 +381,10 @@ Maximum number of items in the RSS feed.
### sectionPagesMenu
See ["Section Menu for Lazy Bloggers"](/templates/menu-templates/#section-menu-for-lazy-bloggers).
### security
See [Security Policy](/about/security-model/#security-policy)
### sitemap
Default [sitemap configuration](/templates/sitemap-template/#configure-sitemapxml).

37
docs/data/docs.json
View file

@ -1775,9 +1775,15 @@
"permalinks": {
"_merge": "none"
},
"privacy": {
"_merge": "none"
},
"related": {
"_merge": "none"
},
"security": {
"_merge": "none"
},
"sitemap": {
"_merge": "none"
},
@ -1822,6 +1828,32 @@
"keepWhitespace": false
}
}
},
"security": {
"enableInlineShortcodes": false,
"exec": {
"allow": [
"^go$",
"^npx$",
"^postcss$"
],
"osEnv": [
"(?i)^(PATH|PATHEXT|APPDATA|TMP|TEMP|TERM)$"
]
},
"funcs": {
"getenv": [
"^HUGO_"
]
},
"http": {
"methods": [
"(?i)GET|POST"
],
"urls": [
".*"
]
}
}
},
"media": {
@ -1966,7 +1998,10 @@
"string": "image/jpeg",
"suffixes": [
"jpg",
"jpeg"
"jpeg",
"jpe",
"jif",
"jfif"
]
},
{

4
helpers/content.go
View file

@ -24,6 +24,7 @@ import (
"unicode"
"unicode/utf8"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/spf13/afero"
@ -64,7 +65,7 @@ type ContentSpec struct {
// NewContentSpec returns a ContentSpec initialized
// with the appropriate fields from the given config.Provider.
func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs) (*ContentSpec, error) {
func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.Fs, ex *hexec.Exec) (*ContentSpec, error) {
spec := &ContentSpec{
summaryLength: cfg.GetInt("summaryLength"),
BuildFuture: cfg.GetBool("buildFuture"),
@ -78,6 +79,7 @@ func NewContentSpec(cfg config.Provider, logger loggers.Logger, contentFs afero.
Cfg: cfg,
ContentFs: contentFs,
Logger: logger,
Exec: ex,
})
if err != nil {
return nil, err

2
helpers/content_test.go
View file

@ -110,7 +110,7 @@ func TestNewContentSpec(t *testing.T) {
cfg.Set("buildExpired", true)
cfg.Set("buildDrafts", true)
spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil)
c.Assert(err, qt.IsNil)
c.Assert(spec.summaryLength, qt.Equals, 32)

2
helpers/general_test.go
View file

@ -30,7 +30,7 @@ import (
func TestResolveMarkup(t *testing.T) {
c := qt.New(t)
cfg := config.New()
spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs())
spec, err := NewContentSpec(cfg, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil)
c.Assert(err, qt.IsNil)
for i, this := range []struct {

2
helpers/testhelpers_test.go
View file

@ -50,7 +50,7 @@ func newTestCfg() config.Provider {
func newTestContentSpec() *ContentSpec {
v := config.New()
spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs())
spec, err := NewContentSpec(v, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil)
if err != nil {
panic(err)
}

2
htesting/test_helpers.go
View file

@ -115,7 +115,7 @@ func IsGitHubAction() bool {
// SupportsAll reports whether the running system supports all Hugo features,
// e.g. Asciidoc, Pandoc etc.
func SupportsAll() bool {
return IsGitHubAction()
return IsGitHubAction() || os.Getenv("CI_LOCAL") != ""
}
// GoMinorVersion returns the minor version of the current Go version,

9
hugolib/config.go
View file

@ -18,6 +18,7 @@ import (
"path/filepath"
"strings"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/common/maps"
@ -41,6 +42,7 @@ import (
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/privacy"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/config/services"
"github.com/gohugoio/hugo/helpers"
"github.com/spf13/afero"
@ -377,6 +379,12 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
return nil, nil, err
}
secConfig, err := security.DecodeConfig(v1)
if err != nil {
return nil, nil, err
}
ex := hexec.New(secConfig)
v1.Set("filecacheConfigs", filecacheConfigs)
var configFilenames []string
@ -405,6 +413,7 @@ func (l configLoader) collectModules(modConfig modules.Config, v1 config.Provide
modulesClient := modules.NewClient(modules.ClientConfig{
Fs: l.Fs,
Logger: l.Logger,
Exec: ex,
HookBeforeFinalize: hook,
WorkingDir: workingDir,
ThemesDir: themesDir,

10
hugolib/js_test.go
View file

@ -20,7 +20,6 @@ import (
"runtime"
"testing"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/htesting"
@ -123,10 +122,9 @@ TS2: {{ template "print" $ts2 }}
b.WithSourceFile("assets/js/included.js", includedJS)
cmd, err := hexec.SafeCommand("npm", "install")
cmd := b.NpmInstall()
err = cmd.Run()
b.Assert(err, qt.IsNil)
out, err := cmd.CombinedOutput()
b.Assert(err, qt.IsNil, qt.Commentf(string(out)))
b.Build(BuildCfg{})
@ -195,8 +193,8 @@ require github.com/gohugoio/hugoTestProjectJSModImports v0.9.0 // indirect
}`)
b.Assert(os.Chdir(workDir), qt.IsNil)
cmd, _ := hexec.SafeCommand("npm", "install")
_, err = cmd.CombinedOutput()
cmd := b.NpmInstall()
err = cmd.Run()
b.Assert(err, qt.IsNil)
b.Build(BuildCfg{})

93
hugolib/page_test.go
View file

@ -24,8 +24,6 @@ import (
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/markup/rst"
"github.com/gohugoio/hugo/markup/asciidocext"
"github.com/gohugoio/hugo/config"
@ -370,6 +368,7 @@ func normalizeExpected(ext, str string) string {
func testAllMarkdownEnginesForPages(t *testing.T,
assertFunc func(t *testing.T, ext string, pages page.Pages), settings map[string]interface{}, pageSources ...string) {
engines := []struct {
ext string
shouldExecute func() bool
@ -377,7 +376,7 @@ func testAllMarkdownEnginesForPages(t *testing.T,
{"md", func() bool { return true }},
{"mmark", func() bool { return true }},
{"ad", func() bool { return asciidocext.Supports() }},
{"rst", func() bool { return rst.Supports() }},
{"rst", func() bool { return true }},
}
for _, e := range engines {
@ -385,48 +384,58 @@ func testAllMarkdownEnginesForPages(t *testing.T,
continue
}
cfg, fs := newTestCfg(func(cfg config.Provider) error {
for k, v := range settings {
cfg.Set(k, v)
t.Run(e.ext, func(t *testing.T) {
cfg, fs := newTestCfg(func(cfg config.Provider) error {
for k, v := range settings {
cfg.Set(k, v)
}
return nil
})
contentDir := "content"
if s := cfg.GetString("contentDir"); s != "" {
contentDir = s
}
return nil
cfg.Set("security", map[string]interface{}{
"exec": map[string]interface{}{
"allow": []string{"^python$", "^rst2html.*", "^asciidoctor$"},
},
})
var fileSourcePairs []string
for i, source := range pageSources {
fileSourcePairs = append(fileSourcePairs, fmt.Sprintf("p%d.%s", i, e.ext), source)
}
for i := 0; i < len(fileSourcePairs); i += 2 {
writeSource(t, fs, filepath.Join(contentDir, fileSourcePairs[i]), fileSourcePairs[i+1])
}
// Add a content page for the home page
homePath := fmt.Sprintf("_index.%s", e.ext)
writeSource(t, fs, filepath.Join(contentDir, homePath), homePage)
b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()
b.Build(BuildCfg{})
s := b.H.Sites[0]
b.Assert(len(s.RegularPages()), qt.Equals, len(pageSources))
assertFunc(t, e.ext, s.RegularPages())
home, err := s.Info.Home()
b.Assert(err, qt.IsNil)
b.Assert(home, qt.Not(qt.IsNil))
b.Assert(home.File().Path(), qt.Equals, homePath)
b.Assert(content(home), qt.Contains, "Home Page Content")
})
contentDir := "content"
if s := cfg.GetString("contentDir"); s != "" {
contentDir = s
}
var fileSourcePairs []string
for i, source := range pageSources {
fileSourcePairs = append(fileSourcePairs, fmt.Sprintf("p%d.%s", i, e.ext), source)
}
for i := 0; i < len(fileSourcePairs); i += 2 {
writeSource(t, fs, filepath.Join(contentDir, fileSourcePairs[i]), fileSourcePairs[i+1])
}
// Add a content page for the home page
homePath := fmt.Sprintf("_index.%s", e.ext)
writeSource(t, fs, filepath.Join(contentDir, homePath), homePage)
b := newTestSitesBuilderFromDepsCfg(t, deps.DepsCfg{Fs: fs, Cfg: cfg}).WithNothingAdded()
b.Build(BuildCfg{SkipRender: true})
s := b.H.Sites[0]
b.Assert(len(s.RegularPages()), qt.Equals, len(pageSources))
assertFunc(t, e.ext, s.RegularPages())
home, err := s.Info.Home()
b.Assert(err, qt.IsNil)
b.Assert(home, qt.Not(qt.IsNil))
b.Assert(home.File().Path(), qt.Equals, homePath)
b.Assert(content(home), qt.Contains, "Home Page Content")
}
}

14
hugolib/resource_chain_babel_test.go
View file

@ -21,8 +21,6 @@ import (
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/common/hexec"
jww "github.com/spf13/jwalterweatherman"
"github.com/gohugoio/hugo/htesting"
@ -51,7 +49,7 @@ func TestResourceChainBabel(t *testing.T) {
"devDependencies": {
"@babel/cli": "7.8.4",
"@babel/core": "7.9.0",
"@babel/core": "7.9.0",
"@babel/preset-env": "7.9.5"
}
}
@ -94,6 +92,12 @@ class Car2 {
v := config.New()
v.Set("workingDir", workDir)
v.Set("disableKinds", []string{"taxonomy", "term", "page"})
v.Set("security", map[string]interface{}{
"exec": map[string]interface{}{
"allow": []string{"^npx$", "^babel$"},
},
})
b := newTestSitesBuilder(t).WithLogger(logger)
// Need to use OS fs for this.
@ -123,8 +127,8 @@ Transpiled3: {{ $transpiled.Permalink }}
b.WithSourceFile("babel.config.js", babelConfig)
b.Assert(os.Chdir(workDir), qt.IsNil)
cmd, _ := hexec.SafeCommand("npm", "install")
_, err = cmd.CombinedOutput()
cmd := b.NpmInstall()
err = cmd.Run()
b.Assert(err, qt.IsNil)
b.Build(BuildCfg{})

8
hugolib/resource_chain_test.go
View file

@ -32,8 +32,6 @@ import (
"testing"
"time"
"github.com/gohugoio/hugo/common/hexec"
jww "github.com/spf13/jwalterweatherman"
"github.com/gohugoio/hugo/common/herrors"
@ -387,8 +385,6 @@ T1: {{ $r.Content }}
}
func TestResourceChainBasic(t *testing.T) {
t.Parallel()
ts := httptest.NewServer(http.FileServer(http.Dir("testdata/")))
t.Cleanup(func() {
ts.Close()
@ -1184,8 +1180,8 @@ class-in-b {
b.WithSourceFile("postcss.config.js", postcssConfig)
b.Assert(os.Chdir(workDir), qt.IsNil)
cmd, err := hexec.SafeCommand("npm", "install")
_, err = cmd.CombinedOutput()
cmd := b.NpmInstall()
err = cmd.Run()
b.Assert(err, qt.IsNil)
b.Build(BuildCfg{})

202
hugolib/securitypolicies_test.go Normal file
View file

@ -0,0 +1,202 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"fmt"
"net/http"
"net/http/httptest"
"runtime"
"testing"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/markup/asciidocext"
"github.com/gohugoio/hugo/markup/pandoc"
"github.com/gohugoio/hugo/markup/rst"
"github.com/gohugoio/hugo/resources/resource_transformers/tocss/dartsass"
)
func TestSecurityPolicies(t *testing.T) {
c := qt.New(t)
testVariant := func(c *qt.C, withBuilder func(b *sitesBuilder), expectErr string) {
c.Helper()
b := newTestSitesBuilder(c)
withBuilder(b)
if expectErr != "" {
err := b.BuildE(BuildCfg{})
b.Assert(err, qt.IsNotNil)
b.Assert(err, qt.ErrorMatches, expectErr)
} else {
b.Build(BuildCfg{})
}
}
httpTestVariant := func(c *qt.C, templ, expectErr string, withBuilder func(b *sitesBuilder)) {
ts := httptest.NewServer(http.FileServer(http.Dir("testdata/")))
c.Cleanup(func() {
ts.Close()
})
cb := func(b *sitesBuilder) {
b.WithTemplatesAdded("index.html", fmt.Sprintf(templ, ts.URL))
if withBuilder != nil {
withBuilder(b)
}
}
testVariant(c, cb, expectErr)
}
c.Run("os.GetEnv, denied", func(c *qt.C) {
c.Parallel()
cb := func(b *sitesBuilder) {
b.WithTemplatesAdded("index.html", `{{ os.Getenv "FOOBAR" }}`)
}
testVariant(c, cb, `(?s).*"FOOBAR" is not whitelisted in policy "security\.funcs\.getenv".*`)
})
c.Run("os.GetEnv, OK", func(c *qt.C) {
c.Parallel()
cb := func(b *sitesBuilder) {
b.WithTemplatesAdded("index.html", `{{ os.Getenv "HUGO_FOO" }}`)
}
testVariant(c, cb, "")
})
c.Run("Asciidoc, denied", func(c *qt.C) {
c.Parallel()
if !asciidocext.Supports() {
c.Skip()
}
cb := func(b *sitesBuilder) {
b.WithContent("page.ad", "foo")
}
testVariant(c, cb, `(?s).*"asciidoctor" is not whitelisted in policy "security\.exec\.allow".*`)
})
c.Run("RST, denied", func(c *qt.C) {
c.Parallel()
if !rst.Supports() {
c.Skip()
}
cb := func(b *sitesBuilder) {
b.WithContent("page.rst", "foo")
}
if runtime.GOOS == "windows" {
testVariant(c, cb, `(?s).*python(\.exe)?" is not whitelisted in policy "security\.exec\.allow".*`)
} else {
testVariant(c, cb, `(?s).*"rst2html(\.py)?" is not whitelisted in policy "security\.exec\.allow".*`)
}
})
c.Run("Pandoc, denied", func(c *qt.C) {
c.Parallel()
if !pandoc.Supports() {
c.Skip()
}
cb := func(b *sitesBuilder) {
b.WithContent("page.pdc", "foo")
}
testVariant(c, cb, `"(?s).*pandoc" is not whitelisted in policy "security\.exec\.allow".*`)
})
c.Run("Dart SASS, OK", func(c *qt.C) {
c.Parallel()
if !dartsass.Supports() {
c.Skip()
}
cb := func(b *sitesBuilder) {
b.WithTemplatesAdded("index.html", `{{ $scss := "body { color: #333; }" | resources.FromString "foo.scss" | resources.ToCSS (dict "transpiler" "dartsass") }}`)
}
testVariant(c, cb, "")
})
c.Run("Dart SASS, denied", func(c *qt.C) {
c.Parallel()
if !dartsass.Supports() {
c.Skip()
}
cb := func(b *sitesBuilder) {
b.WithConfigFile("toml", `
[security]
[security.exec]
allow="none"
`)
b.WithTemplatesAdded("index.html", `{{ $scss := "body { color: #333; }" | resources.FromString "foo.scss" | resources.ToCSS (dict "transpiler" "dartsass") }}`)
}
testVariant(c, cb, `(?s).*"dart-sass-embedded" is not whitelisted in policy "security\.exec\.allow".*`)
})
c.Run("resources.Get, OK", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil)
})
c.Run("resources.Get, denied method", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil)
})
c.Run("resources.Get, denied URL", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
func(b *sitesBuilder) {
b.WithConfigFile("toml", `
[security]
[security.http]
urls="none"
`)
})
})
c.Run("getJSON, OK", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := getJSON "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil)
})
c.Run("getJSON, denied URL", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $json := getJSON "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
func(b *sitesBuilder) {
b.WithConfigFile("toml", `
[security]
[security.http]
urls="none"
`)
})
})
c.Run("getCSV, denied URL", func(c *qt.C) {
c.Parallel()
httpTestVariant(c, `{{ $d := getCSV ";" "%[1]s/cities.csv" }}{{ $d.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
func(b *sitesBuilder) {
b.WithConfigFile("toml", `
[security]
[security.http]
urls="none"
`)
})
})
}

4
hugolib/shortcode.go
View file

@ -257,7 +257,7 @@ func newShortcodeHandler(p *pageState, s *Site, placeholderFunc func() string) *
sh := &shortcodeHandler{
p: p,
s: s,
enableInlineShortcodes: s.enableInlineShortcodes,
enableInlineShortcodes: s.ExecHelper.Sec().EnableInlineShortcodes,
shortcodes: make([]*shortcode, 0, 4),
nameSet: make(map[string]bool),
}
@ -287,7 +287,7 @@ func renderShortcode(
var hasVariants bool
if sc.isInline {
if !p.s.enableInlineShortcodes {
if !p.s.ExecHelper.Sec().EnableInlineShortcodes {
return "", false, nil
}
templName := path.Join("_inline_shortcode", p.File().Path(), sc.name)

6
hugolib/shortcode_test.go
View file

@ -619,6 +619,12 @@ title: "Foo"
cfg.Set("uglyURLs", false)
cfg.Set("verbose", true)
cfg.Set("security", map[string]interface{}{
"exec": map[string]interface{}{
"allow": []string{"^python$", "^rst2html.*", "^asciidoctor$"},
},
})
cfg.Set("markup.highlight.noClasses", false)
cfg.Set("markup.highlight.codeFences", true)
cfg.Set("markup", map[string]interface{}{

42
hugolib/site.go
View file

@ -120,8 +120,6 @@ type Site struct {
disabledKinds map[string]bool
enableInlineShortcodes bool
// Output formats defined in site config per Page Kind, or some defaults
// if not set.
// Output formats defined in Page front matter will override these.
@ -378,25 +376,24 @@ func (s *Site) isEnabled(kind string) bool {
// reset returns a new Site prepared for rebuild.
func (s *Site) reset() *Site {
return &Site{
Deps: s.Deps,
disabledKinds: s.disabledKinds,
titleFunc: s.titleFunc,
relatedDocsHandler: s.relatedDocsHandler.Clone(),
siteRefLinker: s.siteRefLinker,
outputFormats: s.outputFormats,
rc: s.rc,
outputFormatsConfig: s.outputFormatsConfig,
frontmatterHandler: s.frontmatterHandler,
mediaTypesConfig: s.mediaTypesConfig,
language: s.language,
siteBucket: s.siteBucket,
h: s.h,
publisher: s.publisher,
siteConfigConfig: s.siteConfigConfig,
enableInlineShortcodes: s.enableInlineShortcodes,
init: s.init,
PageCollections: s.PageCollections,
siteCfg: s.siteCfg,
Deps: s.Deps,
disabledKinds: s.disabledKinds,
titleFunc: s.titleFunc,
relatedDocsHandler: s.relatedDocsHandler.Clone(),
siteRefLinker: s.siteRefLinker,
outputFormats: s.outputFormats,
rc: s.rc,
outputFormatsConfig: s.outputFormatsConfig,
frontmatterHandler: s.frontmatterHandler,
mediaTypesConfig: s.mediaTypesConfig,
language: s.language,
siteBucket: s.siteBucket,
h: s.h,
publisher: s.publisher,
siteConfigConfig: s.siteConfigConfig,
init: s.init,
PageCollections: s.PageCollections,
siteCfg: s.siteCfg,
}
}
@ -564,8 +561,7 @@ But this also means that your site configuration may not do what you expect. If
outputFormatsConfig: siteOutputFormatsConfig,
mediaTypesConfig: siteMediaTypesConfig,
enableInlineShortcodes: cfg.Language.GetBool("enableInlineShortcodes"),
siteCfg: siteConfig,
siteCfg: siteConfig,
titleFunc: titleFunc,

130
hugolib/testdata/cities.csv vendored Normal file
View file

@ -0,0 +1,130 @@
"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State"
41, 5, 59, "N", 80, 39, 0, "W", "Youngstown", OH
42, 52, 48, "N", 97, 23, 23, "W", "Yankton", SD
46, 35, 59, "N", 120, 30, 36, "W", "Yakima", WA
42, 16, 12, "N", 71, 48, 0, "W", "Worcester", MA
43, 37, 48, "N", 89, 46, 11, "W", "Wisconsin Dells", WI
36, 5, 59, "N", 80, 15, 0, "W", "Winston-Salem", NC
49, 52, 48, "N", 97, 9, 0, "W", "Winnipeg", MB
39, 11, 23, "N", 78, 9, 36, "W", "Winchester", VA
34, 14, 24, "N", 77, 55, 11, "W", "Wilmington", NC
39, 45, 0, "N", 75, 33, 0, "W", "Wilmington", DE
48, 9, 0, "N", 103, 37, 12, "W", "Williston", ND
41, 15, 0, "N", 77, 0, 0, "W", "Williamsport", PA
37, 40, 48, "N", 82, 16, 47, "W", "Williamson", WV
33, 54, 0, "N", 98, 29, 23, "W", "Wichita Falls", TX
37, 41, 23, "N", 97, 20, 23, "W", "Wichita", KS
40, 4, 11, "N", 80, 43, 12, "W", "Wheeling", WV
26, 43, 11, "N", 80, 3, 0, "W", "West Palm Beach", FL
47, 25, 11, "N", 120, 19, 11, "W", "Wenatchee", WA
41, 25, 11, "N", 122, 23, 23, "W", "Weed", CA
31, 13, 11, "N", 82, 20, 59, "W", "Waycross", GA
44, 57, 35, "N", 89, 38, 23, "W", "Wausau", WI
42, 21, 36, "N", 87, 49, 48, "W", "Waukegan", IL
44, 54, 0, "N", 97, 6, 36, "W", "Watertown", SD
43, 58, 47, "N", 75, 55, 11, "W", "Watertown", NY
42, 30, 0, "N", 92, 20, 23, "W", "Waterloo", IA
41, 32, 59, "N", 73, 3, 0, "W", "Waterbury", CT
38, 53, 23, "N", 77, 1, 47, "W", "Washington", DC
41, 50, 59, "N", 79, 8, 23, "W", "Warren", PA
46, 4, 11, "N", 118, 19, 48, "W", "Walla Walla", WA
31, 32, 59, "N", 97, 8, 23, "W", "Waco", TX
38, 40, 48, "N", 87, 31, 47, "W", "Vincennes", IN
28, 48, 35, "N", 97, 0, 36, "W", "Victoria", TX
32, 20, 59, "N", 90, 52, 47, "W", "Vicksburg", MS
49, 16, 12, "N", 123, 7, 12, "W", "Vancouver", BC
46, 55, 11, "N", 98, 0, 36, "W", "Valley City", ND
30, 49, 47, "N", 83, 16, 47, "W", "Valdosta", GA
43, 6, 36, "N", 75, 13, 48, "W", "Utica", NY
39, 54, 0, "N", 79, 43, 48, "W", "Uniontown", PA
32, 20, 59, "N", 95, 18, 0, "W", "Tyler", TX
42, 33, 36, "N", 114, 28, 12, "W", "Twin Falls", ID
33, 12, 35, "N", 87, 34, 11, "W", "Tuscaloosa", AL
34, 15, 35, "N", 88, 42, 35, "W", "Tupelo", MS
36, 9, 35, "N", 95, 54, 36, "W", "Tulsa", OK
32, 13, 12, "N", 110, 58, 12, "W", "Tucson", AZ
37, 10, 11, "N", 104, 30, 36, "W", "Trinidad", CO
40, 13, 47, "N", 74, 46, 11, "W", "Trenton", NJ
44, 45, 35, "N", 85, 37, 47, "W", "Traverse City", MI
43, 39, 0, "N", 79, 22, 47, "W", "Toronto", ON
39, 2, 59, "N", 95, 40, 11, "W", "Topeka", KS
41, 39, 0, "N", 83, 32, 24, "W", "Toledo", OH
33, 25, 48, "N", 94, 3, 0, "W", "Texarkana", TX
39, 28, 12, "N", 87, 24, 36, "W", "Terre Haute", IN
27, 57, 0, "N", 82, 26, 59, "W", "Tampa", FL
30, 27, 0, "N", 84, 16, 47, "W", "Tallahassee", FL
47, 14, 24, "N", 122, 25, 48, "W", "Tacoma", WA
43, 2, 59, "N", 76, 9, 0, "W", "Syracuse", NY
32, 35, 59, "N", 82, 20, 23, "W", "Swainsboro", GA
33, 55, 11, "N", 80, 20, 59, "W", "Sumter", SC
40, 59, 24, "N", 75, 11, 24, "W", "Stroudsburg", PA
37, 57, 35, "N", 121, 17, 24, "W", "Stockton", CA
44, 31, 12, "N", 89, 34, 11, "W", "Stevens Point", WI
40, 21, 36, "N", 80, 37, 12, "W", "Steubenville", OH
40, 37, 11, "N", 103, 13, 12, "W", "Sterling", CO
38, 9, 0, "N", 79, 4, 11, "W", "Staunton", VA
39, 55, 11, "N", 83, 48, 35, "W", "Springfield", OH
37, 13, 12, "N", 93, 17, 24, "W", "Springfield", MO
42, 5, 59, "N", 72, 35, 23, "W", "Springfield", MA
39, 47, 59, "N", 89, 39, 0, "W", "Springfield", IL
47, 40, 11, "N", 117, 24, 36, "W", "Spokane", WA
41, 40, 48, "N", 86, 15, 0, "W", "South Bend", IN
43, 32, 24, "N", 96, 43, 48, "W", "Sioux Falls", SD
42, 29, 24, "N", 96, 23, 23, "W", "Sioux City", IA
32, 30, 35, "N", 93, 45, 0, "W", "Shreveport", LA
33, 38, 23, "N", 96, 36, 36, "W", "Sherman", TX
44, 47, 59, "N", 106, 57, 35, "W", "Sheridan", WY
35, 13, 47, "N", 96, 40, 48, "W", "Seminole", OK
32, 25, 11, "N", 87, 1, 11, "W", "Selma", AL
38, 42, 35, "N", 93, 13, 48, "W", "Sedalia", MO
47, 35, 59, "N", 122, 19, 48, "W", "Seattle", WA
41, 24, 35, "N", 75, 40, 11, "W", "Scranton", PA
41, 52, 11, "N", 103, 39, 36, "W", "Scottsbluff", NB
42, 49, 11, "N", 73, 56, 59, "W", "Schenectady", NY
32, 4, 48, "N", 81, 5, 23, "W", "Savannah", GA
46, 29, 24, "N", 84, 20, 59, "W", "Sault Sainte Marie", MI
27, 20, 24, "N", 82, 31, 47, "W", "Sarasota", FL
38, 26, 23, "N", 122, 43, 12, "W", "Santa Rosa", CA
35, 40, 48, "N", 105, 56, 59, "W", "Santa Fe", NM
34, 25, 11, "N", 119, 41, 59, "W", "Santa Barbara", CA
33, 45, 35, "N", 117, 52, 12, "W", "Santa Ana", CA
37, 20, 24, "N", 121, 52, 47, "W", "San Jose", CA
37, 46, 47, "N", 122, 25, 11, "W", "San Francisco", CA
41, 27, 0, "N", 82, 42, 35, "W", "Sandusky", OH
32, 42, 35, "N", 117, 9, 0, "W", "San Diego", CA
34, 6, 36, "N", 117, 18, 35, "W", "San Bernardino", CA
29, 25, 12, "N", 98, 30, 0, "W", "San Antonio", TX
31, 27, 35, "N", 100, 26, 24, "W", "San Angelo", TX
40, 45, 35, "N", 111, 52, 47, "W", "Salt Lake City", UT
38, 22, 11, "N", 75, 35, 59, "W", "Salisbury", MD
36, 40, 11, "N", 121, 39, 0, "W", "Salinas", CA
38, 50, 24, "N", 97, 36, 36, "W", "Salina", KS
38, 31, 47, "N", 106, 0, 0, "W", "Salida", CO
44, 56, 23, "N", 123, 1, 47, "W", "Salem", OR
44, 57, 0, "N", 93, 5, 59, "W", "Saint Paul", MN
38, 37, 11, "N", 90, 11, 24, "W", "Saint Louis", MO
39, 46, 12, "N", 94, 50, 23, "W", "Saint Joseph", MO
42, 5, 59, "N", 86, 28, 48, "W", "Saint Joseph", MI
44, 25, 11, "N", 72, 1, 11, "W", "Saint Johnsbury", VT
45, 34, 11, "N", 94, 10, 11, "W", "Saint Cloud", MN
29, 53, 23, "N", 81, 19, 11, "W", "Saint Augustine", FL
43, 25, 48, "N", 83, 56, 24, "W", "Saginaw", MI
38, 35, 24, "N", 121, 29, 23, "W", "Sacramento", CA
43, 36, 36, "N", 72, 58, 12, "W", "Rutland", VT
33, 24, 0, "N", 104, 31, 47, "W", "Roswell", NM
35, 56, 23, "N", 77, 48, 0, "W", "Rocky Mount", NC
41, 35, 24, "N", 109, 13, 48, "W", "Rock Springs", WY
42, 16, 12, "N", 89, 5, 59, "W", "Rockford", IL
43, 9, 35, "N", 77, 36, 36, "W", "Rochester", NY
44, 1, 12, "N", 92, 27, 35, "W", "Rochester", MN
37, 16, 12, "N", 79, 56, 24, "W", "Roanoke", VA
37, 32, 24, "N", 77, 26, 59, "W", "Richmond", VA
39, 49, 48, "N", 84, 53, 23, "W", "Richmond", IN
38, 46, 12, "N", 112, 5, 23, "W", "Richfield", UT
45, 38, 23, "N", 89, 25, 11, "W", "Rhinelander", WI
39, 31, 12, "N", 119, 48, 35, "W", "Reno", NV
50, 25, 11, "N", 104, 39, 0, "W", "Regina", SA
40, 10, 48, "N", 122, 14, 23, "W", "Red Bluff", CA
40, 19, 48, "N", 75, 55, 48, "W", "Reading", PA
41, 9, 35, "N", 81, 14, 23, "W", "Ravenna", OH
1 LatD LatM LatS NS LonD LonM LonS EW City State
2 41 5 59 N 80 39 0 W Youngstown OH
3 42 52 48 N 97 23 23 W Yankton SD
4 46 35 59 N 120 30 36 W Yakima WA
5 42 16 12 N 71 48 0 W Worcester MA
6 43 37 48 N 89 46 11 W Wisconsin Dells WI
7 36 5 59 N 80 15 0 W Winston-Salem NC
8 49 52 48 N 97 9 0 W Winnipeg MB
9 39 11 23 N 78 9 36 W Winchester VA
10 34 14 24 N 77 55 11 W Wilmington NC
11 39 45 0 N 75 33 0 W Wilmington DE
12 48 9 0 N 103 37 12 W Williston ND
13 41 15 0 N 77 0 0 W Williamsport PA
14 37 40 48 N 82 16 47 W Williamson WV
15 33 54 0 N 98 29 23 W Wichita Falls TX
16 37 41 23 N 97 20 23 W Wichita KS
17 40 4 11 N 80 43 12 W Wheeling WV
18 26 43 11 N 80 3 0 W West Palm Beach FL
19 47 25 11 N 120 19 11 W Wenatchee WA
20 41 25 11 N 122 23 23 W Weed CA
21 31 13 11 N 82 20 59 W Waycross GA
22 44 57 35 N 89 38 23 W Wausau WI
23 42 21 36 N 87 49 48 W Waukegan IL
24 44 54 0 N 97 6 36 W Watertown SD
25 43 58 47 N 75 55 11 W Watertown NY
26 42 30 0 N 92 20 23 W Waterloo IA
27 41 32 59 N 73 3 0 W Waterbury CT
28 38 53 23 N 77 1 47 W Washington DC
29 41 50 59 N 79 8 23 W Warren PA
30 46 4 11 N 118 19 48 W Walla Walla WA
31 31 32 59 N 97 8 23 W Waco TX
32 38 40 48 N 87 31 47 W Vincennes IN
33 28 48 35 N 97 0 36 W Victoria TX
34 32 20 59 N 90 52 47 W Vicksburg MS
35 49 16 12 N 123 7 12 W Vancouver BC
36 46 55 11 N 98 0 36 W Valley City ND
37 30 49 47 N 83 16 47 W Valdosta GA
38 43 6 36 N 75 13 48 W Utica NY
39 39 54 0 N 79 43 48 W Uniontown PA
40 32 20 59 N 95 18 0 W Tyler TX
41 42 33 36 N 114 28 12 W Twin Falls ID
42 33 12 35 N 87 34 11 W Tuscaloosa AL
43 34 15 35 N 88 42 35 W Tupelo MS
44 36 9 35 N 95 54 36 W Tulsa OK
45 32 13 12 N 110 58 12 W Tucson AZ
46 37 10 11 N 104 30 36 W Trinidad CO
47 40 13 47 N 74 46 11 W Trenton NJ
48 44 45 35 N 85 37 47 W Traverse City MI
49 43 39 0 N 79 22 47 W Toronto ON
50 39 2 59 N 95 40 11 W Topeka KS
51 41 39 0 N 83 32 24 W Toledo OH
52 33 25 48 N 94 3 0 W Texarkana TX
53 39 28 12 N 87 24 36 W Terre Haute IN
54 27 57 0 N 82 26 59 W Tampa FL
55 30 27 0 N 84 16 47 W Tallahassee FL
56 47 14 24 N 122 25 48 W Tacoma WA
57 43 2 59 N 76 9 0 W Syracuse NY
58 32 35 59 N 82 20 23 W Swainsboro GA
59 33 55 11 N 80 20 59 W Sumter SC
60 40 59 24 N 75 11 24 W Stroudsburg PA
61 37 57 35 N 121 17 24 W Stockton CA
62 44 31 12 N 89 34 11 W Stevens Point WI
63 40 21 36 N 80 37 12 W Steubenville OH
64 40 37 11 N 103 13 12 W Sterling CO
65 38 9 0 N 79 4 11 W Staunton VA
66 39 55 11 N 83 48 35 W Springfield OH
67 37 13 12 N 93 17 24 W Springfield MO
68 42 5 59 N 72 35 23 W Springfield MA
69 39 47 59 N 89 39 0 W Springfield IL
70 47 40 11 N 117 24 36 W Spokane WA
71 41 40 48 N 86 15 0 W South Bend IN
72 43 32 24 N 96 43 48 W Sioux Falls SD
73 42 29 24 N 96 23 23 W Sioux City IA
74 32 30 35 N 93 45 0 W Shreveport LA
75 33 38 23 N 96 36 36 W Sherman TX
76 44 47 59 N 106 57 35 W Sheridan WY
77 35 13 47 N 96 40 48 W Seminole OK
78 32 25 11 N 87 1 11 W Selma AL
79 38 42 35 N 93 13 48 W Sedalia MO
80 47 35 59 N 122 19 48 W Seattle WA
81 41 24 35 N 75 40 11 W Scranton PA
82 41 52 11 N 103 39 36 W Scottsbluff NB
83 42 49 11 N 73 56 59 W Schenectady NY
84 32 4 48 N 81 5 23 W Savannah GA
85 46 29 24 N 84 20 59 W Sault Sainte Marie MI
86 27 20 24 N 82 31 47 W Sarasota FL
87 38 26 23 N 122 43 12 W Santa Rosa CA
88 35 40 48 N 105 56 59 W Santa Fe NM
89 34 25 11 N 119 41 59 W Santa Barbara CA
90 33 45 35 N 117 52 12 W Santa Ana CA
91 37 20 24 N 121 52 47 W San Jose CA
92 37 46 47 N 122 25 11 W San Francisco CA
93 41 27 0 N 82 42 35 W Sandusky OH
94 32 42 35 N 117 9 0 W San Diego CA
95 34 6 36 N 117 18 35 W San Bernardino CA
96 29 25 12 N 98 30 0 W San Antonio TX
97 31 27 35 N 100 26 24 W San Angelo TX
98 40 45 35 N 111 52 47 W Salt Lake City UT
99 38 22 11 N 75 35 59 W Salisbury MD
100 36 40 11 N 121 39 0 W Salinas CA
101 38 50 24 N 97 36 36 W Salina KS
102 38 31 47 N 106 0 0 W Salida CO
103 44 56 23 N 123 1 47 W Salem OR
104 44 57 0 N 93 5 59 W Saint Paul MN
105 38 37 11 N 90 11 24 W Saint Louis MO
106 39 46 12 N 94 50 23 W Saint Joseph MO
107 42 5 59 N 86 28 48 W Saint Joseph MI
108 44 25 11 N 72 1 11 W Saint Johnsbury VT
109 45 34 11 N 94 10 11 W Saint Cloud MN
110 29 53 23 N 81 19 11 W Saint Augustine FL
111 43 25 48 N 83 56 24 W Saginaw MI
112 38 35 24 N 121 29 23 W Sacramento CA
113 43 36 36 N 72 58 12 W Rutland VT
114 33 24 0 N 104 31 47 W Roswell NM
115 35 56 23 N 77 48 0 W Rocky Mount NC
116 41 35 24 N 109 13 48 W Rock Springs WY
117 42 16 12 N 89 5 59 W Rockford IL
118 43 9 35 N 77 36 36 W Rochester NY
119 44 1 12 N 92 27 35 W Rochester MN
120 37 16 12 N 79 56 24 W Roanoke VA
121 37 32 24 N 77 26 59 W Richmond VA
122 39 49 48 N 84 53 23 W Richmond IN
123 38 46 12 N 112 5 23 W Richfield UT
124 45 38 23 N 89 25 11 W Rhinelander WI
125 39 31 12 N 119 48 35 W Reno NV
126 50 25 11 N 104 39 0 W Regina SA
127 40 10 48 N 122 14 23 W Red Bluff CA
128 40 19 48 N 75 55 48 W Reading PA
129 41 9 35 N 81 14 23 W Ravenna OH

5
hugolib/testdata/fruits.json vendored Normal file
View file

@ -0,0 +1,5 @@
{
"fruit": "Apple",
"size": "Large",
"color": "Red"
}

12
hugolib/testhelpers_test.go
View file

@ -18,6 +18,7 @@ import (
"time"
"unicode/utf8"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/output"
@ -30,6 +31,7 @@ import (
"github.com/fsnotify/fsnotify"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/deps"
@ -791,6 +793,16 @@ func (s *sitesBuilder) GetPageRel(p page.Page, ref string) page.Page {
return p
}
func (s *sitesBuilder) NpmInstall() hexec.Runner {
sc := security.DefaultConfig
sc.Exec.Allow = security.NewWhitelist("npm")
ex := hexec.New(sc)
command, err := ex.New("npm", "install")
s.Assert(err, qt.IsNil)
return command
}
func newTestHelper(cfg config.Provider, fs *hugofs.Fs, t testing.TB) testHelper {
return testHelper{
Cfg: cfg,

36
markup/asciidocext/convert.go
View file

@ -21,10 +21,9 @@ import (
"path/filepath"
"strings"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/htesting"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/markup/asciidocext/asciidocext_config"
"github.com/gohugoio/hugo/markup/converter"
@ -67,7 +66,11 @@ type asciidocConverter struct {
}
func (a *asciidocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
content, toc, err := a.extractTOC(a.getAsciidocContent(ctx.Src, a.ctx))
b, err := a.getAsciidocContent(ctx.Src, a.ctx)
if err != nil {
return nil, err
}
content, toc, err := a.extractTOC(b)
if err != nil {
return nil, err
}
@ -83,20 +86,19 @@ func (a *asciidocConverter) Supports(_ identity.Identity) bool {
// getAsciidocContent calls asciidoctor as an external helper
// to convert AsciiDoc content to HTML.
func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) []byte {
path := getAsciidoctorExecPath()
if path == "" {
func (a *asciidocConverter) getAsciidocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) {
if !hasAsciiDoc() {
a.cfg.Logger.Errorln("asciidoctor not found in $PATH: Please install.\n",
" Leaving AsciiDoc content unrendered.")
return src
return src, nil
}
args := a.parseArgs(ctx)
args = append(args, "-")
a.cfg.Logger.Infoln("Rendering", ctx.DocumentName, "with", path, "using asciidoctor args", args, "...")
a.cfg.Logger.Infoln("Rendering", ctx.DocumentName, " using asciidoctor args", args, "...")
return internal.ExternallyRenderContent(a.cfg, ctx, src, path, args)
return internal.ExternallyRenderContent(a.cfg, ctx, src, asciiDocBinaryName, args)
}
func (a *asciidocConverter) parseArgs(ctx converter.DocumentContext) []string {
@ -195,12 +197,10 @@ func (a *asciidocConverter) appendArg(args []string, option, value, defaultValue
return args
}
func getAsciidoctorExecPath() string {
path, err := safeexec.LookPath("asciidoctor")
if err != nil {
return ""
}
return path
const asciiDocBinaryName = "asciidoctor"
func hasAsciiDoc() bool {
return hexec.InPath(asciiDocBinaryName)
}
// extractTOC extracts the toc from the given src html.
@ -311,8 +311,12 @@ func nodeContent(node *html.Node) string {
// Supports returns whether Asciidoctor is installed on this computer.
func Supports() bool {
hasBin := hasAsciiDoc()
if htesting.SupportsAll() {
if !hasBin {
panic("asciidoctor not installed")
}
return true
}
return getAsciidoctorExecPath() != ""
return hasBin
}

54
markup/asciidocext/convert_test.go
View file

@ -21,8 +21,10 @@ import (
"path/filepath"
"testing"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/markup/converter"
"github.com/gohugoio/hugo/markup/markup_config"
"github.com/gohugoio/hugo/markup/tableofcontents"
@ -280,20 +282,28 @@ func TestAsciidoctorAttributes(t *testing.T) {
c.Assert(args[4], qt.Equals, "--no-header-footer")
}
func getProvider(c *qt.C, mconf markup_config.Config) converter.Provider {
sc := security.DefaultConfig
sc.Exec.Allow = security.NewWhitelist("asciidoctor")
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
Exec: hexec.New(sc),
},
)
c.Assert(err, qt.IsNil)
return p
}
func TestConvert(t *testing.T) {
if !Supports() {
t.Skip("asciidoctor not installed")
}
c := qt.New(t)
mconf := markup_config.Default
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
},
)
c.Assert(err, qt.IsNil)
p := getProvider(c, markup_config.Default)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
@ -308,14 +318,8 @@ func TestTableOfContents(t *testing.T) {
t.Skip("asciidoctor not installed")
}
c := qt.New(t)
mconf := markup_config.Default
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
},
)
c.Assert(err, qt.IsNil)
p := getProvider(c, markup_config.Default)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: macro
@ -390,14 +394,7 @@ func TestTableOfContentsWithCode(t *testing.T) {
t.Skip("asciidoctor not installed")
}
c := qt.New(t)
mconf := markup_config.Default
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
},
)
c.Assert(err, qt.IsNil)
p := getProvider(c, markup_config.Default)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc: auto
@ -433,13 +430,8 @@ func TestTableOfContentsPreserveTOC(t *testing.T) {
c := qt.New(t)
mconf := markup_config.Default
mconf.AsciidocExt.PreserveTOC = true
p, err := Provider.New(
converter.ProviderConfig{
MarkupConfig: mconf,
Logger: loggers.NewErrorLogger(),
},
)
c.Assert(err, qt.IsNil)
p := getProvider(c, mconf)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)
r, err := conv.Convert(converter.RenderContext{Src: []byte(`:toc:

2
markup/converter/converter.go
View file

@ -16,6 +16,7 @@ package converter
import (
"bytes"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/identity"
@ -32,6 +33,7 @@ type ProviderConfig struct {
Cfg config.Provider // Site config
ContentFs afero.Fs
Logger loggers.Logger
Exec *hexec.Exec
Highlight func(code, lang, optsStr string) (string, error)
}

54
markup/internal/external.go
View file

@ -2,42 +2,56 @@ package internal
import (
"bytes"
"fmt"
"strings"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/markup/converter"
)
func ExternallyRenderContent(
cfg converter.ProviderConfig,
ctx converter.DocumentContext,
content []byte, path string, args []string) []byte {
content []byte, binaryName string, args []string) ([]byte, error) {
logger := cfg.Logger
cmd, err := hexec.SafeCommand(path, args...)
if err != nil {
logger.Errorf("%s rendering %s: %v", path, ctx.DocumentName, err)
return nil
if strings.Contains(binaryName, "/") {
panic(fmt.Sprintf("should be no slash in %q", binaryName))
}
cmd.Stdin = bytes.NewReader(content)
argsv := collections.StringSliceToInterfaceSlice(args)
var out, cmderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &cmderr
argsv = append(argsv, hexec.WithStdout(&out))
argsv = append(argsv, hexec.WithStderr(&cmderr))
argsv = append(argsv, hexec.WithStdin(bytes.NewReader(content)))
cmd, err := cfg.Exec.New(binaryName, argsv...)
if err != nil {
return nil, err
}
err = cmd.Run()
// Most external helpers exit w/ non-zero exit code only if severe, i.e.
// halting errors occurred. -> log stderr output regardless of state of err
for _, item := range strings.Split(cmderr.String(), "\n") {
item := strings.TrimSpace(item)
if item != "" {
logger.Errorf("%s: %s", ctx.DocumentName, item)
if err == nil {
logger.Warnf("%s: %s", ctx.DocumentName, item)
} else {
logger.Errorf("%s: %s", ctx.DocumentName, item)
}
}
}
if err != nil {
logger.Errorf("%s rendering %s: %v", path, ctx.DocumentName, err)
logger.Errorf("%s rendering %s: %v", binaryName, ctx.DocumentName, err)
}
return normalizeExternalHelperLineFeeds(out.Bytes())
return normalizeExternalHelperLineFeeds(out.Bytes()), nil
}
// Strips carriage returns from third-party / external processes (useful for Windows)
@ -45,13 +59,13 @@ func normalizeExternalHelperLineFeeds(content []byte) []byte {
return bytes.Replace(content, []byte("\r"), []byte(""), -1)
}
func GetPythonExecPath() string {
path, err := safeexec.LookPath("python")
if err != nil {
path, err = safeexec.LookPath("python.exe")
if err != nil {
return ""
var pythonBinaryCandidates = []string{"python", "python.exe"}
func GetPythonBinaryAndExecPath() (string, string) {
for _, p := range pythonBinaryCandidates {
if pth := hexec.LookPath(p); pth != "" {
return p, pth
}
}
return path
return "", ""
}

36
markup/pandoc/convert.go
View file

@ -15,7 +15,7 @@
package pandoc
import (
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/markup/internal"
@ -44,7 +44,11 @@ type pandocConverter struct {
}
func (c *pandocConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
return converter.Bytes(c.getPandocContent(ctx.Src, c.ctx)), nil
b, err := c.getPandocContent(ctx.Src, c.ctx)
if err != nil {
return nil, err
}
return converter.Bytes(b), nil
}
func (c *pandocConverter) Supports(feature identity.Identity) bool {
@ -52,31 +56,35 @@ func (c *pandocConverter) Supports(feature identity.Identity) bool {
}
// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) []byte {
func (c *pandocConverter) getPandocContent(src []byte, ctx converter.DocumentContext) ([]byte, error) {
logger := c.cfg.Logger
path := getPandocExecPath()
if path == "" {
binaryName := getPandocBinaryName()
if binaryName == "" {
logger.Println("pandoc not found in $PATH: Please install.\n",
" Leaving pandoc content unrendered.")
return src
return src, nil
}
args := []string{"--mathjax"}
return internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
return internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args)
}
func getPandocExecPath() string {
path, err := safeexec.LookPath("pandoc")
if err != nil {
return ""
}
const pandocBinary = "pandoc"
return path
func getPandocBinaryName() string {
if hexec.InPath(pandocBinary) {
return pandocBinary
}
return ""
}
// Supports returns whether Pandoc is installed on this computer.
func Supports() bool {
hasBin := getPandocBinaryName() != ""
if htesting.SupportsAll() {
if !hasBin {
panic("pandoc not installed")
}
return true
}
return getPandocExecPath() != ""
return hasBin
}

6
markup/pandoc/convert_test.go
View file

@ -16,7 +16,9 @@ package pandoc
import (
"testing"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/markup/converter"
@ -28,7 +30,9 @@ func TestConvert(t *testing.T) {
t.Skip("pandoc not installed")
}
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
sc := security.DefaultConfig
sc.Exec.Allow = security.NewWhitelist("pandoc")
p, err := Provider.New(converter.ProviderConfig{Exec: hexec.New(sc), Logger: loggers.NewErrorLogger()})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)

58
markup/rst/convert.go
View file

@ -18,7 +18,7 @@ import (
"bytes"
"runtime"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/identity"
@ -48,7 +48,11 @@ type rstConverter struct {
}
func (c *rstConverter) Convert(ctx converter.RenderContext) (converter.Result, error) {
return converter.Bytes(c.getRstContent(ctx.Src, c.ctx)), nil
b, err := c.getRstContent(ctx.Src, c.ctx)
if err != nil {
return nil, err
}
return converter.Bytes(b), nil
}
func (c *rstConverter) Supports(feature identity.Identity) bool {
@ -57,31 +61,38 @@ func (c *rstConverter) Supports(feature identity.Identity) bool {
// getRstContent calls the Python script rst2html as an external helper
// to convert reStructuredText content to HTML.
func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) []byte {
func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext) ([]byte, error) {
logger := c.cfg.Logger
path := getRstExecPath()
binaryName, binaryPath := getRstBinaryNameAndPath()
if path == "" {
if binaryName == "" {
logger.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
" Leaving reStructuredText content unrendered.")
return src
return src, nil
}
logger.Infoln("Rendering", ctx.DocumentName, "with", path, "...")
logger.Infoln("Rendering", ctx.DocumentName, "with", binaryName, "...")
var result []byte
var err error
// certain *nix based OSs wrap executables in scripted launchers
// invoking binaries on these OSs via python interpreter causes SyntaxError
// invoke directly so that shebangs work as expected
// handle Windows manually because it doesn't do shebangs
if runtime.GOOS == "windows" {
python := internal.GetPythonExecPath()
args := []string{path, "--leave-comments", "--initial-header-level=2"}
result = internal.ExternallyRenderContent(c.cfg, ctx, src, python, args)
pythonBinary, _ := internal.GetPythonBinaryAndExecPath()
args := []string{binaryPath, "--leave-comments", "--initial-header-level=2"}
result, err = internal.ExternallyRenderContent(c.cfg, ctx, src, pythonBinary, args)
} else {
args := []string{"--leave-comments", "--initial-header-level=2"}
result = internal.ExternallyRenderContent(c.cfg, ctx, src, path, args)
result, err = internal.ExternallyRenderContent(c.cfg, ctx, src, binaryName, args)
}
if err != nil {
return nil, err
}
// TODO(bep) check if rst2html has a body only option.
bodyStart := bytes.Index(result, []byte("<body>\n"))
if bodyStart < 0 {
@ -96,24 +107,29 @@ func (c *rstConverter) getRstContent(src []byte, ctx converter.DocumentContext)
}
}
return result[bodyStart+7 : bodyEnd]
return result[bodyStart+7 : bodyEnd], err
}
func getRstExecPath() string {
path, err := safeexec.LookPath("rst2html")
if err != nil {
path, err = safeexec.LookPath("rst2html.py")
if err != nil {
return ""
var rst2Binaries = []string{"rst2html", "rst2html.py"}
func getRstBinaryNameAndPath() (string, string) {
for _, candidate := range rst2Binaries {
if pth := hexec.LookPath(candidate); pth != "" {
return candidate, pth
}
}
return path
return "", ""
}
// Supports returns whether rst is installed on this computer.
// Supports returns whether rst is (or should be) installed on this computer.
func Supports() bool {
name, _ := getRstBinaryNameAndPath()
hasBin := name != ""
if htesting.SupportsAll() {
if !hasBin {
panic("rst not installed")
}
return true
}
return getRstExecPath() != ""
return hasBin
}

11
markup/rst/convert_test.go
View file

@ -16,7 +16,9 @@ package rst
import (
"testing"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/markup/converter"
@ -28,7 +30,14 @@ func TestConvert(t *testing.T) {
t.Skip("rst not installed")
}
c := qt.New(t)
p, err := Provider.New(converter.ProviderConfig{Logger: loggers.NewErrorLogger()})
sc := security.DefaultConfig
sc.Exec.Allow = security.NewWhitelist("rst", "python")
p, err := Provider.New(
converter.ProviderConfig{
Logger: loggers.NewErrorLogger(),
Exec: hexec.New(sc),
})
c.Assert(err, qt.IsNil)
conv, err := p.New(converter.DocumentContext{})
c.Assert(err, qt.IsNil)

29
modules/client.go
View file

@ -28,6 +28,7 @@ import (
"strings"
"time"
"github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/hexec"
hglob "github.com/gohugoio/hugo/hugofs/glob"
@ -79,7 +80,7 @@ func NewClient(cfg ClientConfig) *Client {
goModFilename = n
}
env := os.Environ()
var env []string
mcfg := cfg.ModuleConfig
config.SetEnvVars(&env,
@ -87,12 +88,9 @@ func NewClient(cfg ClientConfig) *Client {
"GO111MODULE", "on",
"GOPROXY", mcfg.Proxy,
"GOPRIVATE", mcfg.Private,
"GONOPROXY", mcfg.NoProxy)
if cfg.CacheDir != "" {
// Module cache stored below $GOPATH/pkg
config.SetEnvVars(&env, "GOPATH", cfg.CacheDir)
}
"GONOPROXY", mcfg.NoProxy,
"GOPATH", cfg.CacheDir,
)
logger := cfg.Logger
if logger == nil {
@ -609,16 +607,19 @@ func (c *Client) runGo(
}
stderr := new(bytes.Buffer)
cmd, err := hexec.SafeCommandContext(ctx, "go", args...)
argsv := collections.StringSliceToInterfaceSlice(args)
argsv = append(argsv, hexec.WithEnviron(c.environ))
argsv = append(argsv, hexec.WithStderr(io.MultiWriter(stderr, os.Stderr)))
argsv = append(argsv, hexec.WithStdout(stdout))
argsv = append(argsv, hexec.WithDir(c.ccfg.WorkingDir))
argsv = append(argsv, hexec.WithContext(ctx))
cmd, err := c.ccfg.Exec.New("go", argsv...)
if err != nil {
return err
}
cmd.Env = c.environ
cmd.Dir = c.ccfg.WorkingDir
cmd.Stdout = stdout
cmd.Stderr = io.MultiWriter(stderr, os.Stderr)
if err := cmd.Run(); err != nil {
if ee, ok := err.(*exec.Error); ok && ee.Err == exec.ErrNotFound {
c.goBinaryStatus = goBinaryStatusNotFound
@ -727,6 +728,8 @@ type ClientConfig struct {
// Eg. "production"
Environment string
Exec *hexec.Exec
CacheDir string // Module cache
ModuleConfig Config
}

4
modules/client_test.go
View file

@ -21,6 +21,8 @@ import (
"sync/atomic"
"testing"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/htesting"
@ -53,7 +55,9 @@ github.com/gohugoio/hugoTestModules1_darwin/modh2_2@v1.4.0 github.com/gohugoio/h
ccfg := ClientConfig{
Fs: hugofs.Os,
WorkingDir: workingDir,
CacheDir: filepath.Join(workingDir, "modcache"),
ThemesDir: themesDir,
Exec: hexec.New(security.DefaultConfig),
}
withConfig(&ccfg)

16
resources/resource_factories/create/create.go
View file

@ -154,6 +154,9 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
// FromRemote expects one or n-parts of a URL to a resource
// If you provide multiple parts they will be joined together to the final URL.
func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) {
if err := c.validateFromRemoteArgs(uri, options); err != nil {
return nil, err
}
rURL, err := url.Parse(uri)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
@ -262,6 +265,19 @@ func (c *Client) FromRemote(uri string, options map[string]interface{}) (resourc
}
func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error {
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
return err
}
if method, ok := options["method"].(string); ok {
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil {
return err
}
}
return nil
}
func addDefaultHeaders(req *http.Request, accepts ...string) {
for _, accept := range accepts {
if !hasHeaderValue(req.Header, "Accept", accept) {

5
resources/resource_spec.go
View file

@ -26,6 +26,7 @@ import (
"github.com/gohugoio/hugo/resources/jsconfig"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/identity"
@ -51,6 +52,7 @@ func NewSpec(
incr identity.Incrementer,
logger loggers.Logger,
errorHandler herrors.ErrorSender,
execHelper *hexec.Exec,
outputFormats output.Formats,
mimeTypes media.Types) (*Spec, error) {
imgConfig, err := images.DecodeConfig(s.Cfg.GetStringMap("imaging"))
@ -81,6 +83,7 @@ func NewSpec(
Logger: logger,
ErrorSender: errorHandler,
imaging: imaging,
ExecHelper: execHelper,
incr: incr,
MediaTypes: mimeTypes,
OutputFormats: outputFormats,
@ -120,6 +123,8 @@ type Spec struct {
// Holds default filter settings etc.
imaging *images.ImageProcessor
ExecHelper *hexec.Exec
incr identity.Incrementer
imageCache *imageCache
ResourceCache *ResourceCache

46
resources/resource_transformers/babel/babel.go
View file

@ -23,7 +23,6 @@ import (
"regexp"
"strconv"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
@ -59,8 +58,8 @@ func DecodeOptions(m map[string]interface{}) (opts Options, err error) {
return
}
func (opts Options) toArgs() []string {
var args []string
func (opts Options) toArgs() []interface{} {
var args []interface{}
// external is not a known constant on the babel command line
// .sourceMaps must be a boolean, "inline", "both", or undefined
@ -115,21 +114,12 @@ func (t *babelTransformation) Key() internal.ResourceTransformationKey {
// npm install -g @babel/preset-env
// Instead of installing globally, you can also install everything as a dev-dependency (--save-dev instead of -g)
func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx) error {
const localBabelPath = "node_modules/.bin/"
const binaryName = "babel"
// Try first in the project's node_modules.
csiBinPath := filepath.Join(t.rs.WorkingDir, localBabelPath, binaryName)
ex := t.rs.ExecHelper
binary := csiBinPath
if _, err := safeexec.LookPath(binary); err != nil {
// Try PATH
binary = binaryName
if _, err := safeexec.LookPath(binary); err != nil {
// This may be on a CI server etc. Will fall back to pre-built assets.
return herrors.ErrFeatureNotAvailable
}
if err := ex.Sec().CheckAllowedExec(binaryName); err != nil {
return err
}
var configFile string
@ -157,11 +147,11 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx
ctx.ReplaceOutPathExtension(".js")
var cmdArgs []string
var cmdArgs []interface{}
if configFile != "" {
logger.Infoln("babel: use config file", configFile)
cmdArgs = []string{"--config-file", configFile}
cmdArgs = []interface{}{"--config-file", configFile}
}
if optArgs := t.options.toArgs(); len(optArgs) > 0 {
@ -178,18 +168,27 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx
}
cmdArgs = append(cmdArgs, "--out-file="+compileOutput.Name())
stderr := io.MultiWriter(infoW, &errBuf)
cmdArgs = append(cmdArgs, hexec.WithStderr(stderr))
cmdArgs = append(cmdArgs, hexec.WithStdout(stderr))
cmdArgs = append(cmdArgs, hexec.WithEnviron(hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs)))
defer os.Remove(compileOutput.Name())
cmd, err := hexec.SafeCommand(binary, cmdArgs...)
// ARGA [--no-install babel --config-file /private/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/hugo-test-babel812882892/babel.config.js --source-maps --filename=js/main2.js --out-file=/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/compileOut-2237820197.js]
// [--no-install babel --config-file /private/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/hugo-test-babel332846848/babel.config.js --filename=js/main.js --out-file=/var/folders/_g/j3j21hts4fn7__h04w2x8gb40000gn/T/compileOut-1451390834.js 0x10304ee60 0x10304ed60 0x10304f060]
cmd, err := ex.Npx(binaryName, cmdArgs...)
if err != nil {
if hexec.IsNotFound(err) {
// This may be on a CI server etc. Will fall back to pre-built assets.
return herrors.ErrFeatureNotAvailable
}
return err
}
cmd.Stderr = io.MultiWriter(infoW, &errBuf)
cmd.Stdout = cmd.Stderr
cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
@ -201,6 +200,9 @@ func (t *babelTransformation) Transform(ctx *resources.ResourceTransformationCtx
err = cmd.Run()
if err != nil {
if hexec.IsNotFound(err) {
return herrors.ErrFeatureNotAvailable
}
return errors.Wrap(err, errBuf.String())
}

2
resources/resource_transformers/htesting/testhelpers.go
View file

@ -51,7 +51,7 @@ func NewTestResourceSpec() (*resources.Spec, error) {
return nil, err
}
spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes)
spec, err := resources.NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes)
return spec, err
}

47
resources/resource_transformers/postcss/postcss.go
View file

@ -25,8 +25,7 @@ import (
"strconv"
"strings"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/common/collections"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/hugo"
@ -142,22 +141,9 @@ func (t *postcssTransformation) Key() internal.ResourceTransformationKey {
// npm install -g postcss-cli
// npm install -g autoprefixer
func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationCtx) error {
const localPostCSSPath = "node_modules/.bin/"
const binaryName = "postcss"
// Try first in the project's node_modules.
csiBinPath := filepath.Join(t.rs.WorkingDir, localPostCSSPath, binaryName)
binary := csiBinPath
if _, err := safeexec.LookPath(binary); err != nil {
// Try PATH
binary = binaryName
if _, err := safeexec.LookPath(binary); err != nil {
// This may be on a CI server etc. Will fall back to pre-built assets.
return herrors.ErrFeatureNotAvailable
}
}
ex := t.rs.ExecHelper
var configFile string
logger := t.rs.Logger
@ -179,29 +165,33 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
}
}
var cmdArgs []string
var cmdArgs []interface{}
if configFile != "" {
logger.Infoln("postcss: use config file", configFile)
cmdArgs = []string{"--config", configFile}
cmdArgs = []interface{}{"--config", configFile}
}
if optArgs := t.options.toArgs(); len(optArgs) > 0 {
cmdArgs = append(cmdArgs, optArgs...)
}
cmd, err := hexec.SafeCommand(binary, cmdArgs...)
if err != nil {
return err
cmdArgs = append(cmdArgs, collections.StringSliceToInterfaceSlice(optArgs)...)
}
var errBuf bytes.Buffer
infoW := loggers.LoggerToWriterWithPrefix(logger.Info(), "postcss")
cmd.Stdout = ctx.To
cmd.Stderr = io.MultiWriter(infoW, &errBuf)
stderr := io.MultiWriter(infoW, &errBuf)
cmdArgs = append(cmdArgs, hexec.WithStderr(stderr))
cmdArgs = append(cmdArgs, hexec.WithStdout(ctx.To))
cmdArgs = append(cmdArgs, hexec.WithEnviron(hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs)))
cmd.Env = hugo.GetExecEnviron(t.rs.WorkingDir, t.rs.Cfg, t.rs.BaseFs.Assets.Fs)
cmd, err := ex.Npx(binaryName, cmdArgs...)
if err != nil {
if hexec.IsNotFound(err) {
// This may be on a CI server etc. Will fall back to pre-built assets.
return herrors.ErrFeatureNotAvailable
}
return err
}
stdin, err := cmd.StdinPipe()
if err != nil {
@ -231,6 +221,9 @@ func (t *postcssTransformation) Transform(ctx *resources.ResourceTransformationC
err = cmd.Run()
if err != nil {
if hexec.IsNotFound(err) {
return herrors.ErrFeatureNotAvailable
}
return imp.toFileError(errBuf.String())
}

19
resources/resource_transformers/tocss/dartsass/client.go
View file

@ -33,8 +33,13 @@ const transformationName = "tocss-dart"
func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error) {
if !Supports() {
return &Client{dartSassNoAvailable: true}, nil
return &Client{dartSassNotAvailable: true}, nil
}
if err := rs.ExecHelper.Sec().CheckAllowedExec(dartSassEmbeddedBinaryName); err != nil {
return nil, err
}
transpiler, err := godartsass.Start(godartsass.Options{})
if err != nil {
return nil, err
@ -43,15 +48,15 @@ func New(fs *filesystems.SourceFilesystem, rs *resources.Spec) (*Client, error)
}
type Client struct {
dartSassNoAvailable bool
rs *resources.Spec
sfs *filesystems.SourceFilesystem
workFs afero.Fs
transpiler *godartsass.Transpiler
dartSassNotAvailable bool
rs *resources.Spec
sfs *filesystems.SourceFilesystem
workFs afero.Fs
transpiler *godartsass.Transpiler
}
func (c *Client) ToCSS(res resources.ResourceTransformer, args map[string]interface{}) (resource.Resource, error) {
if c.dartSassNoAvailable {
if c.dartSassNotAvailable {
return res.Transform(resources.NewFeatureNotAvailableTransformer(transformationName, args))
}
return res.Transform(&transform{c: c, optsm: args})

13
resources/resource_transformers/tocss/dartsass/transform.go
View file

@ -21,9 +21,8 @@ import (
"path/filepath"
"strings"
"github.com/cli/safeexec"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/media"
@ -38,16 +37,18 @@ import (
"github.com/bep/godartsass"
)
// See https://github.com/sass/dart-sass-embedded/issues/24
const stdinPlaceholder = "HUGOSTDIN"
const (
// See https://github.com/sass/dart-sass-embedded/issues/24
stdinPlaceholder = "HUGOSTDIN"
dartSassEmbeddedBinaryName = "dart-sass-embedded"
)
// Supports returns whether dart-sass-embedded is found in $PATH.
func Supports() bool {
if htesting.SupportsAll() {
return true
}
p, err := safeexec.LookPath("dart-sass-embedded")
return err == nil && p != ""
return hexec.InPath(dartSassEmbeddedBinaryName)
}
type transform struct {

4
resources/testhelpers_test.go
View file

@ -87,7 +87,7 @@ func newTestResourceSpec(desc specDescriptor) *Spec {
filecaches, err := filecache.NewCaches(s)
c.Assert(err, qt.IsNil)
spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes)
spec, err := NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes)
c.Assert(err, qt.IsNil)
return spec
}
@ -126,7 +126,7 @@ func newTestResourceOsFs(c *qt.C) (*Spec, string) {
filecaches, err := filecache.NewCaches(s)
c.Assert(err, qt.IsNil)
spec, err := NewSpec(s, filecaches, nil, nil, nil, output.DefaultFormats, media.DefaultTypes)
spec, err := NewSpec(s, filecaches, nil, nil, nil, nil, output.DefaultFormats, media.DefaultTypes)
c.Assert(err, qt.IsNil)
return spec, workDir

3
tpl/collections/collections_test.go
View file

@ -32,7 +32,6 @@ import (
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/langs"
"github.com/spf13/afero"
)
type tstNoStringer struct{}
@ -973,7 +972,7 @@ func ToTstXIs(slice interface{}) []TstXI {
func newDeps(cfg config.Provider) *deps.Deps {
l := langs.NewLanguage("en", cfg)
l.Set("i18nDir", "i18n")
cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs())
cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil)
if err != nil {
panic(err)
}

7
tpl/data/data.go
View file

@ -24,6 +24,7 @@ import (
"strings"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/common/types"
@ -88,6 +89,9 @@ func (ns *Namespace) GetCSV(sep string, args ...interface{}) (d [][]string, err
err = ns.getResource(cache, unmarshal, req)
if err != nil {
if security.IsAccessDenied(err) {
return nil, err
}
ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetCSV, "Failed to get CSV resource %q: %s", url, err)
return nil, nil
}
@ -121,6 +125,9 @@ func (ns *Namespace) GetJSON(args ...interface{}) (interface{}, error) {
err = ns.getResource(cache, unmarshal, req)
if err != nil {
if security.IsAccessDenied(err) {
return nil, err
}
ns.deps.Log.(loggers.IgnorableLogger).Errorsf(constants.ErrRemoteGetJSON, "Failed to get JSON resource %q: %s", url, err)
return nil, nil
}

7
tpl/data/resources.go
View file

@ -38,6 +38,13 @@ var (
// getRemote loads the content of a remote file. This method is thread safe.
func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
url := req.URL.String()
if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPURL(url); err != nil {
return err
}
if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPMethod("GET"); err != nil {
return err
}
var headers bytes.Buffer
req.Header.Write(&headers)
id := helpers.MD5String(url + headers.String())

7
tpl/data/resources_test.go
View file

@ -22,12 +22,14 @@ import (
"testing"
"time"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/modules"
"github.com/gohugoio/hugo/helpers"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/deps"
@ -193,8 +195,10 @@ func newDeps(cfg config.Provider) *deps.Deps {
}
cfg.Set("allModules", modules.Modules{mod})
ex := hexec.New(security.DefaultConfig)
logger := loggers.NewIgnorableLogger(loggers.NewErrorLogger(), "none")
cs, err := helpers.NewContentSpec(cfg, logger, afero.NewMemMapFs())
cs, err := helpers.NewContentSpec(cfg, logger, afero.NewMemMapFs(), ex)
if err != nil {
panic(err)
}
@ -215,6 +219,7 @@ func newDeps(cfg config.Provider) *deps.Deps {
Cfg: cfg,
Fs: fs,
FileCaches: fileCaches,
ExecHelper: ex,
ContentSpec: cs,
Log: logger,
LogDistinct: helpers.NewDistinctLogger(logger),

4
tpl/os/os.go
View file

@ -56,6 +56,10 @@ func (ns *Namespace) Getenv(key interface{}) (string, error) {
return "", nil
}
if err = ns.deps.ExecHelper.Sec().CheckAllowedGetEnv(skey); err != nil {
return "", err
}
return _os.Getenv(skey), nil
}

2
tpl/transform/transform_test.go
View file

@ -241,7 +241,7 @@ func newDeps(cfg config.Provider) *deps.Deps {
l := langs.NewLanguage("en", cfg)
cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs())
cs, err := helpers.NewContentSpec(l, loggers.NewErrorLogger(), afero.NewMemMapFs(), nil)
if err != nil {
panic(err)
}