tpl/collections: Add collections.Append

Before this commit you would typically use `.Scratch.Add` to manually create slices in a loop.

With variable overwrite in Go 1.11, we can do better. This commit adds the `append` template func.

A made-up example:

```bash
{{ $p1 := index .Site.RegularPages 0 }}{{ $p2 := index .Site.RegularPages 1 }}
{{ $pages := slice }}
{{ if true }}
  {{ $pages = $pages | append $p2 $p1 }}
{{ end }}
```

Note that with 2 slices as arguments, the two examples below will give the same result:

```bash
{{ $s1 := slice "a" "b" | append (slice "c" "d") }}
{{ $s2 := slice "a" "b" | append "c" "d" }}
```

Both of the above will give `[]string{a, b, c, d}`.

This commit also improves the type handling in the `slice` template function. Now `slice "a" "b"` will give a `[]string` slice. The old behaviour was to return a `[]interface{}`.

Fixes #5190

common/collections/collections.go

@@ -24,5 +24,5 @@ type Grouper interface {
 // in collections.Slice template func to get types such as Pages, PageGroups etc.
 // instead of the less useful []interface{}.
 type Slicer interface {
-	Slice(items []interface{}) (interface{}, error)
+	Slice(items interface{}) (interface{}, error)
 }

go.sum

@@ -65,6 +65,7 @@ github.com/magefile/mage v1.4.0 h1:RI7B1CgnPAuu2O9lWszwya61RLmfL0KCdo+QyyI/Bhk=
 github.com/magefile/mage v1.4.0/go.mod h1:IUDi13rsHje59lecXokTfGX0QIzO45uVPlXnJYsXepA=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6 h1:LZhVjIISSbj8qLf2qDPP0D8z0uvOWAW5C85ly5mJW6c=
 github.com/markbates/inflect v0.0.0-20171215194931-a12c3aec81a6/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=

hugolib/collections.go

@@ -16,6 +16,8 @@ package hugolib
 import (
 	"fmt"
 
+	"github.com/gohugoio/hugo/resource"
+
 	"github.com/gohugoio/hugo/common/collections"
 )
 
@@ -24,6 +26,7 @@ var (
 	_ collections.Slicer          = (*Page)(nil)
 	_ collections.Slicer          = PageGroup{}
 	_ collections.Slicer          = WeightedPage{}
+	_ resource.ResourcesConverter = Pages{}
 )
 
 // collections.Slicer implementations below. We keep these bridge implementations
@@ -32,13 +35,17 @@ var (
 
 // Slice is not meant to be used externally. It's a bridge function
 // for the template functions. See collections.Slice.
-func (p *Page) Slice(items []interface{}) (interface{}, error) {
+func (p *Page) Slice(items interface{}) (interface{}, error) {
 	return toPages(items)
 }
 
 // Slice is not meant to be used externally. It's a bridge function
 // for the template functions. See collections.Slice.
-func (p PageGroup) Slice(items []interface{}) (interface{}, error) {
+func (p PageGroup) Slice(in interface{}) (interface{}, error) {
+	switch items := in.(type) {
+	case PageGroup:
+		return items, nil
+	case []interface{}:
 		groups := make(PagesGroup, len(items))
 		for i, v := range items {
 			g, ok := v.(PageGroup)
@@ -48,11 +55,18 @@ func (p PageGroup) Slice(items []interface{}) (interface{}, error) {
 			groups[i] = g
 		}
 		return groups, nil
+	default:
+		return nil, fmt.Errorf("invalid slice type %T", items)
+	}
 }
 
 // Slice is not meant to be used externally. It's a bridge function
 // for the template functions. See collections.Slice.
-func (p WeightedPage) Slice(items []interface{}) (interface{}, error) {
+func (p WeightedPage) Slice(in interface{}) (interface{}, error) {
+	switch items := in.(type) {
+	case WeightedPages:
+		return items, nil
+	case []interface{}:
 		weighted := make(WeightedPages, len(items))
 		for i, v := range items {
 			g, ok := v.(WeightedPage)
@@ -62,6 +76,9 @@ func (p WeightedPage) Slice(items []interface{}) (interface{}, error) {
 			weighted[i] = g
 		}
 		return weighted, nil
+	default:
+		return nil, fmt.Errorf("invalid slice type %T", items)
+	}
 }
 
 // collections.Grouper  implementations below
@@ -76,3 +93,12 @@ func (p *Page) Group(key interface{}, in interface{}) (interface{}, error) {
 	}
 	return PageGroup{Key: key, Pages: pages}, nil
 }
+
+// ToResources wraps resource.ResourcesConverter
+func (pages Pages) ToResources() resource.Resources {
+	r := make(resource.Resources, len(pages))
+	for i, p := range pages {
+		r[i] = p
+	}
+	return r
+}

hugolib/collections_test.go

@@ -86,3 +86,57 @@ tags_weight: %d
 		"pageGroups:2:hugolib.PagesGroup:Page(/page1.md)/Page(/page2.md)",
 		`weightedPages:2::hugolib.WeightedPages:[WeightedPage(10,"Page") WeightedPage(20,"Page")]`)
 }
+
+func TestAppendFunc(t *testing.T) {
+	assert := require.New(t)
+
+	pageContent := `
+---
+title: "Page"
+tags: ["blue", "green"]
+tags_weight: %d
+---
+
+`
+	b := newTestSitesBuilder(t)
+	b.WithSimpleConfigFile().
+		WithContent("page1.md", fmt.Sprintf(pageContent, 10), "page2.md", fmt.Sprintf(pageContent, 20)).
+		WithTemplatesAdded("index.html", `
+
+{{ $p1 := index .Site.RegularPages 0 }}{{ $p2 := index .Site.RegularPages 1 }}
+
+{{ $pages := slice }}
+
+{{ if true }}
+	{{ $pages = $pages | append $p2 $p1 }}
+{{ end }}
+{{ $appendPages := .Site.Pages | append .Site.RegularPages }}
+{{ $appendStrings := slice "a" "b" | append "c" "d" "e" }}
+{{ $appendStringsSlice := slice "a" "b" "c" | append (slice "c" "d") }}
+
+{{ printf "pages:%d:%T:%v/%v" (len $pages) $pages (index $pages 0) (index $pages 1)  }}
+{{ printf "appendPages:%d:%T:%v/%v" (len $appendPages) $appendPages (index $appendPages 0).Kind (index $appendPages 8).Kind  }}
+{{ printf "appendStrings:%T:%v"  $appendStrings $appendStrings  }}
+{{ printf "appendStringsSlice:%T:%v"  $appendStringsSlice $appendStringsSlice }}
+
+{{/* add some slightly related funcs to check what types we get */}}
+{{ $u :=  $appendStrings | union $appendStringsSlice }}
+{{ $i :=  $appendStrings | intersect $appendStringsSlice }}
+{{ printf "union:%T:%v" $u $u  }}
+{{ printf "intersect:%T:%v" $i $i }}
+
+`)
+	b.CreateSites().Build(BuildCfg{})
+
+	assert.Equal(1, len(b.H.Sites))
+	require.Len(t, b.H.Sites[0].RegularPages, 2)
+
+	b.AssertFileContent("public/index.html",
+		"pages:2:hugolib.Pages:Page(/page2.md)/Page(/page1.md)",
+		"appendPages:9:hugolib.Pages:home/page",
+		"appendStrings:[]string:[a b c d e]",
+		"appendStringsSlice:[]string:[a b c c d]",
+		"union:[]string:[a b c d e]",
+		"intersect:[]string:[a b c d]",
+	)
+}

resource/resource.go

@@ -28,6 +28,7 @@ import (
 	"github.com/gohugoio/hugo/output"
 	"github.com/gohugoio/hugo/tpl"
 
+	"github.com/gohugoio/hugo/common/collections"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/common/loggers"
 
@@ -49,6 +50,7 @@ var (
 	_ Cloner                  = (*genericResource)(nil)
 	_ ResourcesLanguageMerger = (*Resources)(nil)
 	_ permalinker             = (*genericResource)(nil)
+	_ collections.Slicer      = (*genericResource)(nil)
 )
 
 var noData = make(map[string]interface{})
@@ -150,6 +152,11 @@ type ReadSeekCloserResource interface {
 // I.e. both pages and images etc.
 type Resources []Resource
 
+// ResourcesConverter converts a given slice of Resource objects to Resources.
+type ResourcesConverter interface {
+	ToResources() Resources
+}
+
 // ByType returns resources of a given resource type (ie. "image").
 func (r Resources) ByType(tp string) Resources {
 	var filtered Resources
@@ -550,6 +557,7 @@ func (l *publishOnce) publish(s Source) error {
 
 // genericResource represents a generic linkable resource.
 type genericResource struct {
+	commonResource
 	resourcePathDescriptor
 
 	title  string
@@ -586,6 +594,9 @@ type genericResource struct {
 	*publishOnce
 }
 
+type commonResource struct {
+}
+
 func (l *genericResource) Data() interface{} {
 	return noData
 }
@@ -621,6 +632,27 @@ func (l genericResource) WithNewBase(base string) Resource {
 	return &l
 }
 
+// Slice is not meant to be used externally. It's a bridge function
+// for the template functions. See collections.Slice.
+func (commonResource) Slice(in interface{}) (interface{}, error) {
+	switch items := in.(type) {
+	case Resources:
+		return items, nil
+	case []interface{}:
+		groups := make(Resources, len(items))
+		for i, v := range items {
+			g, ok := v.(Resource)
+			if !ok {
+				return nil, fmt.Errorf("type %T is not a Resource", v)
+			}
+			groups[i] = g
+		}
+		return groups, nil
+	default:
+		return nil, fmt.Errorf("invalid slice type %T", items)
+	}
+}
+
 func (l *genericResource) initHash() error {
 	var err error
 	l.hashInit.Do(func() {

resource/transform.go

@@ -19,6 +19,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/collections"
 	"github.com/gohugoio/hugo/common/errors"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/helpers"
@@ -37,6 +38,7 @@ import (
 var (
 	_ ContentResource        = (*transformedResource)(nil)
 	_ ReadSeekCloserResource = (*transformedResource)(nil)
+	_ collections.Slicer     = (*transformedResource)(nil)
 )
 
 func (s *Spec) Transform(r Resource, t ResourceTransformation) (Resource, error) {
@@ -166,6 +168,8 @@ type transformedResourceMetadata struct {
 }
 
 type transformedResource struct {
+	commonResource
+
 	cache *ResourceCache
 
 	// This is the filename inside resources/_gen/assets

tpl/collections/append.go

@@ -0,0 +1,74 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+)
+
+// Append appends the arguments up to the last one to the slice in the last argument.
+// This construct allows template constructs like this:
+//     {{ $pages = $pages | append $p2 $p1 }}
+// Note that with 2 arguments where both are slices of the same type,
+// the first slice will be appended to the second:
+//     {{ $pages = $pages | append .Site.RegularPages }}
+func (ns *Namespace) Append(args ...interface{}) (interface{}, error) {
+	if len(args) < 2 {
+		return nil, errors.New("need at least 2 arguments to append")
+	}
+
+	to := args[len(args)-1]
+	from := args[:len(args)-1]
+
+	tov, toIsNil := indirect(reflect.ValueOf(to))
+
+	toIsNil = toIsNil || to == nil
+	var tot reflect.Type
+
+	if !toIsNil {
+		if tov.Kind() != reflect.Slice {
+			return nil, fmt.Errorf("expected a slice, got %T", to)
+		}
+
+		tot = tov.Type().Elem()
+		toIsNil = tov.Len() == 0
+
+		if len(from) == 1 {
+			// If we get []string []string, we append the from slice to to
+			fromv := reflect.ValueOf(from[0])
+			if fromv.Kind() == reflect.Slice {
+				fromt := reflect.TypeOf(from[0]).Elem()
+				if tot == fromt {
+					return reflect.AppendSlice(tov, fromv).Interface(), nil
+				}
+			}
+		}
+	}
+
+	if toIsNil {
+		return ns.Slice(from...), nil
+	}
+
+	for _, f := range from {
+		fv := reflect.ValueOf(f)
+		if tot != fv.Type() {
+			return nil, fmt.Errorf("append element type mismatch: expected %v, got %v", tot, fv.Type())
+		}
+		tov = reflect.Append(tov, fv)
+	}
+
+	return tov.Interface(), nil
+}

tpl/collections/append_test.go

@@ -0,0 +1,77 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collections
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/alecthomas/assert"
+	"github.com/gohugoio/hugo/deps"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAppend(t *testing.T) {
+	t.Parallel()
+
+	ns := New(&deps.Deps{})
+
+	for i, test := range []struct {
+		start    interface{}
+		addend   []interface{}
+		expected interface{}
+	}{
+		{[]string{"a", "b"}, []interface{}{"c"}, []string{"a", "b", "c"}},
+		{[]string{"a", "b"}, []interface{}{"c", "d", "e"}, []string{"a", "b", "c", "d", "e"}},
+		{[]string{"a", "b"}, []interface{}{[]string{"c", "d", "e"}}, []string{"a", "b", "c", "d", "e"}},
+		{nil, []interface{}{"a", "b"}, []string{"a", "b"}},
+		{nil, []interface{}{nil}, []interface{}{nil}},
+		{tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}},
+			[]interface{}{&tstSlicer{"c"}},
+			tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}, &tstSlicer{"c"}}},
+		{&tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}},
+			[]interface{}{&tstSlicer{"c"}},
+			tstSlicers{&tstSlicer{"a"},
+				&tstSlicer{"b"},
+				&tstSlicer{"c"}}},
+		// Errors
+		{"", []interface{}{[]string{"a", "b"}}, false},
+		{[]string{"a", "b"}, []interface{}{}, false},
+		// No string concatenation.
+		{"ab",
+			[]interface{}{"c"},
+			false},
+	} {
+
+		errMsg := fmt.Sprintf("[%d]", i)
+
+		args := append(test.addend, test.start)
+
+		result, err := ns.Append(args...)
+
+		if b, ok := test.expected.(bool); ok && !b {
+			require.Error(t, err, errMsg)
+			continue
+		}
+
+		require.NoError(t, err, errMsg)
+
+		if !reflect.DeepEqual(test.expected, result) {
+			t.Fatalf("%s got\n%T: %v\nexpected\n%T: %v", errMsg, result, result, test.expected, test.expected)
+		}
+	}
+
+	assert.Len(t, ns.Slice(), 0)
+}

tpl/collections/collections.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -520,10 +520,11 @@ func (ns *Namespace) Slice(args ...interface{}) interface{} {
 	}
 
 	first := args[0]
-	allTheSame := true
-	if len(args) > 1 {
-		// This can be a mix of types.
 	firstType := reflect.TypeOf(first)
+
+	allTheSame := firstType != nil
+	if allTheSame && len(args) > 1 {
+		// This can be a mix of types.
 		for i := 1; i < len(args); i++ {
 			if firstType != reflect.TypeOf(args[i]) {
 				allTheSame = false
@@ -538,6 +539,12 @@ func (ns *Namespace) Slice(args ...interface{}) interface{} {
 			if err == nil {
 				return v
 			}
+		} else {
+			slice := reflect.MakeSlice(reflect.SliceOf(firstType), len(args), len(args))
+			for i, arg := range args {
+				slice.Index(i).Set(reflect.ValueOf(arg))
+			}
+			return slice.Interface()
 		}
 	}
 

tpl/collections/collections_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 The Hugo Authors. All rights reserved.
+// Copyright 2018 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -647,7 +647,8 @@ type tstSlicer struct {
 	name string
 }
 
-func (p *tstSlicer) Slice(items []interface{}) (interface{}, error) {
+func (p *tstSlicer) Slice(in interface{}) (interface{}, error) {
+	items := in.([]interface{})
 	result := make(tstSlicers, len(items))
 	for i, v := range items {
 		result[i] = v.(*tstSlicer)
@@ -666,13 +667,13 @@ func TestSlice(t *testing.T) {
 		args     []interface{}
 		expected interface{}
 	}{
-		{[]interface{}{"a", "b"}, []interface{}{"a", "b"}},
+		{[]interface{}{"a", "b"}, []string{"a", "b"}},
 		{[]interface{}{&tstSlicer{"a"}, &tstSlicer{"b"}}, tstSlicers{&tstSlicer{"a"}, &tstSlicer{"b"}}},
 		{[]interface{}{&tstSlicer{"a"}, "b"}, []interface{}{&tstSlicer{"a"}, "b"}},
 		{[]interface{}{}, []interface{}{}},
 		{[]interface{}{nil}, []interface{}{nil}},
 		{[]interface{}{5, "b"}, []interface{}{5, "b"}},
-		{[]interface{}{tstNoStringer{}}, []interface{}{tstNoStringer{}}},
+		{[]interface{}{tstNoStringer{}}, []tstNoStringer{tstNoStringer{}}},
 	} {
 		errMsg := fmt.Sprintf("[%d] %v", i, test.args)
 

tpl/collections/init.go

@@ -138,6 +138,11 @@ func init() {
 			[][2]string{},
 		)
 
+		ns.AddMethodMapping(ctx.Append,
+			[]string{"append"},
+			[][2]string{},
+		)
+
 		ns.AddMethodMapping(ctx.Group,
 			[]string{"group"},
 			[][2]string{},

tpl/path/path.go

@@ -121,6 +121,10 @@ func (ns *Namespace) Join(elements ...interface{}) (string, error) {
 	var pathElements []string
 	for _, elem := range elements {
 		switch v := elem.(type) {
+		case []string:
+			for _, e := range v {
+				pathElements = append(pathElements, filepath.ToSlash(e))
+			}
 		case []interface{}:
 			for _, e := range v {
 				elemStr, err := cast.ToStringE(e)

tpl/resources/resources.go

@@ -88,22 +88,11 @@ func (ns *Namespace) Concat(targetPathIn interface{}, r interface{}) (resource.R
 	var rr resource.Resources
 
 	switch v := r.(type) {
-	// This is what we get from the slice func.
-	case []interface{}:
-		rr = make([]resource.Resource, len(v))
-		for i := 0; i < len(v); i++ {
-			rv, ok := v[i].(resource.Resource)
-			if !ok {
-				return nil, fmt.Errorf("cannot concat type %T", v[i])
-			}
-			rr[i] = rv
-		}
-	// This is what we get from .Resources.Match etc.
 	case resource.Resources:
 		rr = v
+	case resource.ResourcesConverter:
+		rr = v.ToResources()
 	default:
-		// We may support Page collections at one point, but we need to think about ...
-		// what to acutually concatenate.
 		return nil, fmt.Errorf("slice %T not supported in concat", r)
 	}