hugo/tpl/collections/where.go

516 lines
14 KiB
Go
Raw Permalink Normal View History

// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package collections
import (
"errors"
"fmt"
"reflect"
"strings"
Cache reflect.MethodByName The isolated benchmark for the function is obviously much faster: ```bash name old time/op new time/op delta GetMethodByName-10 1.21µs ± 7% 0.23µs ± 5% -81.42% (p=0.029 n=4+4) name old alloc/op new alloc/op delta GetMethodByName-10 680B ± 0% 0B -100.00% (p=0.029 n=4+4) name old allocs/op new allocs/op delta GetMethodByName-10 20.0 ± 0% 0.0 -100.00% (p=0.029 n=4+4) ``` But more pleasing is the overall performance looking at the site benchmarks: ```bash name old time/op new time/op delta SiteNew/Regular_Bundle_with_image-10 6.25ms ± 2% 6.10ms ± 2% ~ (p=0.057 n=4+4) SiteNew/Regular_Bundle_with_JSON_file-10 6.30ms ± 2% 5.66ms ±11% ~ (p=0.057 n=4+4) SiteNew/Regular_Tags_and_categories-10 22.2ms ± 2% 17.4ms ± 1% -21.88% (p=0.029 n=4+4) SiteNew/Regular_Canonify_URLs-10 108ms ± 0% 107ms ± 0% -1.20% (p=0.029 n=4+4) SiteNew/Regular_Deep_content_tree-10 36.1ms ± 1% 33.8ms ± 1% -6.44% (p=0.029 n=4+4) SiteNew/Regular_TOML_front_matter-10 24.9ms ± 1% 22.6ms ± 1% -9.30% (p=0.029 n=4+4) SiteNew/Regular_Many_HTML_templates-10 17.9ms ± 1% 16.7ms ± 1% -6.43% (p=0.029 n=4+4) SiteNew/Regular_Page_collections-10 23.3ms ± 1% 22.0ms ± 0% -5.58% (p=0.029 n=4+4) SiteNew/Regular_List_terms-10 8.00ms ± 1% 7.63ms ± 0% -4.62% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Regular_Bundle_with_image-10 2.10MB ± 0% 2.07MB ± 0% -1.46% (p=0.029 n=4+4) SiteNew/Regular_Bundle_with_JSON_file-10 1.88MB ± 0% 1.85MB ± 0% -1.76% (p=0.029 n=4+4) SiteNew/Regular_Tags_and_categories-10 13.5MB ± 0% 11.6MB ± 0% -13.99% (p=0.029 n=4+4) SiteNew/Regular_Canonify_URLs-10 96.1MB ± 0% 95.8MB ± 0% -0.40% (p=0.029 n=4+4) SiteNew/Regular_Deep_content_tree-10 28.4MB ± 0% 27.3MB ± 0% -3.83% (p=0.029 n=4+4) SiteNew/Regular_TOML_front_matter-10 16.9MB ± 0% 15.1MB ± 0% -10.58% (p=0.029 n=4+4) SiteNew/Regular_Many_HTML_templates-10 8.98MB ± 0% 8.44MB ± 0% -6.04% (p=0.029 n=4+4) SiteNew/Regular_Page_collections-10 17.1MB ± 0% 16.5MB ± 0% -3.91% (p=0.029 n=4+4) SiteNew/Regular_List_terms-10 3.92MB ± 0% 3.72MB ± 0% -5.03% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Regular_Bundle_with_image-10 25.8k ± 0% 24.9k ± 0% -3.49% (p=0.029 n=4+4) SiteNew/Regular_Bundle_with_JSON_file-10 25.8k ± 0% 24.9k ± 0% -3.49% (p=0.029 n=4+4) SiteNew/Regular_Tags_and_categories-10 288k ± 0% 233k ± 0% -18.90% (p=0.029 n=4+4) SiteNew/Regular_Canonify_URLs-10 375k ± 0% 364k ± 0% -2.80% (p=0.029 n=4+4) SiteNew/Regular_Deep_content_tree-10 314k ± 0% 283k ± 0% -9.77% (p=0.029 n=4+4) SiteNew/Regular_TOML_front_matter-10 302k ± 0% 252k ± 0% -16.55% (p=0.029 n=4+4) SiteNew/Regular_Many_HTML_templates-10 133k ± 0% 117k ± 0% -11.81% (p=0.029 n=4+4) SiteNew/Regular_Page_collections-10 202k ± 0% 183k ± 0% -9.55% (p=0.029 n=4+4) SiteNew/Regular_List_terms-10 55.6k ± 0% 49.8k ± 0% -10.40% (p=0.029 n=4+4) ``` Thanks to @quasilyte for the suggestion. Fixes 9386
2022-03-08 04:06:12 -05:00
"github.com/gohugoio/hugo/common/hreflect"
"github.com/gohugoio/hugo/common/maps"
)
// Where returns a filtered subset of a given data type.
func (ns *Namespace) Where(seq, key any, args ...any) (any, error) {
seqv, isNil := indirect(reflect.ValueOf(seq))
if isNil {
return nil, errors.New("can't iterate over a nil value of type " + reflect.ValueOf(seq).Type().String())
}
mv, op, err := parseWhereArgs(args...)
if err != nil {
return nil, err
}
var path []string
kv := reflect.ValueOf(key)
if kv.Kind() == reflect.String {
path = strings.Split(strings.Trim(kv.String(), "."), ".")
}
switch seqv.Kind() {
case reflect.Array, reflect.Slice:
return ns.checkWhereArray(seqv, kv, mv, path, op)
case reflect.Map:
return ns.checkWhereMap(seqv, kv, mv, path, op)
default:
return nil, fmt.Errorf("can't iterate over %v", seq)
}
}
func (ns *Namespace) checkCondition(v, mv reflect.Value, op string) (bool, error) {
v, vIsNil := indirect(v)
if !v.IsValid() {
vIsNil = true
}
mv, mvIsNil := indirect(mv)
if !mv.IsValid() {
mvIsNil = true
}
if vIsNil || mvIsNil {
switch op {
case "", "=", "==", "eq":
return vIsNil == mvIsNil, nil
case "!=", "<>", "ne":
return vIsNil != mvIsNil, nil
}
return false, nil
}
if v.Kind() == reflect.Bool && mv.Kind() == reflect.Bool {
switch op {
case "", "=", "==", "eq":
return v.Bool() == mv.Bool(), nil
case "!=", "<>", "ne":
return v.Bool() != mv.Bool(), nil
}
return false, nil
}
var ivp, imvp *int64
var fvp, fmvp *float64
var svp, smvp *string
var slv, slmv any
var ima []int64
var fma []float64
var sma []string
if mv.Kind() == v.Kind() {
switch v.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
iv := v.Int()
ivp = &iv
imv := mv.Int()
imvp = &imv
case reflect.String:
sv := v.String()
svp = &sv
smv := mv.String()
smvp = &smv
case reflect.Float64:
fv := v.Float()
fvp = &fv
fmv := mv.Float()
fmvp = &fmv
case reflect.Struct:
if hreflect.IsTime(v.Type()) {
iv := ns.toTimeUnix(v)
ivp = &iv
imv := ns.toTimeUnix(mv)
imvp = &imv
}
case reflect.Array, reflect.Slice:
slv = v.Interface()
slmv = mv.Interface()
}
} else if isNumber(v.Kind()) && isNumber(mv.Kind()) {
fv, err := toFloat(v)
if err != nil {
return false, err
}
fvp = &fv
fmv, err := toFloat(mv)
if err != nil {
return false, err
}
fmvp = &fmv
} else {
if mv.Kind() != reflect.Array && mv.Kind() != reflect.Slice {
return false, nil
}
if mv.Len() == 0 {
return false, nil
}
if v.Kind() != reflect.Interface && mv.Type().Elem().Kind() != reflect.Interface && mv.Type().Elem() != v.Type() && v.Kind() != reflect.Array && v.Kind() != reflect.Slice {
return false, nil
}
switch v.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
iv := v.Int()
ivp = &iv
for i := 0; i < mv.Len(); i++ {
if anInt, err := toInt(mv.Index(i)); err == nil {
ima = append(ima, anInt)
}
}
case reflect.String:
sv := v.String()
svp = &sv
for i := 0; i < mv.Len(); i++ {
if aString, err := toString(mv.Index(i)); err == nil {
sma = append(sma, aString)
}
}
case reflect.Float64:
fv := v.Float()
fvp = &fv
for i := 0; i < mv.Len(); i++ {
if aFloat, err := toFloat(mv.Index(i)); err == nil {
fma = append(fma, aFloat)
}
}
case reflect.Struct:
if hreflect.IsTime(v.Type()) {
iv := ns.toTimeUnix(v)
ivp = &iv
for i := 0; i < mv.Len(); i++ {
ima = append(ima, ns.toTimeUnix(mv.Index(i)))
}
}
case reflect.Array, reflect.Slice:
slv = v.Interface()
slmv = mv.Interface()
}
}
switch op {
case "", "=", "==", "eq":
switch {
case ivp != nil && imvp != nil:
return *ivp == *imvp, nil
case svp != nil && smvp != nil:
return *svp == *smvp, nil
case fvp != nil && fmvp != nil:
return *fvp == *fmvp, nil
}
case "!=", "<>", "ne":
switch {
case ivp != nil && imvp != nil:
return *ivp != *imvp, nil
case svp != nil && smvp != nil:
return *svp != *smvp, nil
case fvp != nil && fmvp != nil:
return *fvp != *fmvp, nil
}
case ">=", "ge":
switch {
case ivp != nil && imvp != nil:
return *ivp >= *imvp, nil
case svp != nil && smvp != nil:
return *svp >= *smvp, nil
case fvp != nil && fmvp != nil:
return *fvp >= *fmvp, nil
}
case ">", "gt":
switch {
case ivp != nil && imvp != nil:
return *ivp > *imvp, nil
case svp != nil && smvp != nil:
return *svp > *smvp, nil
case fvp != nil && fmvp != nil:
return *fvp > *fmvp, nil
}
case "<=", "le":
switch {
case ivp != nil && imvp != nil:
return *ivp <= *imvp, nil
case svp != nil && smvp != nil:
return *svp <= *smvp, nil
case fvp != nil && fmvp != nil:
return *fvp <= *fmvp, nil
}
case "<", "lt":
switch {
case ivp != nil && imvp != nil:
return *ivp < *imvp, nil
case svp != nil && smvp != nil:
return *svp < *smvp, nil
case fvp != nil && fmvp != nil:
return *fvp < *fmvp, nil
}
case "in", "not in":
var r bool
switch {
case ivp != nil && len(ima) > 0:
r, _ = ns.In(ima, *ivp)
case fvp != nil && len(fma) > 0:
r, _ = ns.In(fma, *fvp)
case svp != nil:
if len(sma) > 0 {
r, _ = ns.In(sma, *svp)
} else if smvp != nil {
r, _ = ns.In(*smvp, *svp)
}
default:
return false, nil
}
if op == "not in" {
return !r, nil
}
return r, nil
case "intersect":
r, err := ns.Intersect(slv, slmv)
if err != nil {
return false, err
}
if reflect.TypeOf(r).Kind() == reflect.Slice {
s := reflect.ValueOf(r)
if s.Len() > 0 {
return true, nil
}
return false, nil
}
return false, errors.New("invalid intersect values")
default:
return false, errors.New("no such operator")
}
return false, nil
}
func evaluateSubElem(obj reflect.Value, elemName string) (reflect.Value, error) {
if !obj.IsValid() {
return zero, errors.New("can't evaluate an invalid value")
}
typ := obj.Type()
obj, isNil := indirect(obj)
if obj.Kind() == reflect.Interface {
// If obj is an interface, we need to inspect the value it contains
// to see the full set of methods and fields.
// Indirect returns the value that it points to, which is what's needed
// below to be able to reflect on its fields.
obj = reflect.Indirect(obj.Elem())
}
// first, check whether obj has a method. In this case, obj is
// a struct or its pointer. If obj is a struct,
// to check all T and *T method, use obj pointer type Value
objPtr := obj
if objPtr.Kind() != reflect.Interface && objPtr.CanAddr() {
objPtr = objPtr.Addr()
}
Cache reflect.MethodByName The isolated benchmark for the function is obviously much faster: ```bash name old time/op new time/op delta GetMethodByName-10 1.21µs ± 7% 0.23µs ± 5% -81.42% (p=0.029 n=4+4) name old alloc/op new alloc/op delta GetMethodByName-10 680B ± 0% 0B -100.00% (p=0.029 n=4+4) name old allocs/op new allocs/op delta GetMethodByName-10 20.0 ± 0% 0.0 -100.00% (p=0.029 n=4+4) ``` But more pleasing is the overall performance looking at the site benchmarks: ```bash name old time/op new time/op delta SiteNew/Regular_Bundle_with_image-10 6.25ms ± 2% 6.10ms ± 2% ~ (p=0.057 n=4+4) SiteNew/Regular_Bundle_with_JSON_file-10 6.30ms ± 2% 5.66ms ±11% ~ (p=0.057 n=4+4) SiteNew/Regular_Tags_and_categories-10 22.2ms ± 2% 17.4ms ± 1% -21.88% (p=0.029 n=4+4) SiteNew/Regular_Canonify_URLs-10 108ms ± 0% 107ms ± 0% -1.20% (p=0.029 n=4+4) SiteNew/Regular_Deep_content_tree-10 36.1ms ± 1% 33.8ms ± 1% -6.44% (p=0.029 n=4+4) SiteNew/Regular_TOML_front_matter-10 24.9ms ± 1% 22.6ms ± 1% -9.30% (p=0.029 n=4+4) SiteNew/Regular_Many_HTML_templates-10 17.9ms ± 1% 16.7ms ± 1% -6.43% (p=0.029 n=4+4) SiteNew/Regular_Page_collections-10 23.3ms ± 1% 22.0ms ± 0% -5.58% (p=0.029 n=4+4) SiteNew/Regular_List_terms-10 8.00ms ± 1% 7.63ms ± 0% -4.62% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Regular_Bundle_with_image-10 2.10MB ± 0% 2.07MB ± 0% -1.46% (p=0.029 n=4+4) SiteNew/Regular_Bundle_with_JSON_file-10 1.88MB ± 0% 1.85MB ± 0% -1.76% (p=0.029 n=4+4) SiteNew/Regular_Tags_and_categories-10 13.5MB ± 0% 11.6MB ± 0% -13.99% (p=0.029 n=4+4) SiteNew/Regular_Canonify_URLs-10 96.1MB ± 0% 95.8MB ± 0% -0.40% (p=0.029 n=4+4) SiteNew/Regular_Deep_content_tree-10 28.4MB ± 0% 27.3MB ± 0% -3.83% (p=0.029 n=4+4) SiteNew/Regular_TOML_front_matter-10 16.9MB ± 0% 15.1MB ± 0% -10.58% (p=0.029 n=4+4) SiteNew/Regular_Many_HTML_templates-10 8.98MB ± 0% 8.44MB ± 0% -6.04% (p=0.029 n=4+4) SiteNew/Regular_Page_collections-10 17.1MB ± 0% 16.5MB ± 0% -3.91% (p=0.029 n=4+4) SiteNew/Regular_List_terms-10 3.92MB ± 0% 3.72MB ± 0% -5.03% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Regular_Bundle_with_image-10 25.8k ± 0% 24.9k ± 0% -3.49% (p=0.029 n=4+4) SiteNew/Regular_Bundle_with_JSON_file-10 25.8k ± 0% 24.9k ± 0% -3.49% (p=0.029 n=4+4) SiteNew/Regular_Tags_and_categories-10 288k ± 0% 233k ± 0% -18.90% (p=0.029 n=4+4) SiteNew/Regular_Canonify_URLs-10 375k ± 0% 364k ± 0% -2.80% (p=0.029 n=4+4) SiteNew/Regular_Deep_content_tree-10 314k ± 0% 283k ± 0% -9.77% (p=0.029 n=4+4) SiteNew/Regular_TOML_front_matter-10 302k ± 0% 252k ± 0% -16.55% (p=0.029 n=4+4) SiteNew/Regular_Many_HTML_templates-10 133k ± 0% 117k ± 0% -11.81% (p=0.029 n=4+4) SiteNew/Regular_Page_collections-10 202k ± 0% 183k ± 0% -9.55% (p=0.029 n=4+4) SiteNew/Regular_List_terms-10 55.6k ± 0% 49.8k ± 0% -10.40% (p=0.029 n=4+4) ``` Thanks to @quasilyte for the suggestion. Fixes 9386
2022-03-08 04:06:12 -05:00
index := hreflect.GetMethodIndexByName(objPtr.Type(), elemName)
if index != -1 {
mt := objPtr.Type().Method(index)
switch {
case mt.PkgPath != "":
return zero, fmt.Errorf("%s is an unexported method of type %s", elemName, typ)
case mt.Type.NumIn() > 1:
return zero, fmt.Errorf("%s is a method of type %s but requires more than 1 parameter", elemName, typ)
case mt.Type.NumOut() == 0:
return zero, fmt.Errorf("%s is a method of type %s but returns no output", elemName, typ)
case mt.Type.NumOut() > 2:
return zero, fmt.Errorf("%s is a method of type %s but returns more than 2 outputs", elemName, typ)
case mt.Type.NumOut() == 1 && mt.Type.Out(0).Implements(errorType):
return zero, fmt.Errorf("%s is a method of type %s but only returns an error type", elemName, typ)
case mt.Type.NumOut() == 2 && !mt.Type.Out(1).Implements(errorType):
return zero, fmt.Errorf("%s is a method of type %s returning two values but the second value is not an error type", elemName, typ)
}
res := objPtr.Method(mt.Index).Call([]reflect.Value{})
if len(res) == 2 && !res[1].IsNil() {
return zero, fmt.Errorf("error at calling a method %s of type %s: %s", elemName, typ, res[1].Interface().(error))
}
return res[0], nil
}
// elemName isn't a method so next start to check whether it is
// a struct field or a map value. In both cases, it mustn't be
// a nil value
if isNil {
return zero, fmt.Errorf("can't evaluate a nil pointer of type %s by a struct field or map key name %s", typ, elemName)
}
switch obj.Kind() {
case reflect.Struct:
ft, ok := obj.Type().FieldByName(elemName)
if ok {
if ft.PkgPath != "" && !ft.Anonymous {
return zero, fmt.Errorf("%s is an unexported field of struct type %s", elemName, typ)
}
return obj.FieldByIndex(ft.Index), nil
}
return zero, fmt.Errorf("%s isn't a field of struct type %s", elemName, typ)
case reflect.Map:
kv := reflect.ValueOf(elemName)
if kv.Type().AssignableTo(obj.Type().Key()) {
return obj.MapIndex(kv), nil
}
return zero, fmt.Errorf("%s isn't a key of map type %s", elemName, typ)
}
return zero, fmt.Errorf("%s is neither a struct field, a method nor a map element of type %s", elemName, typ)
}
// parseWhereArgs parses the end arguments to the where function. Return a
// match value and an operator, if one is defined.
func parseWhereArgs(args ...any) (mv reflect.Value, op string, err error) {
switch len(args) {
case 1:
mv = reflect.ValueOf(args[0])
case 2:
var ok bool
if op, ok = args[0].(string); !ok {
err = errors.New("operator argument must be string type")
return
}
op = strings.TrimSpace(strings.ToLower(op))
mv = reflect.ValueOf(args[1])
default:
err = errors.New("can't evaluate the array by no match argument or more than or equal to two arguments")
}
return
}
// checkWhereArray handles the where-matching logic when the seqv value is an
// Array or Slice.
func (ns *Namespace) checkWhereArray(seqv, kv, mv reflect.Value, path []string, op string) (any, error) {
rv := reflect.MakeSlice(seqv.Type(), 0, 0)
for i := 0; i < seqv.Len(); i++ {
var vvv reflect.Value
rvv := seqv.Index(i)
if kv.Kind() == reflect.String {
if params, ok := rvv.Interface().(maps.Params); ok {
vvv = reflect.ValueOf(params.Get(path...))
} else {
vvv = rvv
for i, elemName := range path {
var err error
vvv, err = evaluateSubElem(vvv, elemName)
if err != nil {
continue
}
if i < len(path)-1 && vvv.IsValid() {
if params, ok := vvv.Interface().(maps.Params); ok {
// The current path element is the map itself, .Params.
vvv = reflect.ValueOf(params.Get(path[i+1:]...))
break
}
}
}
}
} else {
vv, _ := indirect(rvv)
if vv.Kind() == reflect.Map && kv.Type().AssignableTo(vv.Type().Key()) {
vvv = vv.MapIndex(kv)
}
}
if ok, err := ns.checkCondition(vvv, mv, op); ok {
rv = reflect.Append(rv, rvv)
} else if err != nil {
return nil, err
}
}
return rv.Interface(), nil
}
// checkWhereMap handles the where-matching logic when the seqv value is a Map.
func (ns *Namespace) checkWhereMap(seqv, kv, mv reflect.Value, path []string, op string) (any, error) {
rv := reflect.MakeMap(seqv.Type())
keys := seqv.MapKeys()
for _, k := range keys {
elemv := seqv.MapIndex(k)
switch elemv.Kind() {
case reflect.Array, reflect.Slice:
r, err := ns.checkWhereArray(elemv, kv, mv, path, op)
if err != nil {
return nil, err
}
switch rr := reflect.ValueOf(r); rr.Kind() {
case reflect.Slice:
if rr.Len() > 0 {
rv.SetMapIndex(k, elemv)
}
}
case reflect.Interface:
elemvv, isNil := indirect(elemv)
if isNil {
continue
}
switch elemvv.Kind() {
case reflect.Array, reflect.Slice:
r, err := ns.checkWhereArray(elemvv, kv, mv, path, op)
if err != nil {
return nil, err
}
switch rr := reflect.ValueOf(r); rr.Kind() {
case reflect.Slice:
if rr.Len() > 0 {
rv.SetMapIndex(k, elemv)
}
}
}
}
}
return rv.Interface(), nil
}
// toFloat returns the float value if possible.
func toFloat(v reflect.Value) (float64, error) {
switch v.Kind() {
case reflect.Float32, reflect.Float64:
return v.Float(), nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Convert(reflect.TypeOf(float64(0))).Float(), nil
case reflect.Interface:
return toFloat(v.Elem())
}
return -1, errors.New("unable to convert value to float")
}
// toInt returns the int value if possible, -1 if not.
// TODO(bep) consolidate all these reflect funcs.
func toInt(v reflect.Value) (int64, error) {
switch v.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int(), nil
case reflect.Interface:
return toInt(v.Elem())
}
return -1, errors.New("unable to convert value to int")
}
func toUint(v reflect.Value) (uint64, error) {
switch v.Kind() {
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return v.Uint(), nil
case reflect.Interface:
return toUint(v.Elem())
}
return 0, errors.New("unable to convert value to uint")
}
// toString returns the string value if possible, "" if not.
func toString(v reflect.Value) (string, error) {
switch v.Kind() {
case reflect.String:
return v.String(), nil
case reflect.Interface:
return toString(v.Elem())
}
return "", errors.New("unable to convert value to string")
}
func (ns *Namespace) toTimeUnix(v reflect.Value) int64 {
t, ok := hreflect.AsTime(v, ns.loc)
if !ok {
panic("coding error: argument must be time.Time type reflect Value")
}
return t.Unix()
}