Add hash.XxHash

Also move the non crypto hash funcs into this new package.

This is much faster than e.g. MD5, especially for larger inputs:

```
BenchmarkXxHash/xxHash_43-10         	 9917955	       112.2 ns/op	      56 B/op	       4 allocs/op
BenchmarkXxHash/mdb5_43-10           	 6017239	       204.1 ns/op	      96 B/op	       3 allocs/op
BenchmarkXxHash/fnv32a_43-10         	14407333	        82.30 ns/op	      16 B/op	       1 allocs/op
BenchmarkXxHash/xxHash_4300-10       	 2916892	       409.7 ns/op	      56 B/op	       4 allocs/op
BenchmarkXxHash/mdb5_4300-10         	  159748	      7491 ns/op	    4912 B/op	       3 allocs/op
BenchmarkXxHash/fnv32a_4300-10       	  218210	      5510 ns/op	      16 B/op	       1 allocs/op
```

Fixes #12635
This commit is contained in:
Bjørn Erik Pedersen 2024-07-03 11:20:46 +02:00
parent edeed52fc5
commit 644d55475d
7 changed files with 183 additions and 7 deletions

1
go.mod
View file

@ -21,6 +21,7 @@ require (
github.com/bep/overlayfs v0.9.2 github.com/bep/overlayfs v0.9.2
github.com/bep/simplecobra v0.4.0 github.com/bep/simplecobra v0.4.0
github.com/bep/tmc v0.5.1 github.com/bep/tmc v0.5.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/clbanning/mxj/v2 v2.7.0 github.com/clbanning/mxj/v2 v2.7.0
github.com/cli/safeexec v1.0.1 github.com/cli/safeexec v1.0.1
github.com/disintegration/gift v1.2.1 github.com/disintegration/gift v1.2.1

2
go.sum
View file

@ -147,6 +147,8 @@ github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg= github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=
github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs= github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

View file

@ -25,6 +25,7 @@ import (
"hash" "hash"
"hash/fnv" "hash/fnv"
"github.com/gohugoio/hugo/common/hugo"
"github.com/spf13/cast" "github.com/spf13/cast"
) )
@ -72,6 +73,7 @@ func (ns *Namespace) SHA256(v any) (string, error) {
// FNV32a hashes v using fnv32a algorithm. // FNV32a hashes v using fnv32a algorithm.
// <docsmeta>{"newIn": "0.98.0" }</docsmeta> // <docsmeta>{"newIn": "0.98.0" }</docsmeta>
func (ns *Namespace) FNV32a(v any) (int, error) { func (ns *Namespace) FNV32a(v any) (int, error) {
hugo.Deprecate("crypto.FNV32a", "Use hash.FNV32a.", "v0.129.0")
conv, err := cast.ToStringE(v) conv, err := cast.ToStringE(v)
if err != nil { if err != nil {
return 0, err return 0, err

View file

@ -53,13 +53,6 @@ func init() {
}, },
) )
ns.AddMethodMapping(ctx.FNV32a,
nil,
[][2]string{
{`{{ crypto.FNV32a "Hugo Rocks!!" }}`, `1515779328`},
},
)
ns.AddMethodMapping(ctx.HMAC, ns.AddMethodMapping(ctx.HMAC,
[]string{"hmac"}, []string{"hmac"},
[][2]string{ [][2]string{

93
tpl/hash/hash.go Normal file
View file

@ -0,0 +1,93 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package hash provides non-cryptographic hash functions for template use.
package hash
import (
"context"
"encoding/hex"
"hash/fnv"
"github.com/cespare/xxhash/v2"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/tpl/internal"
"github.com/spf13/cast"
)
// New returns a new instance of the hash-namespaced template functions.
func New() *Namespace {
return &Namespace{}
}
// Namespace provides template functions for the "hash" namespace.
type Namespace struct{}
// FNV32a hashes v using fnv32a algorithm.
func (ns *Namespace) FNV32a(v any) (int, error) {
conv, err := cast.ToStringE(v)
if err != nil {
return 0, err
}
algorithm := fnv.New32a()
algorithm.Write([]byte(conv))
return int(algorithm.Sum32()), nil
}
// XxHash returns the xxHash of the input string.
func (ns *Namespace) XxHash(v any) (string, error) {
conv, err := cast.ToStringE(v)
if err != nil {
return "", err
}
hasher := xxhash.New()
_, err = hasher.WriteString(conv)
if err != nil {
return "", err
}
hash := hasher.Sum(nil)
return hex.EncodeToString(hash), nil
}
const name = "hash"
func init() {
f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
ctx := New()
ns := &internal.TemplateFuncsNamespace{
Name: name,
Context: func(cctx context.Context, args ...any) (any, error) { return ctx, nil },
}
ns.AddMethodMapping(ctx.XxHash,
[]string{"xxhash"},
[][2]string{
{`{{ hash.XxHash "The quick brown fox jumps over the lazy dog" }}`, `0b242d361fda71bc`},
},
)
ns.AddMethodMapping(ctx.FNV32a,
nil,
[][2]string{
{`{{ hash.FNV32a "Hugo Rocks!!" }}`, `1515779328`},
},
)
return ns
}
internal.AddTemplateFuncsNamespace(f)
}

84
tpl/hash/hash_test.go Normal file
View file

@ -0,0 +1,84 @@
// Copyright 2024 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package hash provides non-cryptographic hash functions for template use.
package hash
import (
"crypto/md5"
"encoding/hex"
"fmt"
"strings"
"testing"
qt "github.com/frankban/quicktest"
"github.com/spf13/cast"
)
func TestXxHash(t *testing.T) {
t.Parallel()
c := qt.New(t)
ns := New()
h, err := ns.XxHash("The quick brown fox jumps over the lazy dog")
c.Assert(err, qt.IsNil)
// Facit: https://asecuritysite.com/encryption/xxhash?val=The%20quick%20brown%20fox%20jumps%20over%20the%20lazy%20dog
c.Assert(h, qt.Equals, "0b242d361fda71bc")
}
func BenchmarkXxHash(b *testing.B) {
const inputSmall = "The quick brown fox jumps over the lazy dog"
inputLarge := strings.Repeat(inputSmall, 100)
runBench := func(name, input string, b *testing.B, fn func(v any)) {
b.Run(fmt.Sprintf("%s_%d", name, len(input)), func(b *testing.B) {
for i := 0; i < b.N; i++ {
fn(input)
}
})
}
ns := New()
fnXxHash := func(v any) {
_, err := ns.XxHash(v)
if err != nil {
panic(err)
}
}
fnFNv32a := func(v any) {
_, err := ns.FNV32a(v)
if err != nil {
panic(err)
}
}
// Copied from the crypto tpl/crypto package,
// just to have something to compare the above with.
fnMD5 := func(v any) {
conv, err := cast.ToStringE(v)
if err != nil {
panic(err)
}
hash := md5.Sum([]byte(conv))
_ = hex.EncodeToString(hash[:])
}
for _, input := range []string{inputSmall, inputLarge} {
runBench("xxHash", input, b, fnXxHash)
runBench("mdb5", input, b, fnMD5)
runBench("fnv32a", input, b, fnFNv32a)
}
}

View file

@ -43,6 +43,7 @@ import (
_ "github.com/gohugoio/hugo/tpl/diagrams" _ "github.com/gohugoio/hugo/tpl/diagrams"
_ "github.com/gohugoio/hugo/tpl/encoding" _ "github.com/gohugoio/hugo/tpl/encoding"
_ "github.com/gohugoio/hugo/tpl/fmt" _ "github.com/gohugoio/hugo/tpl/fmt"
_ "github.com/gohugoio/hugo/tpl/hash"
_ "github.com/gohugoio/hugo/tpl/hugo" _ "github.com/gohugoio/hugo/tpl/hugo"
_ "github.com/gohugoio/hugo/tpl/images" _ "github.com/gohugoio/hugo/tpl/images"
_ "github.com/gohugoio/hugo/tpl/inflect" _ "github.com/gohugoio/hugo/tpl/inflect"