mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-07 20:30:36 -05:00
Add hash.XxHash
Also move the non crypto hash funcs into this new package. This is much faster than e.g. MD5, especially for larger inputs: ``` BenchmarkXxHash/xxHash_43-10 9917955 112.2 ns/op 56 B/op 4 allocs/op BenchmarkXxHash/mdb5_43-10 6017239 204.1 ns/op 96 B/op 3 allocs/op BenchmarkXxHash/fnv32a_43-10 14407333 82.30 ns/op 16 B/op 1 allocs/op BenchmarkXxHash/xxHash_4300-10 2916892 409.7 ns/op 56 B/op 4 allocs/op BenchmarkXxHash/mdb5_4300-10 159748 7491 ns/op 4912 B/op 3 allocs/op BenchmarkXxHash/fnv32a_4300-10 218210 5510 ns/op 16 B/op 1 allocs/op ``` Fixes #12635
This commit is contained in:
parent
edeed52fc5
commit
644d55475d
7 changed files with 183 additions and 7 deletions
1
go.mod
1
go.mod
|
@ -21,6 +21,7 @@ require (
|
||||||
github.com/bep/overlayfs v0.9.2
|
github.com/bep/overlayfs v0.9.2
|
||||||
github.com/bep/simplecobra v0.4.0
|
github.com/bep/simplecobra v0.4.0
|
||||||
github.com/bep/tmc v0.5.1
|
github.com/bep/tmc v0.5.1
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0
|
||||||
github.com/clbanning/mxj/v2 v2.7.0
|
github.com/clbanning/mxj/v2 v2.7.0
|
||||||
github.com/cli/safeexec v1.0.1
|
github.com/cli/safeexec v1.0.1
|
||||||
github.com/disintegration/gift v1.2.1
|
github.com/disintegration/gift v1.2.1
|
||||||
|
|
2
go.sum
2
go.sum
|
@ -147,6 +147,8 @@ github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
|
||||||
github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=
|
github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=
|
||||||
github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs=
|
github.com/bep/workers v1.0.0/go.mod h1:7kIESOB86HfR2379pwoMWNy8B50D7r99fRLUyPSNyCs=
|
||||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||||
|
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||||
|
|
|
@ -25,6 +25,7 @@ import (
|
||||||
"hash"
|
"hash"
|
||||||
"hash/fnv"
|
"hash/fnv"
|
||||||
|
|
||||||
|
"github.com/gohugoio/hugo/common/hugo"
|
||||||
"github.com/spf13/cast"
|
"github.com/spf13/cast"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -72,6 +73,7 @@ func (ns *Namespace) SHA256(v any) (string, error) {
|
||||||
// FNV32a hashes v using fnv32a algorithm.
|
// FNV32a hashes v using fnv32a algorithm.
|
||||||
// <docsmeta>{"newIn": "0.98.0" }</docsmeta>
|
// <docsmeta>{"newIn": "0.98.0" }</docsmeta>
|
||||||
func (ns *Namespace) FNV32a(v any) (int, error) {
|
func (ns *Namespace) FNV32a(v any) (int, error) {
|
||||||
|
hugo.Deprecate("crypto.FNV32a", "Use hash.FNV32a.", "v0.129.0")
|
||||||
conv, err := cast.ToStringE(v)
|
conv, err := cast.ToStringE(v)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
|
|
|
@ -53,13 +53,6 @@ func init() {
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
ns.AddMethodMapping(ctx.FNV32a,
|
|
||||||
nil,
|
|
||||||
[][2]string{
|
|
||||||
{`{{ crypto.FNV32a "Hugo Rocks!!" }}`, `1515779328`},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
ns.AddMethodMapping(ctx.HMAC,
|
ns.AddMethodMapping(ctx.HMAC,
|
||||||
[]string{"hmac"},
|
[]string{"hmac"},
|
||||||
[][2]string{
|
[][2]string{
|
||||||
|
|
93
tpl/hash/hash.go
Normal file
93
tpl/hash/hash.go
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
// Copyright 2024 The Hugo Authors. All rights reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
// Package hash provides non-cryptographic hash functions for template use.
|
||||||
|
package hash
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
|
"hash/fnv"
|
||||||
|
|
||||||
|
"github.com/cespare/xxhash/v2"
|
||||||
|
"github.com/gohugoio/hugo/deps"
|
||||||
|
"github.com/gohugoio/hugo/tpl/internal"
|
||||||
|
"github.com/spf13/cast"
|
||||||
|
)
|
||||||
|
|
||||||
|
// New returns a new instance of the hash-namespaced template functions.
|
||||||
|
func New() *Namespace {
|
||||||
|
return &Namespace{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Namespace provides template functions for the "hash" namespace.
|
||||||
|
type Namespace struct{}
|
||||||
|
|
||||||
|
// FNV32a hashes v using fnv32a algorithm.
|
||||||
|
func (ns *Namespace) FNV32a(v any) (int, error) {
|
||||||
|
conv, err := cast.ToStringE(v)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
algorithm := fnv.New32a()
|
||||||
|
algorithm.Write([]byte(conv))
|
||||||
|
return int(algorithm.Sum32()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// XxHash returns the xxHash of the input string.
|
||||||
|
func (ns *Namespace) XxHash(v any) (string, error) {
|
||||||
|
conv, err := cast.ToStringE(v)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
hasher := xxhash.New()
|
||||||
|
|
||||||
|
_, err = hasher.WriteString(conv)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
hash := hasher.Sum(nil)
|
||||||
|
return hex.EncodeToString(hash), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const name = "hash"
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
|
||||||
|
ctx := New()
|
||||||
|
|
||||||
|
ns := &internal.TemplateFuncsNamespace{
|
||||||
|
Name: name,
|
||||||
|
Context: func(cctx context.Context, args ...any) (any, error) { return ctx, nil },
|
||||||
|
}
|
||||||
|
|
||||||
|
ns.AddMethodMapping(ctx.XxHash,
|
||||||
|
[]string{"xxhash"},
|
||||||
|
[][2]string{
|
||||||
|
{`{{ hash.XxHash "The quick brown fox jumps over the lazy dog" }}`, `0b242d361fda71bc`},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
ns.AddMethodMapping(ctx.FNV32a,
|
||||||
|
nil,
|
||||||
|
[][2]string{
|
||||||
|
{`{{ hash.FNV32a "Hugo Rocks!!" }}`, `1515779328`},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return ns
|
||||||
|
}
|
||||||
|
|
||||||
|
internal.AddTemplateFuncsNamespace(f)
|
||||||
|
}
|
84
tpl/hash/hash_test.go
Normal file
84
tpl/hash/hash_test.go
Normal file
|
@ -0,0 +1,84 @@
|
||||||
|
// Copyright 2024 The Hugo Authors. All rights reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
// Package hash provides non-cryptographic hash functions for template use.
|
||||||
|
package hash
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
qt "github.com/frankban/quicktest"
|
||||||
|
"github.com/spf13/cast"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestXxHash(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
c := qt.New(t)
|
||||||
|
|
||||||
|
ns := New()
|
||||||
|
|
||||||
|
h, err := ns.XxHash("The quick brown fox jumps over the lazy dog")
|
||||||
|
c.Assert(err, qt.IsNil)
|
||||||
|
// Facit: https://asecuritysite.com/encryption/xxhash?val=The%20quick%20brown%20fox%20jumps%20over%20the%20lazy%20dog
|
||||||
|
c.Assert(h, qt.Equals, "0b242d361fda71bc")
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkXxHash(b *testing.B) {
|
||||||
|
const inputSmall = "The quick brown fox jumps over the lazy dog"
|
||||||
|
inputLarge := strings.Repeat(inputSmall, 100)
|
||||||
|
|
||||||
|
runBench := func(name, input string, b *testing.B, fn func(v any)) {
|
||||||
|
b.Run(fmt.Sprintf("%s_%d", name, len(input)), func(b *testing.B) {
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
fn(input)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
ns := New()
|
||||||
|
fnXxHash := func(v any) {
|
||||||
|
_, err := ns.XxHash(v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fnFNv32a := func(v any) {
|
||||||
|
_, err := ns.FNV32a(v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Copied from the crypto tpl/crypto package,
|
||||||
|
// just to have something to compare the above with.
|
||||||
|
fnMD5 := func(v any) {
|
||||||
|
conv, err := cast.ToStringE(v)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
hash := md5.Sum([]byte(conv))
|
||||||
|
_ = hex.EncodeToString(hash[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, input := range []string{inputSmall, inputLarge} {
|
||||||
|
runBench("xxHash", input, b, fnXxHash)
|
||||||
|
runBench("mdb5", input, b, fnMD5)
|
||||||
|
runBench("fnv32a", input, b, fnFNv32a)
|
||||||
|
}
|
||||||
|
}
|
|
@ -43,6 +43,7 @@ import (
|
||||||
_ "github.com/gohugoio/hugo/tpl/diagrams"
|
_ "github.com/gohugoio/hugo/tpl/diagrams"
|
||||||
_ "github.com/gohugoio/hugo/tpl/encoding"
|
_ "github.com/gohugoio/hugo/tpl/encoding"
|
||||||
_ "github.com/gohugoio/hugo/tpl/fmt"
|
_ "github.com/gohugoio/hugo/tpl/fmt"
|
||||||
|
_ "github.com/gohugoio/hugo/tpl/hash"
|
||||||
_ "github.com/gohugoio/hugo/tpl/hugo"
|
_ "github.com/gohugoio/hugo/tpl/hugo"
|
||||||
_ "github.com/gohugoio/hugo/tpl/images"
|
_ "github.com/gohugoio/hugo/tpl/images"
|
||||||
_ "github.com/gohugoio/hugo/tpl/inflect"
|
_ "github.com/gohugoio/hugo/tpl/inflect"
|
||||||
|
|
Loading…
Reference in a new issue