mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-21 20:46:30 -05:00
resources: Cache Exif data to disk
```bash name old time/op new time/op delta ImageExif/Cold_cache-4 312µs ±28% 355µs ± 7% ~ (p=0.343 n=4+4) ImageExif/Cold_cache,_10-4 479µs ± 6% 546µs ± 0% +13.91% (p=0.029 n=4+4) ImageExif/Warm_cache-4 272µs ± 1% 81µs ± 5% -70.30% (p=0.029 n=4+4) name old alloc/op new alloc/op delta ImageExif/Cold_cache-4 151kB ± 0% 161kB ± 0% +6.46% (p=0.029 n=4+4) ImageExif/Cold_cache,_10-4 179kB ± 0% 189kB ± 0% +5.49% (p=0.029 n=4+4) ImageExif/Warm_cache-4 151kB ± 0% 13kB ± 0% -91.52% (p=0.029 n=4+4) name old allocs/op new allocs/op delta ImageExif/Cold_cache-4 1.03k ± 0% 1.21k ± 0% +17.78% (p=0.029 n=4+4) ImageExif/Cold_cache,_10-4 1.65k ± 0% 1.83k ± 0% +11.09% (p=0.029 n=4+4) ImageExif/Warm_cache-4 1.03k ± 0% 0.28k ± 0% -72.40% (p=0.029 n=4+4) ``` Fixes #6291
This commit is contained in:
Bjørn Erik Pedersen
parent
de9cbf6195
commit
ce47c21a29
7 changed files with 172 additions and 41 deletions
4
go.mod
4
go.mod
|
|
@ -12,6 +12,7 @@ require (
|
|||
github.com/bep/debounce v1.2.0
|
||||
github.com/bep/gitmap v1.1.0
|
||||
github.com/bep/go-tocss v0.6.0
|
||||
github.com/bep/tmc v0.5.0
|
||||
github.com/disintegration/gift v1.2.1
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385
|
||||
|
|
@ -20,7 +21,7 @@ require (
|
|||
github.com/fsnotify/fsnotify v1.4.7
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95
|
||||
github.com/google/go-cmp v0.3.0
|
||||
github.com/google/go-cmp v0.3.1
|
||||
github.com/gorilla/websocket v1.4.0
|
||||
github.com/jdkato/prose v1.1.0
|
||||
github.com/kyokomi/emoji v1.5.1
|
||||
|
|
@ -63,6 +64,7 @@ require (
|
|||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect
|
||||
google.golang.org/appengine v1.6.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
)
|
||||
|
||||
|
|
|
|||
6
go.sum
6
go.sum
|
|
@ -65,6 +65,8 @@ github.com/bep/gitmap v1.1.0 h1:vOMIdVB+2gd1VbfJPNJdLqAmn0af6NK98t4fK/GoCdA=
|
|||
github.com/bep/gitmap v1.1.0/go.mod h1:g9VRETxFUXNWzMiuxOwcudo6DfZkW9jOsOW0Ft4kYaY=
|
||||
github.com/bep/go-tocss v0.6.0 h1:lJf+nIjsQDpifUr+NgHi9QMBnrr9cFvMvEBT+uV9Q9E=
|
||||
github.com/bep/go-tocss v0.6.0/go.mod h1:d9d3crzlTl+PUZLFzBUjfFCpp68K+ku10mzTlnqU/+A=
|
||||
github.com/bep/tmc v0.5.0 h1:AP43LlBcCeJuXqwuQkVbTUOG6gQCo04Et4dHqOOx4hA=
|
||||
github.com/bep/tmc v0.5.0/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
|
||||
|
|
@ -138,6 +140,8 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
|||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE=
|
||||
|
|
@ -449,6 +453,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
|
|||
gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
|
|
|
|||
|
|
@ -631,7 +631,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
|
|||
v.SetDefault("disableAliases", false)
|
||||
v.SetDefault("debug", false)
|
||||
v.SetDefault("disableFastRender", false)
|
||||
v.SetDefault("timeout", 10000) // 10 seconds
|
||||
v.SetDefault("timeout", 15000) // 15 seconds
|
||||
v.SetDefault("enableInlineShortcodes", false)
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -14,15 +14,20 @@
|
|||
package resources
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
_ "image/gif"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gohugoio/hugo/cache/filecache"
|
||||
"github.com/gohugoio/hugo/resources/images/exif"
|
||||
|
||||
"github.com/gohugoio/hugo/resources/internal"
|
||||
|
|
@ -55,43 +60,81 @@ type imageResource struct {
|
|||
// original (first).
|
||||
root *imageResource
|
||||
|
||||
exifInit sync.Once
|
||||
exifInitErr error
|
||||
exif *exif.Exif
|
||||
metaInit sync.Once
|
||||
metaInitErr error
|
||||
meta *imageMeta
|
||||
|
||||
baseResource
|
||||
}
|
||||
|
||||
type imageMeta struct {
|
||||
Exif *exif.Exif
|
||||
}
|
||||
|
||||
func (i *imageResource) Exif() (*exif.Exif, error) {
|
||||
return i.root.getExif()
|
||||
}
|
||||
|
||||
func (i *imageResource) getExif() (*exif.Exif, error) {
|
||||
|
||||
i.exifInit.Do(func() {
|
||||
i.metaInit.Do(func() {
|
||||
|
||||
supportsExif := i.Format == images.JPEG || i.Format == images.TIFF
|
||||
if !supportsExif {
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
f, err := i.root.ReadSeekCloser()
|
||||
if err != nil {
|
||||
i.exifInitErr = err
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
key := i.getImageMetaCacheTargetPath()
|
||||
|
||||
x, err := i.getSpec().imaging.DecodeExif(f)
|
||||
if err != nil {
|
||||
i.exifInitErr = err
|
||||
return
|
||||
read := func(info filecache.ItemInfo, r io.Reader) error {
|
||||
meta := &imageMeta{}
|
||||
data, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(data, &meta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
i.meta = meta
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
i.exif = x
|
||||
create := func(info filecache.ItemInfo, w io.WriteCloser) (err error) {
|
||||
|
||||
f, err := i.root.ReadSeekCloser()
|
||||
if err != nil {
|
||||
i.metaInitErr = err
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
x, err := i.getSpec().imaging.DecodeExif(f)
|
||||
if err != nil {
|
||||
i.metaInitErr = err
|
||||
return
|
||||
}
|
||||
|
||||
i.meta = &imageMeta{Exif: x}
|
||||
|
||||
// Also write it to cache
|
||||
enc := json.NewEncoder(w)
|
||||
return enc.Encode(i.meta)
|
||||
|
||||
}
|
||||
|
||||
_, i.metaInitErr = i.getSpec().imageCache.fileCache.ReadOrCreate(key, read, create)
|
||||
|
||||
})
|
||||
|
||||
return i.exif, i.exifInitErr
|
||||
if i.metaInitErr != nil {
|
||||
return nil, i.metaInitErr
|
||||
}
|
||||
|
||||
return i.meta.Exif, nil
|
||||
}
|
||||
|
||||
func (i *imageResource) Clone() resource.Resource {
|
||||
|
|
@ -271,6 +314,17 @@ func (i *imageResource) setBasePath(conf images.ImageConfig) {
|
|||
i.getResourcePaths().relTargetDirFile = i.relTargetPathFromConfig(conf)
|
||||
}
|
||||
|
||||
func (i *imageResource) getImageMetaCacheTargetPath() string {
|
||||
const imageMetaVersionNumber = 1 // Increment to invalidate the meta cache
|
||||
|
||||
cfg := i.getSpec().imaging.Cfg
|
||||
df := i.getResourcePaths().relTargetDirFile
|
||||
p1, _ := helpers.FileAndExt(df.file)
|
||||
h, _ := i.hash()
|
||||
idStr := internal.HashString(h, i.size(), imageMetaVersionNumber, cfg)
|
||||
return path.Join(df.dir, fmt.Sprintf("%s%s.json", p1, idStr))
|
||||
}
|
||||
|
||||
func (i *imageResource) relTargetPathFromConfig(conf images.ImageConfig) dirFile {
|
||||
p1, p2 := helpers.FileAndExt(i.getResourcePaths().relTargetDirFile.file)
|
||||
if conf.Action == "trace" {
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ package resources
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
|
@ -22,6 +23,7 @@ import (
|
|||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
|
||||
|
|
@ -50,6 +52,14 @@ var eq = qt.CmpEquals(
|
|||
cmp.Comparer(func(m1, m2 media.Type) bool {
|
||||
return m1.Type() == m2.Type()
|
||||
}),
|
||||
cmp.Comparer(
|
||||
func(v1, v2 *big.Rat) bool {
|
||||
return v1.RatString() == v2.RatString()
|
||||
},
|
||||
),
|
||||
cmp.Comparer(func(v1, v2 time.Time) bool {
|
||||
return v1.Unix() == v2.Unix()
|
||||
}),
|
||||
)
|
||||
|
||||
func TestImageTransformBasic(t *testing.T) {
|
||||
|
|
@ -336,27 +346,35 @@ func TestSVGImageContent(t *testing.T) {
|
|||
|
||||
func TestImageExif(t *testing.T) {
|
||||
c := qt.New(t)
|
||||
image := fetchImage(c, "sunset.jpg")
|
||||
fs := afero.NewMemMapFs()
|
||||
spec := newTestResourceSpec(specDescriptor{fs: fs, c: c})
|
||||
image := fetchResourceForSpec(spec, c, "sunset.jpg").(resource.Image)
|
||||
|
||||
x, err := image.Exif()
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(x, qt.Not(qt.IsNil))
|
||||
getAndCheckExif := func(c *qt.C, image resource.Image) {
|
||||
x, err := image.Exif()
|
||||
c.Assert(err, qt.IsNil)
|
||||
c.Assert(x, qt.Not(qt.IsNil))
|
||||
|
||||
c.Assert(x.Date.Format("2006-01-02"), qt.Equals, "2017-10-27")
|
||||
c.Assert(x.Date.Format("2006-01-02"), qt.Equals, "2017-10-27")
|
||||
|
||||
// Malaga: https://goo.gl/taazZy
|
||||
c.Assert(x.Lat, qt.Equals, float64(36.59744166666667))
|
||||
c.Assert(x.Long, qt.Equals, float64(-4.50846))
|
||||
// Malaga: https://goo.gl/taazZy
|
||||
c.Assert(x.Lat, qt.Equals, float64(36.59744166666667))
|
||||
c.Assert(x.Long, qt.Equals, float64(-4.50846))
|
||||
|
||||
v, found := x.Values["LensModel"]
|
||||
c.Assert(found, qt.Equals, true)
|
||||
lensModel, ok := v.(string)
|
||||
c.Assert(ok, qt.Equals, true)
|
||||
c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM")
|
||||
v, found := x.Tags["LensModel"]
|
||||
c.Assert(found, qt.Equals, true)
|
||||
lensModel, ok := v.(string)
|
||||
c.Assert(ok, qt.Equals, true)
|
||||
c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM")
|
||||
resized, _ := image.Resize("300x200")
|
||||
x2, _ := resized.Exif()
|
||||
c.Assert(x2, eq, x)
|
||||
}
|
||||
|
||||
resized, _ := image.Resize("300x200")
|
||||
x2, _ := resized.Exif()
|
||||
c.Assert(x2, qt.Equals, x)
|
||||
getAndCheckExif(c, image)
|
||||
image = fetchResourceForSpec(spec, c, "sunset.jpg").(resource.Image)
|
||||
// This will read from file cache.
|
||||
getAndCheckExif(c, image)
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import (
|
|||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/bep/tmc"
|
||||
|
||||
_exif "github.com/rwcarlsen/goexif/exif"
|
||||
"github.com/rwcarlsen/goexif/tiff"
|
||||
)
|
||||
|
|
@ -31,10 +33,10 @@ import (
|
|||
const exifTimeLayout = "2006:01:02 15:04:05"
|
||||
|
||||
type Exif struct {
|
||||
Lat float64
|
||||
Long float64
|
||||
Date time.Time
|
||||
Values map[string]interface{}
|
||||
Lat float64
|
||||
Long float64
|
||||
Date time.Time
|
||||
Tags Tags
|
||||
}
|
||||
|
||||
type Decoder struct {
|
||||
|
|
@ -139,7 +141,7 @@ func (d *Decoder) Decode(r io.Reader) (ex *Exif, err error) {
|
|||
return
|
||||
}
|
||||
|
||||
ex = &Exif{Lat: lat, Long: long, Date: tm, Values: walker.vals}
|
||||
ex = &Exif{Lat: lat, Long: long, Date: tm, Tags: walker.vals}
|
||||
|
||||
return
|
||||
}
|
||||
|
|
@ -240,3 +242,30 @@ func nullString(in []byte) string {
|
|||
|
||||
return ""
|
||||
}
|
||||
|
||||
var tcodec *tmc.Codec
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
tcodec, err = tmc.New()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type Tags map[string]interface{}
|
||||
|
||||
func (v *Tags) UnmarshalJSON(b []byte) error {
|
||||
vv := make(map[string]interface{})
|
||||
if err := tcodec.Unmarshal(b, &vv); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*v = vv
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v Tags) MarshalJSON() ([]byte, error) {
|
||||
return tcodec.Marshal(v)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,12 +14,15 @@
|
|||
package exif
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gohugoio/hugo/htesting/hqt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
)
|
||||
|
|
@ -40,16 +43,24 @@ func TestExif(t *testing.T) {
|
|||
c.Assert(x.Lat, qt.Equals, float64(36.59744166666667))
|
||||
c.Assert(x.Long, qt.Equals, float64(-4.50846))
|
||||
|
||||
v, found := x.Values["LensModel"]
|
||||
v, found := x.Tags["LensModel"]
|
||||
c.Assert(found, qt.Equals, true)
|
||||
lensModel, ok := v.(string)
|
||||
c.Assert(ok, qt.Equals, true)
|
||||
c.Assert(lensModel, qt.Equals, "smc PENTAX-DA* 16-50mm F2.8 ED AL [IF] SDM")
|
||||
|
||||
v, found = x.Values["DateTime"]
|
||||
v, found = x.Tags["DateTime"]
|
||||
c.Assert(found, qt.Equals, true)
|
||||
c.Assert(v, hqt.IsSameType, time.Time{})
|
||||
|
||||
// Verify that it survives a round-trip to JSON and back.
|
||||
data, err := json.Marshal(x)
|
||||
c.Assert(err, qt.IsNil)
|
||||
x2 := &Exif{}
|
||||
err = json.Unmarshal(data, x2)
|
||||
|
||||
c.Assert(x2, eq, x)
|
||||
|
||||
}
|
||||
|
||||
func TestExifPNG(t *testing.T) {
|
||||
|
|
@ -81,3 +92,14 @@ func BenchmarkDecodeExif(b *testing.B) {
|
|||
f.Seek(0, 0)
|
||||
}
|
||||
}
|
||||
|
||||
var eq = qt.CmpEquals(
|
||||
cmp.Comparer(
|
||||
func(v1, v2 *big.Rat) bool {
|
||||
return v1.RatString() == v2.RatString()
|
||||
},
|
||||
),
|
||||
cmp.Comparer(func(v1, v2 time.Time) bool {
|
||||
return v1.Unix() == v2.Unix()
|
||||
}),
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue