hugo/resources/transform.go
Bjørn Erik Pedersen e4d6ec94b5 Allow user to handle/ignore errors in resources.Get
In Hugo 0.90.0 we introduced remote support in `resources.Get`.

But with remote resources comes with a higher chance of failing a build (network issues, remote server down etc.).

Before this commit we always failed the build on any unexpected error.

This commit allows the user to check for any error (and potentially fall back to a default local resource):

```htmlbars
{{ $result := resources.Get "https://gohugo.io/img/hugo-logo.png" }}
{{ with $result }}
        {{ if .Err }}
        {{/* log the error, insert a default image etc. *}}
        {{ else }}
        <img src="{{ .RelPermalink }}" width="{{ .Width }}" height="{{ .Height }}" alt="">
        {{ end }}
{{ end }}
```

Note that the default behaviour is still to fail the build, but we will delay that error until you start using the `Resource`.

Fixes #9529
2021-12-10 11:10:41 +01:00

648 lines
16 KiB
Go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resources
import (
"bytes"
"fmt"
"image"
"io"
"path"
"strings"
"sync"
"github.com/gohugoio/hugo/common/paths"
"github.com/pkg/errors"
"github.com/gohugoio/hugo/resources/images/exif"
"github.com/spf13/afero"
bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/media"
)
var (
_ resource.ContentResource = (*resourceAdapter)(nil)
_ resource.ReadSeekCloserResource = (*resourceAdapter)(nil)
_ resource.Resource = (*resourceAdapter)(nil)
_ resource.Source = (*resourceAdapter)(nil)
_ resource.Identifier = (*resourceAdapter)(nil)
_ resource.ResourceMetaProvider = (*resourceAdapter)(nil)
)
// These are transformations that need special support in Hugo that may not
// be available when building the theme/site so we write the transformation
// result to disk and reuse if needed for these,
// TODO(bep) it's a little fragile having these constants redefined here.
var transformationsToCacheOnDisk = map[string]bool{
"postcss": true,
"tocss": true,
"tocss-dart": true,
}
func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResource) *resourceAdapter {
var po *publishOnce
if lazyPublish {
po = &publishOnce{}
}
return &resourceAdapter{
resourceTransformations: &resourceTransformations{},
resourceAdapterInner: &resourceAdapterInner{
spec: spec,
publishOnce: po,
target: target,
},
}
}
// ResourceTransformation is the interface that a resource transformation step
// needs to implement.
type ResourceTransformation interface {
Key() internal.ResourceTransformationKey
Transform(ctx *ResourceTransformationCtx) error
}
type ResourceTransformationCtx struct {
// The content to transform.
From io.Reader
// The target of content transformation.
// The current implementation requires that r is written to w
// even if no transformation is performed.
To io.Writer
// This is the relative path to the original source. Unix styled slashes.
SourcePath string
// This is the relative target path to the resource. Unix styled slashes.
InPath string
// The relative target path to the transformed resource. Unix styled slashes.
OutPath string
// The input media type
InMediaType media.Type
// The media type of the transformed resource.
OutMediaType media.Type
// Data data can be set on the transformed Resource. Not that this need
// to be simple types, as it needs to be serialized to JSON and back.
Data map[string]interface{}
// This is used to publish additional artifacts, e.g. source maps.
// We may improve this.
OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error)
}
// AddOutPathIdentifier transforming InPath to OutPath adding an identifier,
// eg '.min' before any extension.
func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) {
ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier)
}
// PublishSourceMap writes the content to the target folder of the main resource
// with the ".map" extension added.
func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
target := ctx.OutPath + ".map"
f, err := ctx.OpenResourcePublisher(target)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write([]byte(content))
return err
}
// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
// extension, e.g. ".scss"
func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
dir, file := path.Split(ctx.InPath)
base, _ := paths.PathAndExt(file)
ctx.OutPath = path.Join(dir, (base + newExt))
}
func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
dir, file := path.Split(inPath)
base, ext := paths.PathAndExt(file)
return path.Join(dir, (base + identifier + ext))
}
type publishOnce struct {
publisherInit sync.Once
publisherErr error
}
type resourceAdapter struct {
commonResource
*resourceTransformations
*resourceAdapterInner
}
func (r *resourceAdapter) Content() (interface{}, error) {
r.init(false, true)
if r.transformationsErr != nil {
return nil, r.transformationsErr
}
return r.target.Content()
}
func (r *resourceAdapter) Err() error {
return nil
}
func (r *resourceAdapter) Data() interface{} {
r.init(false, false)
return r.target.Data()
}
func (r *resourceAdapter) Fill(spec string) (resource.Image, error) {
return r.getImageOps().Fill(spec)
}
func (r *resourceAdapter) Fit(spec string) (resource.Image, error) {
return r.getImageOps().Fit(spec)
}
func (r *resourceAdapter) Filter(filters ...interface{}) (resource.Image, error) {
return r.getImageOps().Filter(filters...)
}
func (r *resourceAdapter) Height() int {
return r.getImageOps().Height()
}
func (r *resourceAdapter) Exif() *exif.Exif {
return r.getImageOps().Exif()
}
func (r *resourceAdapter) Key() string {
r.init(false, false)
return r.target.(resource.Identifier).Key()
}
func (r *resourceAdapter) MediaType() media.Type {
r.init(false, false)
return r.target.MediaType()
}
func (r *resourceAdapter) Name() string {
r.init(false, false)
return r.target.Name()
}
func (r *resourceAdapter) Params() maps.Params {
r.init(false, false)
return r.target.Params()
}
func (r *resourceAdapter) Permalink() string {
r.init(true, false)
return r.target.Permalink()
}
func (r *resourceAdapter) Publish() error {
r.init(false, false)
return r.target.Publish()
}
func (r *resourceAdapter) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
r.init(false, false)
return r.target.ReadSeekCloser()
}
func (r *resourceAdapter) RelPermalink() string {
r.init(true, false)
return r.target.RelPermalink()
}
func (r *resourceAdapter) Resize(spec string) (resource.Image, error) {
return r.getImageOps().Resize(spec)
}
func (r *resourceAdapter) ResourceType() string {
r.init(false, false)
return r.target.ResourceType()
}
func (r *resourceAdapter) String() string {
return r.Name()
}
func (r *resourceAdapter) Title() string {
r.init(false, false)
return r.target.Title()
}
func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransformer, error) {
r.resourceTransformations = &resourceTransformations{
transformations: append(r.transformations, t...),
}
r.resourceAdapterInner = &resourceAdapterInner{
spec: r.spec,
publishOnce: &publishOnce{},
target: r.target,
}
return &r, nil
}
func (r *resourceAdapter) Width() int {
return r.getImageOps().Width()
}
func (r *resourceAdapter) DecodeImage() (image.Image, error) {
return r.getImageOps().DecodeImage()
}
func (r *resourceAdapter) getImageOps() resource.ImageOps {
img, ok := r.target.(resource.ImageOps)
if !ok {
panic(fmt.Sprintf("%T is not an image", r.target))
}
r.init(false, false)
return img
}
func (r *resourceAdapter) getMetaAssigner() metaAssigner {
return r.target
}
func (r *resourceAdapter) getSpec() *Spec {
return r.spec
}
func (r *resourceAdapter) publish() {
if r.publishOnce == nil {
return
}
r.publisherInit.Do(func() {
r.publisherErr = r.target.Publish()
if r.publisherErr != nil {
r.spec.Logger.Errorf("Failed to publish Resource: %s", r.publisherErr)
}
})
}
func (r *resourceAdapter) TransformationKey() string {
// Files with a suffix will be stored in cache (both on disk and in memory)
// partitioned by their suffix.
var key string
for _, tr := range r.transformations {
key = key + "_" + tr.Key().Value()
}
base := ResourceCacheKey(r.target.Key())
return r.spec.ResourceCache.cleanKey(base) + "_" + helpers.MD5String(key)
}
func (r *resourceAdapter) transform(publish, setContent bool) error {
cache := r.spec.ResourceCache
key := r.TransformationKey()
cached, found := cache.get(key)
if found {
r.resourceAdapterInner = cached.(*resourceAdapterInner)
return nil
}
// Acquire a write lock for the named transformation.
cache.nlocker.Lock(key)
// Check the cache again.
cached, found = cache.get(key)
if found {
r.resourceAdapterInner = cached.(*resourceAdapterInner)
cache.nlocker.Unlock(key)
return nil
}
defer cache.nlocker.Unlock(key)
defer cache.set(key, r.resourceAdapterInner)
b1 := bp.GetBuffer()
b2 := bp.GetBuffer()
defer bp.PutBuffer(b1)
defer bp.PutBuffer(b2)
tctx := &ResourceTransformationCtx{
Data: make(map[string]interface{}),
OpenResourcePublisher: r.target.openPublishFileForWriting,
}
tctx.InMediaType = r.target.MediaType()
tctx.OutMediaType = r.target.MediaType()
startCtx := *tctx
updates := &transformationUpdate{startCtx: startCtx}
var contentrc hugio.ReadSeekCloser
contentrc, err := contentReadSeekerCloser(r.target)
if err != nil {
return err
}
defer contentrc.Close()
tctx.From = contentrc
tctx.To = b1
tctx.InPath = r.target.TargetPath()
tctx.SourcePath = tctx.InPath
counter := 0
writeToFileCache := false
var transformedContentr io.Reader
for i, tr := range r.transformations {
if i != 0 {
tctx.InMediaType = tctx.OutMediaType
}
mayBeCachedOnDisk := transformationsToCacheOnDisk[tr.Key().Name]
if !writeToFileCache {
writeToFileCache = mayBeCachedOnDisk
}
if i > 0 {
hasWrites := tctx.To.(*bytes.Buffer).Len() > 0
if hasWrites {
counter++
// Switch the buffers
if counter%2 == 0 {
tctx.From = b2
b1.Reset()
tctx.To = b1
} else {
tctx.From = b1
b2.Reset()
tctx.To = b2
}
}
}
newErr := func(err error) error {
msg := fmt.Sprintf("%s: failed to transform %q (%s)", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type())
if err == herrors.ErrFeatureNotAvailable {
var errMsg string
if tr.Key().Name == "postcss" {
// This transformation is not available in this
// Most likely because PostCSS is not installed.
errMsg = ". Check your PostCSS installation; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
} else if tr.Key().Name == "tocss" {
errMsg = ". Check your Hugo installation; you need the extended version to build SCSS/SASS."
} else if tr.Key().Name == "tocss-dart" {
errMsg = ". You need dart-sass-embedded in your system $PATH."
} else if tr.Key().Name == "babel" {
errMsg = ". You need to install Babel, see https://gohugo.io/hugo-pipes/babel/"
}
return errors.Wrap(err, msg+errMsg)
}
return errors.Wrap(err, msg)
}
var tryFileCache bool
if mayBeCachedOnDisk && r.spec.BuildConfig.UseResourceCache(nil) {
tryFileCache = true
} else {
err = tr.Transform(tctx)
if err != nil && err != herrors.ErrFeatureNotAvailable {
return newErr(err)
}
if mayBeCachedOnDisk {
tryFileCache = r.spec.BuildConfig.UseResourceCache(err)
}
if err != nil && !tryFileCache {
return newErr(err)
}
}
if tryFileCache {
f := r.target.tryTransformedFileCache(key, updates)
if f == nil {
if err != nil {
return newErr(err)
}
return newErr(errors.Errorf("resource %q not found in file cache", key))
}
transformedContentr = f
updates.sourceFs = cache.fileCache.Fs
defer f.Close()
// The reader above is all we need.
break
}
if tctx.OutPath != "" {
tctx.InPath = tctx.OutPath
tctx.OutPath = ""
}
}
if transformedContentr == nil {
updates.updateFromCtx(tctx)
}
var publishwriters []io.WriteCloser
if publish {
publicw, err := r.target.openPublishFileForWriting(updates.targetPath)
if err != nil {
return err
}
publishwriters = append(publishwriters, publicw)
}
if transformedContentr == nil {
if writeToFileCache {
// Also write it to the cache
fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata())
if err != nil {
return err
}
updates.sourceFilename = &fi.Name
updates.sourceFs = cache.fileCache.Fs
publishwriters = append(publishwriters, metaw)
}
// Any transformations reading from From must also write to To.
// This means that if the target buffer is empty, we can just reuse
// the original reader.
if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 {
transformedContentr = tctx.To.(*bytes.Buffer)
} else {
transformedContentr = contentrc
}
}
// Also write it to memory
var contentmemw *bytes.Buffer
setContent = setContent || !writeToFileCache
if setContent {
contentmemw = bp.GetBuffer()
defer bp.PutBuffer(contentmemw)
publishwriters = append(publishwriters, hugio.ToWriteCloser(contentmemw))
}
publishw := hugio.NewMultiWriteCloser(publishwriters...)
_, err = io.Copy(publishw, transformedContentr)
if err != nil {
return err
}
publishw.Close()
if setContent {
s := contentmemw.String()
updates.content = &s
}
newTarget, err := r.target.cloneWithUpdates(updates)
if err != nil {
return err
}
r.target = newTarget
return nil
}
func (r *resourceAdapter) init(publish, setContent bool) {
r.initTransform(publish, setContent)
}
func (r *resourceAdapter) initTransform(publish, setContent bool) {
r.transformationsInit.Do(func() {
if len(r.transformations) == 0 {
// Nothing to do.
return
}
if publish {
// The transformation will write the content directly to
// the destination.
r.publishOnce = nil
}
r.transformationsErr = r.transform(publish, setContent)
if r.transformationsErr != nil {
if r.spec.ErrorSender != nil {
r.spec.ErrorSender.SendError(r.transformationsErr)
} else {
r.spec.Logger.Errorf("Transformation failed: %s", r.transformationsErr)
}
}
})
if publish && r.publishOnce != nil {
r.publish()
}
}
type resourceAdapterInner struct {
target transformableResource
spec *Spec
// Handles publishing (to /public) if needed.
*publishOnce
}
type resourceTransformations struct {
transformationsInit sync.Once
transformationsErr error
transformations []ResourceTransformation
}
type transformableResource interface {
baseResourceInternal
resource.ContentProvider
resource.Resource
resource.Identifier
}
type transformationUpdate struct {
content *string
sourceFilename *string
sourceFs afero.Fs
targetPath string
mediaType media.Type
data map[string]interface{}
startCtx ResourceTransformationCtx
}
func (u *transformationUpdate) isContentChanged() bool {
return u.content != nil || u.sourceFilename != nil
}
func (u *transformationUpdate) toTransformedResourceMetadata() transformedResourceMetadata {
return transformedResourceMetadata{
MediaTypeV: u.mediaType.Type(),
Target: u.targetPath,
MetaData: u.data,
}
}
func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) {
u.targetPath = ctx.OutPath
u.mediaType = ctx.OutMediaType
u.data = ctx.Data
u.targetPath = ctx.InPath
}
// We will persist this information to disk.
type transformedResourceMetadata struct {
Target string `json:"Target"`
MediaTypeV string `json:"MediaType"`
MetaData map[string]interface{} `json:"Data"`
}
// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource.
func contentReadSeekerCloser(r resource.Resource) (hugio.ReadSeekCloser, error) {
switch rr := r.(type) {
case resource.ReadSeekCloserResource:
rc, err := rr.ReadSeekCloser()
if err != nil {
return nil, err
}
return rc, nil
default:
return nil, fmt.Errorf("cannot transform content of Resource of type %T", r)
}
}