hugo/resources/transform.go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resources

import (
	"bytes"
	"context"
	"fmt"
	"image"
	"io"
	"path"
	"strings"
	"sync"

	"github.com/gohugoio/hugo/common/constants"
	"github.com/gohugoio/hugo/common/paths"
	"github.com/gohugoio/hugo/identity"

	"github.com/gohugoio/hugo/resources/images"
	"github.com/gohugoio/hugo/resources/images/exif"
	"github.com/spf13/afero"

	bp "github.com/gohugoio/hugo/bufferpool"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugio"
	"github.com/gohugoio/hugo/common/maps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/resources/internal"
	"github.com/gohugoio/hugo/resources/resource"

	"github.com/gohugoio/hugo/media"
)

var (
	_ resource.ContentResource           = (*resourceAdapter)(nil)
	_ resourceCopier                     = (*resourceAdapter)(nil)
	_ resource.ReadSeekCloserResource    = (*resourceAdapter)(nil)
	_ resource.Resource                  = (*resourceAdapter)(nil)
	_ resource.Staler                    = (*resourceAdapterInner)(nil)
	_ identity.IdentityGroupProvider     = (*resourceAdapterInner)(nil)
	_ resource.Source                    = (*resourceAdapter)(nil)
	_ resource.Identifier                = (*resourceAdapter)(nil)
	_ resource.ResourceNameTitleProvider = (*resourceAdapter)(nil)
	_ resource.WithResourceMetaProvider  = (*resourceAdapter)(nil)
	_ identity.DependencyManagerProvider = (*resourceAdapter)(nil)
	_ identity.IdentityGroupProvider     = (*resourceAdapter)(nil)
	_ resource.NameNormalizedProvider    = (*resourceAdapter)(nil)
)

// These are transformations that need special support in Hugo that may not
// be available when building the theme/site so we write the transformation
// result to disk and reuse if needed for these,
// TODO(bep) it's a little fragile having these constants redefined here.
var transformationsToCacheOnDisk = map[string]bool{
	"postcss":    true,
	"tocss":      true,
	"tocss-dart": true,
}

func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResource) *resourceAdapter {
	var po *publishOnce
	if lazyPublish {
		po = &publishOnce{}
	}
	return &resourceAdapter{
		resourceTransformations: &resourceTransformations{},
		metaProvider:            target,
		resourceAdapterInner: &resourceAdapterInner{
			ctx:         context.Background(),
			spec:        spec,
			publishOnce: po,
			target:      target,
			Staler:      &AtomicStaler{},
		},
	}
}

// ResourceTransformation is the interface that a resource transformation step
// needs to implement.
type ResourceTransformation interface {
	Key() internal.ResourceTransformationKey
	Transform(ctx *ResourceTransformationCtx) error
}

type ResourceTransformationCtx struct {
	// The context that started the transformation.
	Ctx context.Context

	// The dependency manager to use for dependency tracking.
	DependencyManager identity.Manager

	// The content to transform.
	From io.Reader

	// The target of content transformation.
	// The current implementation requires that r is written to w
	// even if no transformation is performed.
	To io.Writer

	// This is the relative path to the original source. Unix styled slashes.
	SourcePath string

	// This is the relative target path to the resource. Unix styled slashes.
	InPath string

	// The relative target path to the transformed resource. Unix styled slashes.
	OutPath string

	// The input media type
	InMediaType media.Type

	// The media type of the transformed resource.
	OutMediaType media.Type

	// Data data can be set on the transformed Resource. Not that this need
	// to be simple types, as it needs to be serialized to JSON and back.
	Data map[string]any

	// This is used to publish additional artifacts, e.g. source maps.
	// We may improve this.
	OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error)
}

// AddOutPathIdentifier transforming InPath to OutPath adding an identifier,
// eg '.min' before any extension.
func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) {
	ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier)
}

// PublishSourceMap writes the content to the target folder of the main resource
// with the ".map" extension added.
func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
	target := ctx.OutPath + ".map"
	f, err := ctx.OpenResourcePublisher(target)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.Write([]byte(content))
	return err
}

// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
// extension, e.g. ".scss"
func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
	dir, file := path.Split(ctx.InPath)
	base, _ := paths.PathAndExt(file)
	ctx.OutPath = path.Join(dir, (base + newExt))
}

func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
	dir, file := path.Split(inPath)
	base, ext := paths.PathAndExt(file)
	return path.Join(dir, (base + identifier + ext))
}

type publishOnce struct {
	publisherInit sync.Once
	publisherErr  error
}

type resourceAdapter struct {
	commonResource
	*resourceTransformations
	*resourceAdapterInner
	metaProvider resource.ResourceMetaProvider
}

var _ identity.ForEeachIdentityByNameProvider = (*resourceAdapter)(nil)

func (r *resourceAdapter) Content(ctx context.Context) (any, error) {
	r.init(false, true)
	if r.transformationsErr != nil {
		return nil, r.transformationsErr
	}
	return r.target.Content(ctx)
}

func (r *resourceAdapter) Err() resource.ResourceError {
	return nil
}

func (r *resourceAdapter) GetIdentity() identity.Identity {
	return identity.FirstIdentity(r.target)
}

func (r *resourceAdapter) Data() any {
	r.init(false, false)
	return r.target.Data()
}

func (r *resourceAdapter) ForEeachIdentityByName(name string, f func(identity.Identity) bool) {
	if constants.IsFieldRelOrPermalink(name) && !r.resourceTransformations.hasTransformationPermalinkHash() {
		// Special case for links without any content hash in the URL.
		// We don't need to rebuild all pages that use this resource,
		// but we want to make sure that the resource is accessed at least once.
		f(identity.NewFindFirstManagerIdentityProvider(r.target.GetDependencyManager(), r.target.GetIdentityGroup()))
		return
	}
	f(r.target.GetIdentityGroup())
	f(r.target.GetDependencyManager())
}

func (r *resourceAdapter) GetIdentityGroup() identity.Identity {
	return r.target.GetIdentityGroup()
}

func (r *resourceAdapter) GetDependencyManager() identity.Manager {
	return r.target.GetDependencyManager()
}

func (r resourceAdapter) cloneTo(targetPath string) resource.Resource {
	newtTarget := r.target.cloneTo(targetPath)
	newInner := &resourceAdapterInner{
		ctx:    r.ctx,
		spec:   r.spec,
		Staler: r.Staler,
		target: newtTarget.(transformableResource),
	}
	if r.resourceAdapterInner.publishOnce != nil {
		newInner.publishOnce = &publishOnce{}
	}
	r.resourceAdapterInner = newInner
	return &r
}

func (r *resourceAdapter) Process(spec string) (images.ImageResource, error) {
	return r.getImageOps().Process(spec)
}

func (r *resourceAdapter) Crop(spec string) (images.ImageResource, error) {
	return r.getImageOps().Crop(spec)
}

func (r *resourceAdapter) Fill(spec string) (images.ImageResource, error) {
	return r.getImageOps().Fill(spec)
}

func (r *resourceAdapter) Fit(spec string) (images.ImageResource, error) {
	return r.getImageOps().Fit(spec)
}

func (r *resourceAdapter) Filter(filters ...any) (images.ImageResource, error) {
	return r.getImageOps().Filter(filters...)
}

func (r *resourceAdapter) Resize(spec string) (images.ImageResource, error) {
	return r.getImageOps().Resize(spec)
}

func (r *resourceAdapter) Height() int {
	return r.getImageOps().Height()
}

func (r *resourceAdapter) Exif() *exif.ExifInfo {
	return r.getImageOps().Exif()
}

func (r *resourceAdapter) Colors() ([]images.Color, error) {
	return r.getImageOps().Colors()
}

func (r *resourceAdapter) Key() string {
	r.init(false, false)
	return r.target.(resource.Identifier).Key()
}

func (r *resourceAdapter) MediaType() media.Type {
	r.init(false, false)
	return r.target.MediaType()
}

func (r *resourceAdapter) Name() string {
	r.init(false, false)
	return r.metaProvider.Name()
}

func (r *resourceAdapter) NameNormalized() string {
	r.init(false, false)
	return r.target.(resource.NameNormalizedProvider).NameNormalized()
}

func (r *resourceAdapter) Params() maps.Params {
	r.init(false, false)
	return r.metaProvider.Params()
}

func (r *resourceAdapter) Permalink() string {
	r.init(true, false)
	return r.target.Permalink()
}

func (r *resourceAdapter) Publish() error {
	r.init(false, false)

	return r.target.Publish()
}

func (r *resourceAdapter) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
	r.init(false, false)
	return r.target.ReadSeekCloser()
}

func (r *resourceAdapter) RelPermalink() string {
	r.init(true, false)
	return r.target.RelPermalink()
}

func (r *resourceAdapter) ResourceType() string {
	r.init(false, false)
	return r.target.ResourceType()
}

func (r *resourceAdapter) String() string {
	return r.Name()
}

func (r *resourceAdapter) Title() string {
	r.init(false, false)
	return r.metaProvider.Title()
}

func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransformer, error) {
	return r.TransformWithContext(context.Background(), t...)
}

func (r resourceAdapter) TransformWithContext(ctx context.Context, t ...ResourceTransformation) (ResourceTransformer, error) {
	r.resourceTransformations = &resourceTransformations{
		transformations: append(r.transformations, t...),
	}

	r.resourceAdapterInner = &resourceAdapterInner{
		ctx:         ctx,
		spec:        r.spec,
		Staler:      r.Staler,
		publishOnce: &publishOnce{},
		target:      r.target,
	}

	return &r, nil
}

func (r *resourceAdapter) Width() int {
	return r.getImageOps().Width()
}

func (r *resourceAdapter) DecodeImage() (image.Image, error) {
	return r.getImageOps().DecodeImage()
}

func (r resourceAdapter) WithResourceMeta(mp resource.ResourceMetaProvider) resource.Resource {
	r.metaProvider = mp
	return &r
}

func (r *resourceAdapter) getImageOps() images.ImageResourceOps {
	img, ok := r.target.(images.ImageResourceOps)
	if !ok {
		if r.MediaType().SubType == "svg" {
			panic("this method is only available for raster images. To determine if an image is SVG, you can do {{ if eq .MediaType.SubType \"svg\" }}{{ end }}")
		}
		fmt.Println(r.MediaType().SubType)
		panic("this method is only available for image resources")
	}
	r.init(false, false)
	return img
}

func (r *resourceAdapter) publish() {
	if r.publishOnce == nil {
		return
	}

	r.publisherInit.Do(func() {
		r.publisherErr = r.target.Publish()

		if r.publisherErr != nil {
			r.spec.Logger.Errorf("Failed to publish Resource: %s", r.publisherErr)
		}
	})
}

func (r *resourceAdapter) TransformationKey() string {
	var key string
	for _, tr := range r.transformations {
		key = key + "_" + tr.Key().Value()
	}
	return r.spec.ResourceCache.cleanKey(r.target.Key()) + "_" + helpers.MD5String(key)
}

func (r *resourceAdapter) getOrTransform(publish, setContent bool) error {
	key := r.TransformationKey()
	res, err := r.spec.ResourceCache.cacheResourceTransformation.GetOrCreate(key, func(string) (*resourceAdapterInner, error) {
		return r.transform(key, publish, setContent)
	})
	if err != nil {
		return err
	}

	r.resourceAdapterInner = res
	return nil
}

func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) {
	cache := r.spec.ResourceCache

	b1 := bp.GetBuffer()
	b2 := bp.GetBuffer()
	defer bp.PutBuffer(b1)
	defer bp.PutBuffer(b2)

	tctx := &ResourceTransformationCtx{
		Ctx:                   r.ctx,
		Data:                  make(map[string]any),
		OpenResourcePublisher: r.target.openPublishFileForWriting,
		DependencyManager:     r.target.GetDependencyManager(),
	}

	tctx.InMediaType = r.target.MediaType()
	tctx.OutMediaType = r.target.MediaType()

	startCtx := *tctx
	updates := &transformationUpdate{startCtx: startCtx}

	var contentrc hugio.ReadSeekCloser

	contentrc, err := contentReadSeekerCloser(r.target)
	if err != nil {
		return nil, err
	}

	defer contentrc.Close()

	tctx.From = contentrc
	tctx.To = b1

	tctx.InPath = r.target.TargetPath()
	tctx.SourcePath = strings.TrimPrefix(tctx.InPath, "/")

	counter := 0
	writeToFileCache := false

	var transformedContentr io.Reader

	for i, tr := range r.transformations {
		if i != 0 {
			tctx.InMediaType = tctx.OutMediaType
		}

		mayBeCachedOnDisk := transformationsToCacheOnDisk[tr.Key().Name]
		if !writeToFileCache {
			writeToFileCache = mayBeCachedOnDisk
		}

		if i > 0 {
			hasWrites := tctx.To.(*bytes.Buffer).Len() > 0
			if hasWrites {
				counter++
				// Switch the buffers
				if counter%2 == 0 {
					tctx.From = b2
					b1.Reset()
					tctx.To = b1
				} else {
					tctx.From = b1
					b2.Reset()
					tctx.To = b2
				}
			}
		}

		newErr := func(err error) error {
			msg := fmt.Sprintf("%s: failed to transform %q (%s)", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type)

			if herrors.IsFeatureNotAvailableError(err) {
				var errMsg string
				if tr.Key().Name == "postcss" {
					// This transformation is not available in this
					// Most likely because PostCSS is not installed.
					errMsg = ". Check your PostCSS installation; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
				} else if tr.Key().Name == "tocss" {
					errMsg = ". Check your Hugo installation; you need the extended version to build SCSS/SASS with transpiler set to 'libsass'."
				} else if tr.Key().Name == "tocss-dart" {
					errMsg = ". You need dart-sass-embedded in your system $PATH."
				} else if tr.Key().Name == "babel" {
					errMsg = ". You need to install Babel, see https://gohugo.io/hugo-pipes/babel/"
				}

				return fmt.Errorf(msg+errMsg+": %w", err)
			}

			return fmt.Errorf(msg+": %w", err)
		}

		bcfg := r.spec.BuildConfig()
		var tryFileCache bool
		if mayBeCachedOnDisk && bcfg.UseResourceCache(nil) {
			tryFileCache = true
		} else {
			err = tr.Transform(tctx)
			if err != nil && err != herrors.ErrFeatureNotAvailable {
				return nil, newErr(err)
			}

			if mayBeCachedOnDisk {
				tryFileCache = bcfg.UseResourceCache(err)
			}
			if err != nil && !tryFileCache {
				return nil, newErr(err)
			}
		}

		if tryFileCache {
			f := r.target.tryTransformedFileCache(key, updates)
			if f == nil {
				if err != nil {
					return nil, newErr(err)
				}
				return nil, newErr(fmt.Errorf("resource %q not found in file cache", key))
			}
			transformedContentr = f
			updates.sourceFs = cache.fileCache.Fs
			defer f.Close()

			// The reader above is all we need.
			break
		}

		if tctx.OutPath != "" {
			tctx.InPath = tctx.OutPath
			tctx.OutPath = ""
		}
	}

	if transformedContentr == nil {
		updates.updateFromCtx(tctx)
	}

	var publishwriters []io.WriteCloser

	if publish {
		publicw, err := r.target.openPublishFileForWriting(updates.targetPath)
		if err != nil {
			return nil, err
		}
		publishwriters = append(publishwriters, publicw)
	}

	if transformedContentr == nil {
		if writeToFileCache {
			// Also write it to the cache
			fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata())
			if err != nil {
				return nil, err
			}
			updates.sourceFilename = &fi.Name
			updates.sourceFs = cache.fileCache.Fs
			publishwriters = append(publishwriters, metaw)
		}

		// Any transformations reading from From must also write to To.
		// This means that if the target buffer is empty, we can just reuse
		// the original reader.
		if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 {
			transformedContentr = tctx.To.(*bytes.Buffer)
		} else {
			transformedContentr = contentrc
		}
	}

	// Also write it to memory
	var contentmemw *bytes.Buffer

	setContent = setContent || !writeToFileCache

	if setContent {
		contentmemw = bp.GetBuffer()
		defer bp.PutBuffer(contentmemw)
		publishwriters = append(publishwriters, hugio.ToWriteCloser(contentmemw))
	}

	publishw := hugio.NewMultiWriteCloser(publishwriters...)
	_, err = io.Copy(publishw, transformedContentr)
	if err != nil {
		return nil, err
	}
	publishw.Close()

	if setContent {
		s := contentmemw.String()
		updates.content = &s
	}

	newTarget, err := r.target.cloneWithUpdates(updates)
	if err != nil {
		return nil, err
	}
	r.target = newTarget

	return r.resourceAdapterInner, nil
}

func (r *resourceAdapter) init(publish, setContent bool) {
	r.initTransform(publish, setContent)
}

func (r *resourceAdapter) initTransform(publish, setContent bool) {
	r.transformationsInit.Do(func() {
		if len(r.transformations) == 0 {
			// Nothing to do.
			return
		}

		if publish {
			// The transformation will write the content directly to
			// the destination.
			r.publishOnce = nil
		}

		r.transformationsErr = r.getOrTransform(publish, setContent)
		if r.transformationsErr != nil {
			if r.spec.ErrorSender != nil {
				r.spec.ErrorSender.SendError(r.transformationsErr)
			} else {
				r.spec.Logger.Errorf("Transformation failed: %s", r.transformationsErr)
			}
		}
	})

	if publish && r.publishOnce != nil {
		r.publish()
	}
}

type resourceAdapterInner struct {
	// The context that started this transformation.
	ctx context.Context

	target transformableResource

	resource.Staler

	spec *Spec

	// Handles publishing (to /public) if needed.
	*publishOnce
}

func (r *resourceAdapterInner) GetIdentityGroup() identity.Identity {
	return r.target.GetIdentityGroup()
}

func (r *resourceAdapterInner) StaleVersion() uint32 {
	// Both of these are incremented on change.
	return r.Staler.StaleVersion() + r.target.StaleVersion()
}

type resourceTransformations struct {
	transformationsInit sync.Once
	transformationsErr  error
	transformations     []ResourceTransformation
}

// hasTransformationPermalinkHash reports whether any of the transformations
// in the chain creates a permalink that's based on the content, e.g. fingerprint.
func (r *resourceTransformations) hasTransformationPermalinkHash() bool {
	for _, t := range r.transformations {
		if constants.IsResourceTransformationPermalinkHash(t.Key().Name) {
			return true
		}
	}
	return false
}

type transformableResource interface {
	baseResourceInternal

	resource.ContentProvider
	resource.Resource
	resource.Identifier
	resource.Staler
	resourceCopier
}

type transformationUpdate struct {
	content        *string
	sourceFilename *string
	sourceFs       afero.Fs
	targetPath     string
	mediaType      media.Type
	data           map[string]any

	startCtx ResourceTransformationCtx
}

func (u *transformationUpdate) isContentChanged() bool {
	return u.content != nil || u.sourceFilename != nil
}

func (u *transformationUpdate) toTransformedResourceMetadata() transformedResourceMetadata {
	return transformedResourceMetadata{
		MediaTypeV: u.mediaType.Type,
		Target:     u.targetPath,
		MetaData:   u.data,
	}
}

func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) {
	u.targetPath = ctx.OutPath
	u.mediaType = ctx.OutMediaType
	u.data = ctx.Data
	u.targetPath = ctx.InPath
}

// We will persist this information to disk.
type transformedResourceMetadata struct {
	Target     string         `json:"Target"`
	MediaTypeV string         `json:"MediaType"`
	MetaData   map[string]any `json:"Data"`
}

// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource.
func contentReadSeekerCloser(r resource.Resource) (hugio.ReadSeekCloser, error) {
	switch rr := r.(type) {
	case resource.ReadSeekCloserResource:
		rc, err := rr.ReadSeekCloser()
		if err != nil {
			return nil, err
		}
		return rc, nil
	default:
		return nil, fmt.Errorf("cannot transform content of Resource of type %T", r)

	}
}