hugo/resources/transform.go
Bjørn Erik Pedersen e2d66e3218
Create pages from _content.gotmpl
Closes #12427
Closes #12485
Closes #6310
Closes #5074
2024-05-14 13:12:08 +02:00

747 lines
20 KiB
Go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package resources
import (
"bytes"
"context"
"fmt"
"image"
"io"
"path"
"strings"
"sync"
"github.com/gohugoio/hugo/common/constants"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/images"
"github.com/gohugoio/hugo/resources/images/exif"
"github.com/spf13/afero"
bp "github.com/gohugoio/hugo/bufferpool"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/resources/internal"
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/media"
)
var (
_ resource.ContentResource = (*resourceAdapter)(nil)
_ resourceCopier = (*resourceAdapter)(nil)
_ resource.ReadSeekCloserResource = (*resourceAdapter)(nil)
_ resource.Resource = (*resourceAdapter)(nil)
_ resource.Staler = (*resourceAdapterInner)(nil)
_ identity.IdentityGroupProvider = (*resourceAdapterInner)(nil)
_ resource.Source = (*resourceAdapter)(nil)
_ resource.Identifier = (*resourceAdapter)(nil)
_ resource.ResourceNameTitleProvider = (*resourceAdapter)(nil)
_ resource.WithResourceMetaProvider = (*resourceAdapter)(nil)
_ identity.DependencyManagerProvider = (*resourceAdapter)(nil)
_ identity.IdentityGroupProvider = (*resourceAdapter)(nil)
_ resource.NameNormalizedProvider = (*resourceAdapter)(nil)
)
// These are transformations that need special support in Hugo that may not
// be available when building the theme/site so we write the transformation
// result to disk and reuse if needed for these,
// TODO(bep) it's a little fragile having these constants redefined here.
var transformationsToCacheOnDisk = map[string]bool{
"postcss": true,
"tocss": true,
"tocss-dart": true,
}
func newResourceAdapter(spec *Spec, lazyPublish bool, target transformableResource) *resourceAdapter {
var po *publishOnce
if lazyPublish {
po = &publishOnce{}
}
return &resourceAdapter{
resourceTransformations: &resourceTransformations{},
metaProvider: target,
resourceAdapterInner: &resourceAdapterInner{
ctx: context.Background(),
spec: spec,
publishOnce: po,
target: target,
Staler: &AtomicStaler{},
},
}
}
// ResourceTransformation is the interface that a resource transformation step
// needs to implement.
type ResourceTransformation interface {
Key() internal.ResourceTransformationKey
Transform(ctx *ResourceTransformationCtx) error
}
type ResourceTransformationCtx struct {
// The context that started the transformation.
Ctx context.Context
// The dependency manager to use for dependency tracking.
DependencyManager identity.Manager
// The content to transform.
From io.Reader
// The target of content transformation.
// The current implementation requires that r is written to w
// even if no transformation is performed.
To io.Writer
// This is the relative path to the original source. Unix styled slashes.
SourcePath string
// This is the relative target path to the resource. Unix styled slashes.
InPath string
// The relative target path to the transformed resource. Unix styled slashes.
OutPath string
// The input media type
InMediaType media.Type
// The media type of the transformed resource.
OutMediaType media.Type
// Data data can be set on the transformed Resource. Not that this need
// to be simple types, as it needs to be serialized to JSON and back.
Data map[string]any
// This is used to publish additional artifacts, e.g. source maps.
// We may improve this.
OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error)
}
// AddOutPathIdentifier transforming InPath to OutPath adding an identifier,
// eg '.min' before any extension.
func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) {
ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier)
}
// PublishSourceMap writes the content to the target folder of the main resource
// with the ".map" extension added.
func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
target := ctx.OutPath + ".map"
f, err := ctx.OpenResourcePublisher(target)
if err != nil {
return err
}
defer f.Close()
_, err = f.Write([]byte(content))
return err
}
// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
// extension, e.g. ".scss"
func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
dir, file := path.Split(ctx.InPath)
base, _ := paths.PathAndExt(file)
ctx.OutPath = path.Join(dir, (base + newExt))
}
func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
dir, file := path.Split(inPath)
base, ext := paths.PathAndExt(file)
return path.Join(dir, (base + identifier + ext))
}
type publishOnce struct {
publisherInit sync.Once
publisherErr error
}
type resourceAdapter struct {
commonResource
*resourceTransformations
*resourceAdapterInner
metaProvider resource.ResourceMetaProvider
}
var _ identity.ForEeachIdentityByNameProvider = (*resourceAdapter)(nil)
func (r *resourceAdapter) Content(ctx context.Context) (any, error) {
r.init(false, true)
if r.transformationsErr != nil {
return nil, r.transformationsErr
}
return r.target.Content(ctx)
}
func (r *resourceAdapter) Err() resource.ResourceError {
return nil
}
func (r *resourceAdapter) GetIdentity() identity.Identity {
return identity.FirstIdentity(r.target)
}
func (r *resourceAdapter) Data() any {
r.init(false, false)
return r.target.Data()
}
func (r *resourceAdapter) ForEeachIdentityByName(name string, f func(identity.Identity) bool) {
if constants.IsFieldRelOrPermalink(name) && !r.resourceTransformations.hasTransformationPermalinkHash() {
// Special case for links without any content hash in the URL.
// We don't need to rebuild all pages that use this resource,
// but we want to make sure that the resource is accessed at least once.
f(identity.NewFindFirstManagerIdentityProvider(r.target.GetDependencyManager(), r.target.GetIdentityGroup()))
return
}
f(r.target.GetIdentityGroup())
f(r.target.GetDependencyManager())
}
func (r *resourceAdapter) GetIdentityGroup() identity.Identity {
return r.target.GetIdentityGroup()
}
func (r *resourceAdapter) GetDependencyManager() identity.Manager {
return r.target.GetDependencyManager()
}
func (r resourceAdapter) cloneTo(targetPath string) resource.Resource {
newtTarget := r.target.cloneTo(targetPath)
newInner := &resourceAdapterInner{
ctx: r.ctx,
spec: r.spec,
Staler: r.Staler,
target: newtTarget.(transformableResource),
}
if r.resourceAdapterInner.publishOnce != nil {
newInner.publishOnce = &publishOnce{}
}
r.resourceAdapterInner = newInner
return &r
}
func (r *resourceAdapter) Process(spec string) (images.ImageResource, error) {
return r.getImageOps().Process(spec)
}
func (r *resourceAdapter) Crop(spec string) (images.ImageResource, error) {
return r.getImageOps().Crop(spec)
}
func (r *resourceAdapter) Fill(spec string) (images.ImageResource, error) {
return r.getImageOps().Fill(spec)
}
func (r *resourceAdapter) Fit(spec string) (images.ImageResource, error) {
return r.getImageOps().Fit(spec)
}
func (r *resourceAdapter) Filter(filters ...any) (images.ImageResource, error) {
return r.getImageOps().Filter(filters...)
}
func (r *resourceAdapter) Resize(spec string) (images.ImageResource, error) {
return r.getImageOps().Resize(spec)
}
func (r *resourceAdapter) Height() int {
return r.getImageOps().Height()
}
func (r *resourceAdapter) Exif() *exif.ExifInfo {
return r.getImageOps().Exif()
}
func (r *resourceAdapter) Colors() ([]images.Color, error) {
return r.getImageOps().Colors()
}
func (r *resourceAdapter) Key() string {
r.init(false, false)
return r.target.(resource.Identifier).Key()
}
func (r *resourceAdapter) MediaType() media.Type {
r.init(false, false)
return r.target.MediaType()
}
func (r *resourceAdapter) Name() string {
r.init(false, false)
return r.metaProvider.Name()
}
func (r *resourceAdapter) NameNormalized() string {
r.init(false, false)
return r.target.(resource.NameNormalizedProvider).NameNormalized()
}
func (r *resourceAdapter) Params() maps.Params {
r.init(false, false)
return r.metaProvider.Params()
}
func (r *resourceAdapter) Permalink() string {
r.init(true, false)
return r.target.Permalink()
}
func (r *resourceAdapter) Publish() error {
r.init(false, false)
return r.target.Publish()
}
func (r *resourceAdapter) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
r.init(false, false)
return r.target.ReadSeekCloser()
}
func (r *resourceAdapter) RelPermalink() string {
r.init(true, false)
return r.target.RelPermalink()
}
func (r *resourceAdapter) ResourceType() string {
r.init(false, false)
return r.target.ResourceType()
}
func (r *resourceAdapter) String() string {
return r.Name()
}
func (r *resourceAdapter) Title() string {
r.init(false, false)
return r.metaProvider.Title()
}
func (r resourceAdapter) Transform(t ...ResourceTransformation) (ResourceTransformer, error) {
return r.TransformWithContext(context.Background(), t...)
}
func (r resourceAdapter) TransformWithContext(ctx context.Context, t ...ResourceTransformation) (ResourceTransformer, error) {
r.resourceTransformations = &resourceTransformations{
transformations: append(r.transformations, t...),
}
r.resourceAdapterInner = &resourceAdapterInner{
ctx: ctx,
spec: r.spec,
Staler: r.Staler,
publishOnce: &publishOnce{},
target: r.target,
}
return &r, nil
}
func (r *resourceAdapter) Width() int {
return r.getImageOps().Width()
}
func (r *resourceAdapter) DecodeImage() (image.Image, error) {
return r.getImageOps().DecodeImage()
}
func (r resourceAdapter) WithResourceMeta(mp resource.ResourceMetaProvider) resource.Resource {
r.metaProvider = mp
return &r
}
func (r *resourceAdapter) getImageOps() images.ImageResourceOps {
img, ok := r.target.(images.ImageResourceOps)
if !ok {
if r.MediaType().SubType == "svg" {
panic("this method is only available for raster images. To determine if an image is SVG, you can do {{ if eq .MediaType.SubType \"svg\" }}{{ end }}")
}
fmt.Println(r.MediaType().SubType)
panic("this method is only available for image resources")
}
r.init(false, false)
return img
}
func (r *resourceAdapter) publish() {
if r.publishOnce == nil {
return
}
r.publisherInit.Do(func() {
r.publisherErr = r.target.Publish()
if r.publisherErr != nil {
r.spec.Logger.Errorf("Failed to publish Resource: %s", r.publisherErr)
}
})
}
func (r *resourceAdapter) TransformationKey() string {
var key string
for _, tr := range r.transformations {
key = key + "_" + tr.Key().Value()
}
return r.spec.ResourceCache.cleanKey(r.target.Key()) + "_" + helpers.MD5String(key)
}
func (r *resourceAdapter) getOrTransform(publish, setContent bool) error {
key := r.TransformationKey()
res, err := r.spec.ResourceCache.cacheResourceTransformation.GetOrCreate(key, func(string) (*resourceAdapterInner, error) {
return r.transform(key, publish, setContent)
})
if err != nil {
return err
}
r.resourceAdapterInner = res
return nil
}
func (r *resourceAdapter) transform(key string, publish, setContent bool) (*resourceAdapterInner, error) {
cache := r.spec.ResourceCache
b1 := bp.GetBuffer()
b2 := bp.GetBuffer()
defer bp.PutBuffer(b1)
defer bp.PutBuffer(b2)
tctx := &ResourceTransformationCtx{
Ctx: r.ctx,
Data: make(map[string]any),
OpenResourcePublisher: r.target.openPublishFileForWriting,
DependencyManager: r.target.GetDependencyManager(),
}
tctx.InMediaType = r.target.MediaType()
tctx.OutMediaType = r.target.MediaType()
startCtx := *tctx
updates := &transformationUpdate{startCtx: startCtx}
var contentrc hugio.ReadSeekCloser
contentrc, err := contentReadSeekerCloser(r.target)
if err != nil {
return nil, err
}
defer contentrc.Close()
tctx.From = contentrc
tctx.To = b1
tctx.InPath = r.target.TargetPath()
tctx.SourcePath = strings.TrimPrefix(tctx.InPath, "/")
counter := 0
writeToFileCache := false
var transformedContentr io.Reader
for i, tr := range r.transformations {
if i != 0 {
tctx.InMediaType = tctx.OutMediaType
}
mayBeCachedOnDisk := transformationsToCacheOnDisk[tr.Key().Name]
if !writeToFileCache {
writeToFileCache = mayBeCachedOnDisk
}
if i > 0 {
hasWrites := tctx.To.(*bytes.Buffer).Len() > 0
if hasWrites {
counter++
// Switch the buffers
if counter%2 == 0 {
tctx.From = b2
b1.Reset()
tctx.To = b1
} else {
tctx.From = b1
b2.Reset()
tctx.To = b2
}
}
}
newErr := func(err error) error {
msg := fmt.Sprintf("%s: failed to transform %q (%s)", strings.ToUpper(tr.Key().Name), tctx.InPath, tctx.InMediaType.Type)
if herrors.IsFeatureNotAvailableError(err) {
var errMsg string
if tr.Key().Name == "postcss" {
// This transformation is not available in this
// Most likely because PostCSS is not installed.
errMsg = ". Check your PostCSS installation; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
} else if tr.Key().Name == "tocss" {
errMsg = ". Check your Hugo installation; you need the extended version to build SCSS/SASS with transpiler set to 'libsass'."
} else if tr.Key().Name == "tocss-dart" {
errMsg = ". You need dart-sass-embedded in your system $PATH."
} else if tr.Key().Name == "babel" {
errMsg = ". You need to install Babel, see https://gohugo.io/hugo-pipes/babel/"
}
return fmt.Errorf(msg+errMsg+": %w", err)
}
return fmt.Errorf(msg+": %w", err)
}
bcfg := r.spec.BuildConfig()
var tryFileCache bool
if mayBeCachedOnDisk && bcfg.UseResourceCache(nil) {
tryFileCache = true
} else {
err = tr.Transform(tctx)
if err != nil && err != herrors.ErrFeatureNotAvailable {
return nil, newErr(err)
}
if mayBeCachedOnDisk {
tryFileCache = bcfg.UseResourceCache(err)
}
if err != nil && !tryFileCache {
return nil, newErr(err)
}
}
if tryFileCache {
f := r.target.tryTransformedFileCache(key, updates)
if f == nil {
if err != nil {
return nil, newErr(err)
}
return nil, newErr(fmt.Errorf("resource %q not found in file cache", key))
}
transformedContentr = f
updates.sourceFs = cache.fileCache.Fs
defer f.Close()
// The reader above is all we need.
break
}
if tctx.OutPath != "" {
tctx.InPath = tctx.OutPath
tctx.OutPath = ""
}
}
if transformedContentr == nil {
updates.updateFromCtx(tctx)
}
var publishwriters []io.WriteCloser
if publish {
publicw, err := r.target.openPublishFileForWriting(updates.targetPath)
if err != nil {
return nil, err
}
publishwriters = append(publishwriters, publicw)
}
if transformedContentr == nil {
if writeToFileCache {
// Also write it to the cache
fi, metaw, err := cache.writeMeta(key, updates.toTransformedResourceMetadata())
if err != nil {
return nil, err
}
updates.sourceFilename = &fi.Name
updates.sourceFs = cache.fileCache.Fs
publishwriters = append(publishwriters, metaw)
}
// Any transformations reading from From must also write to To.
// This means that if the target buffer is empty, we can just reuse
// the original reader.
if b, ok := tctx.To.(*bytes.Buffer); ok && b.Len() > 0 {
transformedContentr = tctx.To.(*bytes.Buffer)
} else {
transformedContentr = contentrc
}
}
// Also write it to memory
var contentmemw *bytes.Buffer
setContent = setContent || !writeToFileCache
if setContent {
contentmemw = bp.GetBuffer()
defer bp.PutBuffer(contentmemw)
publishwriters = append(publishwriters, hugio.ToWriteCloser(contentmemw))
}
publishw := hugio.NewMultiWriteCloser(publishwriters...)
_, err = io.Copy(publishw, transformedContentr)
if err != nil {
return nil, err
}
publishw.Close()
if setContent {
s := contentmemw.String()
updates.content = &s
}
newTarget, err := r.target.cloneWithUpdates(updates)
if err != nil {
return nil, err
}
r.target = newTarget
return r.resourceAdapterInner, nil
}
func (r *resourceAdapter) init(publish, setContent bool) {
r.initTransform(publish, setContent)
}
func (r *resourceAdapter) initTransform(publish, setContent bool) {
r.transformationsInit.Do(func() {
if len(r.transformations) == 0 {
// Nothing to do.
return
}
if publish {
// The transformation will write the content directly to
// the destination.
r.publishOnce = nil
}
r.transformationsErr = r.getOrTransform(publish, setContent)
if r.transformationsErr != nil {
if r.spec.ErrorSender != nil {
r.spec.ErrorSender.SendError(r.transformationsErr)
} else {
r.spec.Logger.Errorf("Transformation failed: %s", r.transformationsErr)
}
}
})
if publish && r.publishOnce != nil {
r.publish()
}
}
type resourceAdapterInner struct {
// The context that started this transformation.
ctx context.Context
target transformableResource
resource.Staler
spec *Spec
// Handles publishing (to /public) if needed.
*publishOnce
}
func (r *resourceAdapterInner) GetIdentityGroup() identity.Identity {
return r.target.GetIdentityGroup()
}
func (r *resourceAdapterInner) StaleVersion() uint32 {
// Both of these are incremented on change.
return r.Staler.StaleVersion() + r.target.StaleVersion()
}
type resourceTransformations struct {
transformationsInit sync.Once
transformationsErr error
transformations []ResourceTransformation
}
// hasTransformationPermalinkHash reports whether any of the transformations
// in the chain creates a permalink that's based on the content, e.g. fingerprint.
func (r *resourceTransformations) hasTransformationPermalinkHash() bool {
for _, t := range r.transformations {
if constants.IsResourceTransformationPermalinkHash(t.Key().Name) {
return true
}
}
return false
}
type transformableResource interface {
baseResourceInternal
resource.ContentProvider
resource.Resource
resource.Identifier
resource.Staler
resourceCopier
}
type transformationUpdate struct {
content *string
sourceFilename *string
sourceFs afero.Fs
targetPath string
mediaType media.Type
data map[string]any
startCtx ResourceTransformationCtx
}
func (u *transformationUpdate) isContentChanged() bool {
return u.content != nil || u.sourceFilename != nil
}
func (u *transformationUpdate) toTransformedResourceMetadata() transformedResourceMetadata {
return transformedResourceMetadata{
MediaTypeV: u.mediaType.Type,
Target: u.targetPath,
MetaData: u.data,
}
}
func (u *transformationUpdate) updateFromCtx(ctx *ResourceTransformationCtx) {
u.targetPath = ctx.OutPath
u.mediaType = ctx.OutMediaType
u.data = ctx.Data
u.targetPath = ctx.InPath
}
// We will persist this information to disk.
type transformedResourceMetadata struct {
Target string `json:"Target"`
MediaTypeV string `json:"MediaType"`
MetaData map[string]any `json:"Data"`
}
// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource.
func contentReadSeekerCloser(r resource.Resource) (hugio.ReadSeekCloser, error) {
switch rr := r.(type) {
case resource.ReadSeekCloserResource:
rc, err := rr.ReadSeekCloser()
if err != nil {
return nil, err
}
return rc, nil
default:
return nil, fmt.Errorf("cannot transform content of Resource of type %T", r)
}
}