hugo/resources/transform.go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package resources

import (
	"bytes"
	"path"
	"strconv"
	"strings"

	"github.com/pkg/errors"

	"github.com/gohugoio/hugo/common/collections"
	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/hugio"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/resources/resource"
	"github.com/mitchellh/hashstructure"

	"fmt"
	"io"
	"sync"

	"github.com/gohugoio/hugo/media"

	bp "github.com/gohugoio/hugo/bufferpool"
)

var (
	_ resource.ContentResource        = (*transformedResource)(nil)
	_ resource.ReadSeekCloserResource = (*transformedResource)(nil)
	_ collections.Slicer              = (*transformedResource)(nil)
	_ resource.Identifier             = (*transformedResource)(nil)
)

func (s *Spec) Transform(r resource.Resource, t ResourceTransformation) (resource.Resource, error) {
	if r == nil {
		return nil, errors.New("got nil Resource in transformation. Make sure you check with 'with' or 'if' when you get a resource, e.g. with resources.Get.")
	}

	return &transformedResource{
		Resource:                    r,
		transformation:              t,
		transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})},
		cache:                       s.ResourceCache}, nil
}

type ResourceTransformationCtx struct {
	// The content to transform.
	From io.Reader

	// The target of content transformation.
	// The current implementation requires that r is written to w
	// even if no transformation is performed.
	To io.Writer

	// This is the relative path to the original source. Unix styled slashes.
	SourcePath string

	// This is the relative target path to the resource. Unix styled slashes.
	InPath string

	// The relative target path to the transformed resource. Unix styled slashes.
	OutPath string

	// The input media type
	InMediaType media.Type

	// The media type of the transformed resource.
	OutMediaType media.Type

	// Data data can be set on the transformed Resource. Not that this need
	// to be simple types, as it needs to be serialized to JSON and back.
	Data map[string]interface{}

	// This is used to publis additional artifacts, e.g. source maps.
	// We may improve this.
	OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error)
}

// AddOutPathIdentifier transforming InPath to OutPath adding an identifier,
// eg '.min' before any extension.
func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) {
	ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier)
}

func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string {
	dir, file := path.Split(inPath)
	base, ext := helpers.PathAndExt(file)
	return path.Join(dir, (base + identifier + ext))
}

// ReplaceOutPathExtension transforming InPath to OutPath replacing the file
// extension, e.g. ".scss"
func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) {
	dir, file := path.Split(ctx.InPath)
	base, _ := helpers.PathAndExt(file)
	ctx.OutPath = path.Join(dir, (base + newExt))
}

// PublishSourceMap writes the content to the target folder of the main resource
// with the ".map" extension added.
func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error {
	target := ctx.OutPath + ".map"
	f, err := ctx.OpenResourcePublisher(target)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.Write([]byte(content))
	return err
}

// ResourceTransformationKey are provided by the different transformation implementations.
// It identifies the transformation (name) and its configuration (elements).
// We combine this in a chain with the rest of the transformations
// with the target filename and a content hash of the origin to use as cache key.
type ResourceTransformationKey struct {
	name     string
	elements []interface{}
}

// NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation
// name and elements. We will create a 64 bit FNV hash from the elements, which when combined
// with the other key elements should be unique for all practical applications.
func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey {
	return ResourceTransformationKey{name: name, elements: elements}
}

// Do not change this without good reasons.
func (k ResourceTransformationKey) key() string {
	if len(k.elements) == 0 {
		return k.name
	}

	sb := bp.GetBuffer()
	defer bp.PutBuffer(sb)

	sb.WriteString(k.name)
	for _, element := range k.elements {
		hash, err := hashstructure.Hash(element, nil)
		if err != nil {
			panic(err)
		}
		sb.WriteString("_")
		sb.WriteString(strconv.FormatUint(hash, 10))
	}

	return sb.String()
}

// ResourceTransformation is the interface that a resource transformation step
// needs to implement.
type ResourceTransformation interface {
	Key() ResourceTransformationKey
	Transform(ctx *ResourceTransformationCtx) error
}

// We will persist this information to disk.
type transformedResourceMetadata struct {
	Target     string                 `json:"Target"`
	MediaTypeV string                 `json:"MediaType"`
	MetaData   map[string]interface{} `json:"Data"`
}

type transformedResource struct {
	commonResource

	cache *ResourceCache

	// This is the filename inside resources/_gen/assets
	sourceFilename string

	linker permalinker

	// The transformation to apply.
	transformation ResourceTransformation

	// We apply the tranformations lazily.
	transformInit sync.Once
	transformErr  error

	// We delay publishing until either .RelPermalink or .Permalink
	// is invoked.
	publishInit sync.Once
	published   bool

	// The transformed values
	content     string
	contentInit sync.Once
	transformedResourceMetadata

	// The source
	resource.Resource
}

func (r *transformedResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) {
	if err := r.initContent(); err != nil {
		return nil, err
	}
	return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
}

func (r *transformedResource) transferTransformedValues(another *transformedResource) {
	if another.content != "" {
		r.contentInit.Do(func() {
			r.content = another.content
		})
	}
	r.transformedResourceMetadata = another.transformedResourceMetadata
}

func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser {
	fi, f, meta, found := r.cache.getFromFile(key)
	if !found {
		return nil
	}
	r.transformedResourceMetadata = meta
	r.sourceFilename = fi.Name

	return f
}

func (r *transformedResource) Content() (interface{}, error) {
	if err := r.initTransform(true, false); err != nil {
		return nil, err
	}
	if err := r.initContent(); err != nil {
		return "", err
	}
	return r.content, nil
}

func (r *transformedResource) Data() interface{} {
	if err := r.initTransform(false, false); err != nil {
		return noData
	}
	return r.MetaData
}

func (r *transformedResource) MediaType() media.Type {
	if err := r.initTransform(false, false); err != nil {
		return media.Type{}
	}
	m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV)
	return m
}

func (r *transformedResource) Key() string {
	if err := r.initTransform(false, false); err != nil {
		return ""
	}
	return r.linker.relPermalinkFor(r.Target)
}

func (r *transformedResource) Permalink() string {
	if err := r.initTransform(false, true); err != nil {
		return ""
	}
	return r.linker.permalinkFor(r.Target)
}

func (r *transformedResource) RelPermalink() string {
	if err := r.initTransform(false, true); err != nil {
		return ""
	}
	return r.linker.relPermalinkFor(r.Target)
}

func (r *transformedResource) initContent() error {
	var err error
	r.contentInit.Do(func() {
		var b []byte
		_, b, err = r.cache.fileCache.GetBytes(r.sourceFilename)
		if err != nil {
			return
		}
		r.content = string(b)
	})
	return err
}

func (r *transformedResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) {
	return helpers.OpenFilesForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathsFor(relTargetPath)...)
}

func (r *transformedResource) transform(setContent, publish bool) (err error) {

	// This can be the last resource in a chain.
	// Rewind and create a processing chain.
	var chain []resource.Resource
	current := r
	for {
		rr := current.Resource
		chain = append(chain[:0], append([]resource.Resource{rr}, chain[0:]...)...)
		if tr, ok := rr.(*transformedResource); ok {
			current = tr
		} else {
			break
		}
	}

	// Append the current transformer at the end
	chain = append(chain, r)

	first := chain[0]

	// Files with a suffix will be stored in cache (both on disk and in memory)
	// partitioned by their suffix. There will be other files below /other.
	// This partition is also how we determine what to delete on server reloads.
	var key, base string
	for _, element := range chain {
		switch v := element.(type) {
		case *transformedResource:
			key = key + "_" + v.transformation.Key().key()
		case permalinker:
			r.linker = v
			p := v.TargetPath()
			if p == "" {
				panic("target path needed for key creation")
			}
			partition := ResourceKeyPartition(p)
			base = partition + "/" + p
		default:
			return fmt.Errorf("transformation not supported for type %T", element)
		}
	}

	key = r.cache.cleanKey(base + "_" + helpers.MD5String(key))

	cached, found := r.cache.get(key)
	if found {
		r.transferTransformedValues(cached.(*transformedResource))
		return
	}

	// Acquire a write lock for the named transformation.
	r.cache.nlocker.Lock(key)
	// Check the cache again.
	cached, found = r.cache.get(key)
	if found {
		r.transferTransformedValues(cached.(*transformedResource))
		r.cache.nlocker.Unlock(key)
		return
	}

	defer r.cache.nlocker.Unlock(key)
	defer r.cache.set(key, r)

	b1 := bp.GetBuffer()
	b2 := bp.GetBuffer()
	defer bp.PutBuffer(b1)
	defer bp.PutBuffer(b2)

	tctx := &ResourceTransformationCtx{
		Data:                  r.transformedResourceMetadata.MetaData,
		OpenResourcePublisher: r.openPublishFileForWriting,
	}

	tctx.InMediaType = first.MediaType()
	tctx.OutMediaType = first.MediaType()

	contentrc, err := contentReadSeekerCloser(first)
	if err != nil {
		return err
	}
	defer contentrc.Close()

	tctx.From = contentrc
	tctx.To = b1

	if r.linker != nil {
		tctx.InPath = r.linker.TargetPath()
		tctx.SourcePath = tctx.InPath
	}

	counter := 0

	var transformedContentr io.Reader

	for _, element := range chain {
		tr, ok := element.(*transformedResource)
		if !ok {
			continue
		}
		counter++
		if counter != 1 {
			tctx.InMediaType = tctx.OutMediaType
		}
		if counter%2 == 0 {
			tctx.From = b1
			b2.Reset()
			tctx.To = b2
		} else {
			if counter != 1 {
				// The first reader is the file.
				tctx.From = b2
			}
			b1.Reset()
			tctx.To = b1
		}

		if err := tr.transformation.Transform(tctx); err != nil {

			if err == herrors.ErrFeatureNotAvailable {
				// This transformation is not available in this
				// Hugo installation (scss not compiled in, PostCSS not available etc.)
				// If a prepared bundle for this transformation chain is available, use that.
				f := r.tryTransformedFileCache(key)
				if f == nil {
					errMsg := err.Error()
					if tr.transformation.Key().name == "postcss" {
						errMsg = "PostCSS not found; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/"
					}
					return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.transformation.Key().name), tctx.InPath, tctx.InMediaType.Type(), errMsg)
				}
				transformedContentr = f
				defer f.Close()

				// The reader above is all we need.
				break
			}

			// Abort.
			return err
		}

		if tctx.OutPath != "" {
			tctx.InPath = tctx.OutPath
			tctx.OutPath = ""
		}
	}

	if transformedContentr == nil {
		r.Target = tctx.InPath
		r.MediaTypeV = tctx.OutMediaType.Type()
	}

	var publishwriters []io.WriteCloser

	if publish {
		publicw, err := r.openPublishFileForWriting(r.Target)
		if err != nil {
			r.transformErr = err
			return err
		}
		defer publicw.Close()

		publishwriters = append(publishwriters, publicw)
	}

	if transformedContentr == nil {
		// Also write it to the cache
		fi, metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata)
		if err != nil {
			return err
		}
		r.sourceFilename = fi.Name

		publishwriters = append(publishwriters, metaw)

		if counter > 0 {
			transformedContentr = tctx.To.(*bytes.Buffer)
		} else {
			transformedContentr = contentrc
		}
	}

	// Also write it to memory
	var contentmemw *bytes.Buffer

	if setContent {
		contentmemw = bp.GetBuffer()
		defer bp.PutBuffer(contentmemw)
		publishwriters = append(publishwriters, hugio.ToWriteCloser(contentmemw))
	}

	publishw := hugio.NewMultiWriteCloser(publishwriters...)
	_, r.transformErr = io.Copy(publishw, transformedContentr)
	publishw.Close()

	if setContent {
		r.contentInit.Do(func() {
			r.content = contentmemw.String()
		})
	}

	return nil
}

func (r *transformedResource) initTransform(setContent, publish bool) error {
	r.transformInit.Do(func() {
		r.published = publish
		if err := r.transform(setContent, publish); err != nil {
			r.transformErr = err
			r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err)
		}

	})

	if !publish {
		return r.transformErr
	}

	r.publishInit.Do(func() {
		if r.published {
			return
		}

		r.published = true

		// Copy the file from cache to /public
		_, src, err := r.cache.fileCache.Get(r.sourceFilename)
		if src == nil {
			panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename))
		}

		if err == nil {
			defer src.Close()

			var dst io.WriteCloser
			dst, err = r.openPublishFileForWriting(r.Target)
			if err == nil {
				defer dst.Close()
				io.Copy(dst, src)
			}
		}

		if err != nil {
			r.transformErr = err
			r.cache.rs.Logger.ERROR.Println("error: failed to publish resource:", err)
			return
		}

	})

	return r.transformErr
}

// contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource.
func contentReadSeekerCloser(r resource.Resource) (hugio.ReadSeekCloser, error) {
	switch rr := r.(type) {
	case resource.ReadSeekCloserResource:
		rc, err := rr.ReadSeekCloser()
		if err != nil {
			return nil, err
		}
		return rc, nil
	default:
		return nil, fmt.Errorf("cannot transform content of Resource of type %T", r)

	}
}