// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package resources import ( "bytes" "path" "strconv" "strings" "github.com/gohugoio/hugo/common/collections" "github.com/gohugoio/hugo/common/herrors" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources/resource" "github.com/mitchellh/hashstructure" "fmt" "io" "sync" "github.com/gohugoio/hugo/media" bp "github.com/gohugoio/hugo/bufferpool" ) var ( _ resource.ContentResource = (*transformedResource)(nil) _ resource.ReadSeekCloserResource = (*transformedResource)(nil) _ collections.Slicer = (*transformedResource)(nil) _ resource.Identifier = (*transformedResource)(nil) ) func (s *Spec) Transform(r resource.Resource, t ResourceTransformation) (resource.Resource, error) { return &transformedResource{ Resource: r, transformation: t, transformedResourceMetadata: transformedResourceMetadata{MetaData: make(map[string]interface{})}, cache: s.ResourceCache}, nil } type ResourceTransformationCtx struct { // The content to transform. From io.Reader // The target of content transformation. // The current implementation requires that r is written to w // even if no transformation is performed. To io.Writer // This is the relative path to the original source. Unix styled slashes. SourcePath string // This is the relative target path to the resource. Unix styled slashes. InPath string // The relative target path to the transformed resource. Unix styled slashes. OutPath string // The input media type InMediaType media.Type // The media type of the transformed resource. OutMediaType media.Type // Data data can be set on the transformed Resource. Not that this need // to be simple types, as it needs to be serialized to JSON and back. Data map[string]interface{} // This is used to publis additional artifacts, e.g. source maps. // We may improve this. OpenResourcePublisher func(relTargetPath string) (io.WriteCloser, error) } // AddOutPathIdentifier transforming InPath to OutPath adding an identifier, // eg '.min' before any extension. func (ctx *ResourceTransformationCtx) AddOutPathIdentifier(identifier string) { ctx.OutPath = ctx.addPathIdentifier(ctx.InPath, identifier) } func (ctx *ResourceTransformationCtx) addPathIdentifier(inPath, identifier string) string { dir, file := path.Split(inPath) base, ext := helpers.PathAndExt(file) return path.Join(dir, (base + identifier + ext)) } // ReplaceOutPathExtension transforming InPath to OutPath replacing the file // extension, e.g. ".scss" func (ctx *ResourceTransformationCtx) ReplaceOutPathExtension(newExt string) { dir, file := path.Split(ctx.InPath) base, _ := helpers.PathAndExt(file) ctx.OutPath = path.Join(dir, (base + newExt)) } // PublishSourceMap writes the content to the target folder of the main resource // with the ".map" extension added. func (ctx *ResourceTransformationCtx) PublishSourceMap(content string) error { target := ctx.OutPath + ".map" f, err := ctx.OpenResourcePublisher(target) if err != nil { return err } defer f.Close() _, err = f.Write([]byte(content)) return err } // ResourceTransformationKey are provided by the different transformation implementations. // It identifies the transformation (name) and its configuration (elements). // We combine this in a chain with the rest of the transformations // with the target filename and a content hash of the origin to use as cache key. type ResourceTransformationKey struct { name string elements []interface{} } // NewResourceTransformationKey creates a new ResourceTransformationKey from the transformation // name and elements. We will create a 64 bit FNV hash from the elements, which when combined // with the other key elements should be unique for all practical applications. func NewResourceTransformationKey(name string, elements ...interface{}) ResourceTransformationKey { return ResourceTransformationKey{name: name, elements: elements} } // Do not change this without good reasons. func (k ResourceTransformationKey) key() string { if len(k.elements) == 0 { return k.name } sb := bp.GetBuffer() defer bp.PutBuffer(sb) sb.WriteString(k.name) for _, element := range k.elements { hash, err := hashstructure.Hash(element, nil) if err != nil { panic(err) } sb.WriteString("_") sb.WriteString(strconv.FormatUint(hash, 10)) } return sb.String() } // ResourceTransformation is the interface that a resource transformation step // needs to implement. type ResourceTransformation interface { Key() ResourceTransformationKey Transform(ctx *ResourceTransformationCtx) error } // We will persist this information to disk. type transformedResourceMetadata struct { Target string `json:"Target"` MediaTypeV string `json:"MediaType"` MetaData map[string]interface{} `json:"Data"` } type transformedResource struct { commonResource cache *ResourceCache // This is the filename inside resources/_gen/assets sourceFilename string linker permalinker // The transformation to apply. transformation ResourceTransformation // We apply the tranformations lazily. transformInit sync.Once transformErr error // We delay publishing until either .RelPermalink or .Permalink // is invoked. publishInit sync.Once published bool // The transformed values content string contentInit sync.Once transformedResourceMetadata // The source resource.Resource } func (r *transformedResource) ReadSeekCloser() (hugio.ReadSeekCloser, error) { if err := r.initContent(); err != nil { return nil, err } return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil } func (r *transformedResource) transferTransformedValues(another *transformedResource) { if another.content != "" { r.contentInit.Do(func() { r.content = another.content }) } r.transformedResourceMetadata = another.transformedResourceMetadata } func (r *transformedResource) tryTransformedFileCache(key string) io.ReadCloser { fi, f, meta, found := r.cache.getFromFile(key) if !found { return nil } r.transformedResourceMetadata = meta r.sourceFilename = fi.Name return f } func (r *transformedResource) Content() (interface{}, error) { if err := r.initTransform(true, false); err != nil { return nil, err } if err := r.initContent(); err != nil { return "", err } return r.content, nil } func (r *transformedResource) Data() interface{} { if err := r.initTransform(false, false); err != nil { return noData } return r.MetaData } func (r *transformedResource) MediaType() media.Type { if err := r.initTransform(false, false); err != nil { return media.Type{} } m, _ := r.cache.rs.MediaTypes.GetByType(r.MediaTypeV) return m } func (r *transformedResource) Key() string { if err := r.initTransform(false, false); err != nil { return "" } return r.linker.relPermalinkFor(r.Target) } func (r *transformedResource) Permalink() string { if err := r.initTransform(false, true); err != nil { return "" } return r.linker.permalinkFor(r.Target) } func (r *transformedResource) RelPermalink() string { if err := r.initTransform(false, true); err != nil { return "" } return r.linker.relPermalinkFor(r.Target) } func (r *transformedResource) initContent() error { var err error r.contentInit.Do(func() { var b []byte _, b, err = r.cache.fileCache.GetBytes(r.sourceFilename) if err != nil { return } r.content = string(b) }) return err } func (r *transformedResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) { return helpers.OpenFilesForWriting(r.cache.rs.PublishFs, r.linker.relTargetPathsFor(relTargetPath)...) } func (r *transformedResource) transform(setContent, publish bool) (err error) { // This can be the last resource in a chain. // Rewind and create a processing chain. var chain []resource.Resource current := r for { rr := current.Resource chain = append(chain[:0], append([]resource.Resource{rr}, chain[0:]...)...) if tr, ok := rr.(*transformedResource); ok { current = tr } else { break } } // Append the current transformer at the end chain = append(chain, r) first := chain[0] // Files with a suffix will be stored in cache (both on disk and in memory) // partitioned by their suffix. There will be other files below /other. // This partition is also how we determine what to delete on server reloads. var key, base string for _, element := range chain { switch v := element.(type) { case *transformedResource: key = key + "_" + v.transformation.Key().key() case permalinker: r.linker = v p := v.TargetPath() if p == "" { panic("target path needed for key creation") } partition := ResourceKeyPartition(p) base = partition + "/" + p default: return fmt.Errorf("transformation not supported for type %T", element) } } key = r.cache.cleanKey(base + "_" + helpers.MD5String(key)) cached, found := r.cache.get(key) if found { r.transferTransformedValues(cached.(*transformedResource)) return } // Acquire a write lock for the named transformation. r.cache.nlocker.Lock(key) // Check the cache again. cached, found = r.cache.get(key) if found { r.transferTransformedValues(cached.(*transformedResource)) r.cache.nlocker.Unlock(key) return } defer r.cache.nlocker.Unlock(key) defer r.cache.set(key, r) b1 := bp.GetBuffer() b2 := bp.GetBuffer() defer bp.PutBuffer(b1) defer bp.PutBuffer(b2) tctx := &ResourceTransformationCtx{ Data: r.transformedResourceMetadata.MetaData, OpenResourcePublisher: r.openPublishFileForWriting, } tctx.InMediaType = first.MediaType() tctx.OutMediaType = first.MediaType() contentrc, err := contentReadSeekerCloser(first) if err != nil { return err } defer contentrc.Close() tctx.From = contentrc tctx.To = b1 if r.linker != nil { tctx.InPath = r.linker.TargetPath() tctx.SourcePath = tctx.InPath } counter := 0 var transformedContentr io.Reader for _, element := range chain { tr, ok := element.(*transformedResource) if !ok { continue } counter++ if counter != 1 { tctx.InMediaType = tctx.OutMediaType } if counter%2 == 0 { tctx.From = b1 b2.Reset() tctx.To = b2 } else { if counter != 1 { // The first reader is the file. tctx.From = b2 } b1.Reset() tctx.To = b1 } if err := tr.transformation.Transform(tctx); err != nil { if err == herrors.ErrFeatureNotAvailable { // This transformation is not available in this // Hugo installation (scss not compiled in, PostCSS not available etc.) // If a prepared bundle for this transformation chain is available, use that. f := r.tryTransformedFileCache(key) if f == nil { errMsg := err.Error() if tr.transformation.Key().name == "postcss" { errMsg = "PostCSS not found; install with \"npm install postcss-cli\". See https://gohugo.io/hugo-pipes/postcss/" } return fmt.Errorf("%s: failed to transform %q (%s): %s", strings.ToUpper(tr.transformation.Key().name), tctx.InPath, tctx.InMediaType.Type(), errMsg) } transformedContentr = f defer f.Close() // The reader above is all we need. break } // Abort. return err } if tctx.OutPath != "" { tctx.InPath = tctx.OutPath tctx.OutPath = "" } } if transformedContentr == nil { r.Target = tctx.InPath r.MediaTypeV = tctx.OutMediaType.Type() } var publishwriters []io.WriteCloser if publish { publicw, err := r.openPublishFileForWriting(r.Target) if err != nil { r.transformErr = err return err } defer publicw.Close() publishwriters = append(publishwriters, publicw) } if transformedContentr == nil { // Also write it to the cache fi, metaw, err := r.cache.writeMeta(key, r.transformedResourceMetadata) if err != nil { return err } r.sourceFilename = fi.Name publishwriters = append(publishwriters, metaw) if counter > 0 { transformedContentr = tctx.To.(*bytes.Buffer) } else { transformedContentr = contentrc } } // Also write it to memory var contentmemw *bytes.Buffer if setContent { contentmemw = bp.GetBuffer() defer bp.PutBuffer(contentmemw) publishwriters = append(publishwriters, hugio.ToWriteCloser(contentmemw)) } publishw := hugio.NewMultiWriteCloser(publishwriters...) _, r.transformErr = io.Copy(publishw, transformedContentr) publishw.Close() if setContent { r.contentInit.Do(func() { r.content = contentmemw.String() }) } return nil } func (r *transformedResource) initTransform(setContent, publish bool) error { r.transformInit.Do(func() { r.published = publish if err := r.transform(setContent, publish); err != nil { r.transformErr = err r.cache.rs.Logger.ERROR.Println("error: failed to transform resource:", err) } }) if !publish { return r.transformErr } r.publishInit.Do(func() { if r.published { return } r.published = true // Copy the file from cache to /public _, src, err := r.cache.fileCache.Get(r.sourceFilename) if src == nil { panic(fmt.Sprintf("[BUG] resource cache file not found: %q", r.sourceFilename)) } if err == nil { defer src.Close() var dst io.WriteCloser dst, err = r.openPublishFileForWriting(r.Target) if err == nil { defer dst.Close() io.Copy(dst, src) } } if err != nil { r.transformErr = err r.cache.rs.Logger.ERROR.Println("error: failed to publish resource:", err) return } }) return r.transformErr } // contentReadSeekerCloser returns a ReadSeekerCloser if possible for a given Resource. func contentReadSeekerCloser(r resource.Resource) (hugio.ReadSeekCloser, error) { switch rr := r.(type) { case resource.ReadSeekCloserResource: rc, err := rr.ReadSeekCloser() if err != nil { return nil, err } return rc, nil default: return nil, fmt.Errorf("cannot transform content of Resource of type %T", r) } }