// Copyright 2019 The Hugo Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package resources import ( "fmt" "io" "io/ioutil" "os" "path" "path/filepath" "sync" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/source" "github.com/pkg/errors" "github.com/gohugoio/hugo/common/hugio" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/resource" "github.com/spf13/afero" "github.com/gohugoio/hugo/helpers" ) var ( _ resource.ContentResource = (*genericResource)(nil) _ resource.ReadSeekCloserResource = (*genericResource)(nil) _ resource.Resource = (*genericResource)(nil) _ resource.Source = (*genericResource)(nil) _ resource.Cloner = (*genericResource)(nil) _ resource.ResourcesLanguageMerger = (*resource.Resources)(nil) _ permalinker = (*genericResource)(nil) _ resource.Identifier = (*genericResource)(nil) _ fileInfo = (*genericResource)(nil) ) type ResourceSourceDescriptor struct { // TargetPaths is a callback to fetch paths's relative to its owner. TargetPaths func() page.TargetPaths // Need one of these to load the resource content. SourceFile source.File OpenReadSeekCloser resource.OpenReadSeekCloser FileInfo os.FileInfo // If OpenReadSeekerCloser is not set, we use this to open the file. SourceFilename string Fs afero.Fs // The relative target filename without any language code. RelTargetFilename string // Any base paths prepended to the target path. This will also typically be the // language code, but setting it here means that it should not have any effect on // the permalink. // This may be several values. In multihost mode we may publish the same resources to // multiple targets. TargetBasePaths []string // Delay publishing until either Permalink or RelPermalink is called. Maybe never. LazyPublish bool } func (r ResourceSourceDescriptor) Filename() string { if r.SourceFile != nil { return r.SourceFile.Filename() } return r.SourceFilename } type ResourceTransformer interface { resource.Resource Transformer } type Transformer interface { Transform(...ResourceTransformation) (ResourceTransformer, error) } type baseResourceResource interface { resource.Cloner resource.ContentProvider resource.Resource resource.Identifier } type baseResourceInternal interface { resource.Source fileInfo metaAssigner targetPather ReadSeekCloser() (hugio.ReadSeekCloser, error) // Internal cloneWithUpdates(*transformationUpdate) (baseResource, error) tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser specProvider getResourcePaths() *resourcePathDescriptor getTargetFilenames() []string openDestinationsForWriting() (io.WriteCloser, error) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string } type specProvider interface { getSpec() *Spec } type baseResource interface { baseResourceResource baseResourceInternal } type commonResource struct { } // Slice is not meant to be used externally. It's a bridge function // for the template functions. See collections.Slice. func (commonResource) Slice(in interface{}) (interface{}, error) { switch items := in.(type) { case resource.Resources: return items, nil case []interface{}: groups := make(resource.Resources, len(items)) for i, v := range items { g, ok := v.(resource.Resource) if !ok { return nil, fmt.Errorf("type %T is not a Resource", v) } groups[i] = g { } } return groups, nil default: return nil, fmt.Errorf("invalid slice type %T", items) } } type dirFile struct { // This is the directory component with Unix-style slashes. dir string // This is the file component. file string } func (d dirFile) path() string { return path.Join(d.dir, d.file) } type fileInfo interface { getSourceFilename() string setSourceFilename(string) setSourceFs(afero.Fs) hash() (string, error) size() int } // genericResource represents a generic linkable resource. type genericResource struct { *resourcePathDescriptor *resourceFileInfo *resourceContent spec *Spec title string name string params map[string]interface{} data map[string]interface{} resourceType string mediaType media.Type } func (l *genericResource) Clone() resource.Resource { return l.clone() } func (l *genericResource) Content() (interface{}, error) { if err := l.initContent(); err != nil { return nil, err } return l.content, nil } func (l *genericResource) Data() interface{} { return l.data } func (l *genericResource) Key() string { return l.relTargetDirFile.path() } func (l *genericResource) MediaType() media.Type { return l.mediaType } func (l *genericResource) Name() string { return l.name } func (l *genericResource) Params() map[string]interface{} { return l.params } func (l *genericResource) Permalink() string { return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(l.relTargetDirFile.path(), true), l.spec.BaseURL.HostURL()) } func (l *genericResource) Publish() error { fr, err := l.ReadSeekCloser() if err != nil { return err } defer fr.Close() fw, err := helpers.OpenFilesForWriting(l.spec.BaseFs.PublishFs, l.getTargetFilenames()...) if err != nil { return err } defer fw.Close() _, err = io.Copy(fw, fr) return err } func (l *genericResource) RelPermalink() string { return l.relPermalinkFor(l.relTargetDirFile.path()) } func (l *genericResource) ResourceType() string { return l.resourceType } func (l *genericResource) String() string { return fmt.Sprintf("Resource(%s: %s)", l.resourceType, l.name) } // Path is stored with Unix style slashes. func (l *genericResource) TargetPath() string { return l.relTargetDirFile.path() } func (l *genericResource) Title() string { return l.title } func (l *genericResource) createBasePath(rel string, isURL bool) string { if l.targetPathBuilder == nil { return rel } tp := l.targetPathBuilder() if isURL { return path.Join(tp.SubResourceBaseLink, rel) } // TODO(bep) path return path.Join(filepath.ToSlash(tp.SubResourceBaseTarget), rel) } func (l *genericResource) initContent() error { var err error l.contentInit.Do(func() { var r hugio.ReadSeekCloser r, err = l.ReadSeekCloser() if err != nil { return } defer r.Close() var b []byte b, err = ioutil.ReadAll(r) if err != nil { return } l.content = string(b) }) return err } func (l *genericResource) setName(name string) { l.name = name } func (l *genericResource) getResourcePaths() *resourcePathDescriptor { return l.resourcePathDescriptor } func (l *genericResource) getSpec() *Spec { return l.spec } func (l *genericResource) getTargetFilenames() []string { paths := l.relTargetPaths() for i, p := range paths { paths[i] = filepath.Clean(p) } return paths } func (l *genericResource) setTitle(title string) { l.title = title } func (r *genericResource) tryTransformedFileCache(key string, u *transformationUpdate) io.ReadCloser { fi, f, meta, found := r.spec.ResourceCache.getFromFile(key) if !found { return nil } u.sourceFilename = &fi.Name mt, _ := r.spec.MediaTypes.GetByType(meta.MediaTypeV) u.mediaType = mt u.data = meta.MetaData u.targetPath = meta.Target return f } func (r *genericResource) mergeData(in map[string]interface{}) { if len(in) == 0 { return } if r.data == nil { r.data = make(map[string]interface{}) } for k, v := range in { if _, found := r.data[k]; !found { r.data[k] = v } } } func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResource, error) { r := rc.clone() if u.content != nil { r.contentInit.Do(func() { r.content = *u.content r.openReadSeekerCloser = func() (hugio.ReadSeekCloser, error) { return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil } }) } r.mediaType = u.mediaType if u.sourceFilename != nil { r.setSourceFilename(*u.sourceFilename) } if u.sourceFs != nil { r.setSourceFs(u.sourceFs) } if u.targetPath == "" { return nil, errors.New("missing targetPath") } fpath, fname := path.Split(u.targetPath) r.resourcePathDescriptor.relTargetDirFile = dirFile{dir: fpath, file: fname} r.mergeData(u.data) return r, nil } func (l genericResource) clone() *genericResource { gi := *l.resourceFileInfo rp := *l.resourcePathDescriptor l.resourceFileInfo = &gi l.resourcePathDescriptor = &rp l.resourceContent = &resourceContent{} return &l } // returns an opened file or nil if nothing to write. func (l *genericResource) openDestinationsForWriting() (io.WriteCloser, error) { targetFilenames := l.getTargetFilenames() var changedFilenames []string // Fast path: // This is a processed version of the original; // check if it already existis at the destination. for _, targetFilename := range targetFilenames { if _, err := l.getSpec().BaseFs.PublishFs.Stat(targetFilename); err == nil { continue } changedFilenames = append(changedFilenames, targetFilename) } if len(changedFilenames) == 0 { return nil, nil } return helpers.OpenFilesForWriting(l.getSpec().BaseFs.PublishFs, changedFilenames...) } func (r *genericResource) openPublishFileForWriting(relTargetPath string) (io.WriteCloser, error) { return helpers.OpenFilesForWriting(r.spec.BaseFs.PublishFs, r.relTargetPathsFor(relTargetPath)...) } func (l *genericResource) permalinkFor(target string) string { return l.spec.PermalinkForBaseURL(l.relPermalinkForRel(target, true), l.spec.BaseURL.HostURL()) } func (l *genericResource) relPermalinkFor(target string) string { return l.relPermalinkForRel(target, false) } func (l *genericResource) relPermalinkForRel(rel string, isAbs bool) string { return l.spec.PathSpec.URLizeFilename(l.relTargetPathForRel(rel, false, isAbs, true)) } func (l *genericResource) relTargetPathForRel(rel string, addBaseTargetPath, isAbs, isURL bool) string { if addBaseTargetPath && len(l.baseTargetPathDirs) > 1 { panic("multiple baseTargetPathDirs") } var basePath string if addBaseTargetPath && len(l.baseTargetPathDirs) > 0 { basePath = l.baseTargetPathDirs[0] } return l.relTargetPathForRelAndBasePath(rel, basePath, isAbs, isURL) } func (l *genericResource) relTargetPathForRelAndBasePath(rel, basePath string, isAbs, isURL bool) string { rel = l.createBasePath(rel, isURL) if basePath != "" { rel = path.Join(basePath, rel) } if l.baseOffset != "" { rel = path.Join(l.baseOffset, rel) } if isURL { bp := l.spec.PathSpec.GetBasePath(!isAbs) if bp != "" { rel = path.Join(bp, rel) } } if len(rel) == 0 || rel[0] != '/' { rel = "/" + rel } return rel } func (l *genericResource) relTargetPaths() []string { return l.relTargetPathsForRel(l.TargetPath()) } func (l *genericResource) relTargetPathsFor(target string) []string { return l.relTargetPathsForRel(target) } func (l *genericResource) relTargetPathsForRel(rel string) []string { if len(l.baseTargetPathDirs) == 0 { return []string{l.relTargetPathForRelAndBasePath(rel, "", false, false)} } targetPaths := make([]string, len(l.baseTargetPathDirs)) for i, dir := range l.baseTargetPathDirs { targetPaths[i] = l.relTargetPathForRelAndBasePath(rel, dir, false, false) } return targetPaths } func (l *genericResource) updateParams(params map[string]interface{}) { if l.params == nil { l.params = params return } // Sets the params not already set for k, v := range params { if _, found := l.params[k]; !found { l.params[k] = v } } } type targetPather interface { TargetPath() string } type permalinker interface { targetPather permalinkFor(target string) string relPermalinkFor(target string) string relTargetPaths() []string relTargetPathsFor(target string) []string } type resourceContent struct { content string contentInit sync.Once } type resourceFileInfo struct { // Will be set if this resource is backed by something other than a file. openReadSeekerCloser resource.OpenReadSeekCloser // This may be set to tell us to look in another filesystem for this resource. // We, by default, use the sourceFs filesystem in the spec below. sourceFs afero.Fs // Absolute filename to the source, including any content folder path. // Note that this is absolute in relation to the filesystem it is stored in. // It can be a base path filesystem, and then this filename will not match // the path to the file on the real filesystem. sourceFilename string fi os.FileInfo // A hash of the source content. Is only calculated in caching situations. h *resourceHash } func (fi *resourceFileInfo) ReadSeekCloser() (hugio.ReadSeekCloser, error) { if fi.openReadSeekerCloser != nil { return fi.openReadSeekerCloser() } f, err := fi.getSourceFs().Open(fi.getSourceFilename()) if err != nil { return nil, err } return f, nil } func (fi *resourceFileInfo) getSourceFilename() string { return fi.sourceFilename } func (fi *resourceFileInfo) setSourceFilename(s string) { // Make sure it's always loaded by sourceFilename. fi.openReadSeekerCloser = nil fi.sourceFilename = s } func (fi *resourceFileInfo) getSourceFs() afero.Fs { return fi.sourceFs } func (fi *resourceFileInfo) setSourceFs(fs afero.Fs) { fi.sourceFs = fs } func (fi *resourceFileInfo) hash() (string, error) { var err error fi.h.init.Do(func() { var hash string var f hugio.ReadSeekCloser f, err = fi.ReadSeekCloser() if err != nil { err = errors.Wrap(err, "failed to open source file") return } defer f.Close() hash, err = helpers.MD5FromFileFast(f) if err != nil { return } fi.h.value = hash }) return fi.h.value, err } func (fi *resourceFileInfo) size() int { if fi.fi == nil { return 0 } return int(fi.fi.Size()) } type resourceHash struct { value string init sync.Once } type resourcePathDescriptor struct { // The relative target directory and filename. relTargetDirFile dirFile // Callback used to construct a target path relative to its owner. targetPathBuilder func() page.TargetPaths // This will normally be the same as above, but this will only apply to publishing // of resources. It may be mulltiple values when in multihost mode. baseTargetPathDirs []string // baseOffset is set when the output format's path has a offset, e.g. for AMP. baseOffset string }