hugo/tpl/tplimpl/template.go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tplimpl

import (
	"io"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/gohugoio/hugo/common/types"

	"github.com/gohugoio/hugo/helpers"

	"github.com/gohugoio/hugo/output"

	"github.com/gohugoio/hugo/deps"
	"github.com/spf13/afero"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/hugofs/files"
	"github.com/pkg/errors"

	"github.com/gohugoio/hugo/tpl/tplimpl/embedded"

	htmltemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
	texttemplate "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate"

	"github.com/gohugoio/hugo/identity"
	"github.com/gohugoio/hugo/tpl"
)

const (
	textTmplNamePrefix = "_text/"

	shortcodesPathPrefix = "shortcodes/"
	internalPathPrefix   = "_internal/"
	baseFileBase         = "baseof"
)

// The identifiers may be truncated in the log, e.g.
// "executing "main" at <$scaled.SRelPermalin...>: can't evaluate field SRelPermalink in type *resource.Image"
var identifiersRe = regexp.MustCompile(`at \<(.*?)(\.{3})?\>:`)

var embeddedTemplatesAliases = map[string][]string{
	"shortcodes/twitter.html": {"shortcodes/tweet.html"},
}

var (
	_ tpl.TemplateManager    = (*templateExec)(nil)
	_ tpl.TemplateHandler    = (*templateExec)(nil)
	_ tpl.TemplateFuncGetter = (*templateExec)(nil)
	_ tpl.TemplateFinder     = (*templateExec)(nil)

	_ tpl.Template = (*templateState)(nil)
	_ tpl.Info     = (*templateState)(nil)
)

var defineRe = regexp.MustCompile(`{{-?\s?define`)

func newIdentity(name string) identity.Manager {
	return identity.NewManager(identity.NewPathIdentity(files.ComponentFolderLayouts, name))
}

func newStandaloneTextTemplate(funcs map[string]interface{}) tpl.TemplateParseFinder {
	return &textTemplateWrapperWithLock{
		RWMutex:  &sync.RWMutex{},
		Template: texttemplate.New("").Funcs(funcs),
	}
}

func newTemplateExec(d *deps.Deps) (*templateExec, error) {
	exec, funcs := newTemplateExecuter(d)
	funcMap := make(map[string]interface{})
	for k, v := range funcs {
		funcMap[k] = v.Interface()
	}

	h := &templateHandler{
		nameBaseTemplateName: make(map[string]string),
		transformNotFound:    make(map[string]*templateState),
		identityNotFound:     make(map[string][]identity.Manager),

		shortcodes:   make(map[string]*shortcodeTemplates),
		templateInfo: make(map[string]tpl.Info),
		baseof:       make(map[string]templateInfo),
		needsBaseof:  make(map[string]templateInfo),

		main: newTemplateNamespace(funcMap, false),

		Deps:                d,
		layoutHandler:       output.NewLayoutHandler(),
		layoutsFs:           d.BaseFs.Layouts.Fs,
		layoutTemplateCache: make(map[layoutCacheKey]tpl.Template),
	}

	if err := h.loadEmbedded(); err != nil {
		return nil, err
	}

	if err := h.loadTemplates(); err != nil {
		return nil, err
	}

	e := &templateExec{
		d:               d,
		executor:        exec,
		funcs:           funcs,
		templateHandler: h,
	}

	d.SetTmpl(e)
	d.SetTextTmpl(newStandaloneTextTemplate(funcMap))

	if d.WithTemplate != nil {
		if err := d.WithTemplate(e); err != nil {
			return nil, err

		}
	}

	return e, nil
}

func newTemplateNamespace(funcs map[string]interface{}, lock bool) *templateNamespace {
	var mu *sync.RWMutex
	if lock {
		mu = &sync.RWMutex{}
	}

	return &templateNamespace{
		prototypeHTML: htmltemplate.New("").Funcs(funcs),
		prototypeText: texttemplate.New("").Funcs(funcs),
		templateStateMap: &templateStateMap{
			mu:        mu,
			templates: make(map[string]*templateState),
		},
	}
}

func newTemplateState(templ tpl.Template, info templateInfo) *templateState {
	return &templateState{
		info:      info,
		typ:       info.resolveType(),
		Template:  templ,
		Manager:   newIdentity(info.name),
		parseInfo: tpl.DefaultParseInfo,
	}
}

type layoutCacheKey struct {
	d output.LayoutDescriptor
	f string
}

type templateExec struct {
	d        *deps.Deps
	executor texttemplate.Executer
	funcs    map[string]reflect.Value

	*templateHandler
}

func (t templateExec) Clone(d *deps.Deps) *templateExec {
	exec, funcs := newTemplateExecuter(d)
	t.executor = exec
	t.funcs = funcs
	t.d = d
	return &t
}

func (t *templateExec) Execute(templ tpl.Template, wr io.Writer, data interface{}) error {
	if rlocker, ok := templ.(types.RLocker); ok {
		rlocker.RLock()
		defer rlocker.RUnlock()
	}
	if t.Metrics != nil {
		defer t.Metrics.MeasureSince(templ.Name(), time.Now())
	}

	execErr := t.executor.Execute(templ, wr, data)
	if execErr != nil {
		execErr = t.addFileContext(templ, execErr)
	}
	return execErr
}

func (t *templateExec) GetFunc(name string) (reflect.Value, bool) {
	v, found := t.funcs[name]
	return v, found
}

func (t *templateExec) MarkReady() error {
	var err error
	t.readyInit.Do(func() {
		// We only need the clones if base templates are in use.
		if len(t.needsBaseof) > 0 {
			err = t.main.createPrototypes()
		}
	})

	if err != nil {
		return err
	}

	if t.Deps.BuildFlags.HasLateTemplate.Load() {
		// This costs memory, so try to avoid it if we don't have to.
		// The late templates are used to handle HTML in files in /content
		// without front matter.
		t.readyLateInit.Do(func() {
			t.late = t.main.Clone(true)
			t.late.createPrototypes()
		})
	}

	return nil
}

type templateHandler struct {
	main        *templateNamespace
	needsBaseof map[string]templateInfo
	baseof      map[string]templateInfo

	late *templateNamespace // Templates added after main has started executing.

	readyInit     sync.Once
	readyLateInit sync.Once

	// This is the filesystem to load the templates from. All the templates are
	// stored in the root of this filesystem.
	layoutsFs afero.Fs

	layoutHandler *output.LayoutHandler

	layoutTemplateCache   map[layoutCacheKey]tpl.Template
	layoutTemplateCacheMu sync.RWMutex

	*deps.Deps

	// Used to get proper filenames in errors
	nameBaseTemplateName map[string]string

	// Holds name and source of template definitions not found during the first
	// AST transformation pass.
	transformNotFound map[string]*templateState

	// Holds identities of templates not found during first pass.
	identityNotFound map[string][]identity.Manager

	// shortcodes maps shortcode name to template variants
	// (language, output format etc.) of that shortcode.
	shortcodes map[string]*shortcodeTemplates

	// templateInfo maps template name to some additional information about that template.
	// Note that for shortcodes that same information is embedded in the
	// shortcodeTemplates type.
	templateInfo map[string]tpl.Info
}

// AddLateTemplate is used to add a template after the
// regular templates have started its execution.
// These are currently "pure HTML content files".
func (t *templateHandler) AddLateTemplate(name, tpl string) error {
	_, err := t.late.parse(t.newTemplateInfo(name, tpl))
	return err
}

// AddTemplate parses and adds a template to the collection.
// Templates with name prefixed with "_text" will be handled as plain
// text templates.
func (t *templateHandler) AddTemplate(name, tpl string) error {
	templ, err := t.addTemplateTo(t.newTemplateInfo(name, tpl), t.main)
	if err == nil {
		t.applyTemplateTransformers(t.main, templ)
	}
	return err
}

func (t *templateHandler) Lookup(name string) (tpl.Template, bool) {
	templ, found := t.main.Lookup(name)
	if found {
		return templ, true
	}

	if t.late != nil {
		return t.late.Lookup(name)
	}

	return nil, false
}

func (t *templateHandler) LookupLayout(d output.LayoutDescriptor, f output.Format) (tpl.Template, bool, error) {
	key := layoutCacheKey{d, f.Name}
	t.layoutTemplateCacheMu.RLock()
	if cacheVal, found := t.layoutTemplateCache[key]; found {
		t.layoutTemplateCacheMu.RUnlock()
		return cacheVal, true, nil
	}
	t.layoutTemplateCacheMu.RUnlock()

	t.layoutTemplateCacheMu.Lock()
	defer t.layoutTemplateCacheMu.Unlock()

	templ, found, err := t.findLayout(d, f)
	if err == nil && found {
		t.layoutTemplateCache[key] = templ
		return templ, true, nil
	}

	return nil, false, err
}

// This currently only applies to shortcodes and what we get here is the
// shortcode name.
func (t *templateHandler) LookupVariant(name string, variants tpl.TemplateVariants) (tpl.Template, bool, bool) {
	name = templateBaseName(templateShortcode, name)
	s, found := t.shortcodes[name]
	if !found {
		return nil, false, false
	}

	sv, found := s.fromVariants(variants)
	if !found {
		return nil, false, false
	}

	more := len(s.variants) > 1

	return sv.ts, true, more

}

func (t *templateHandler) HasTemplate(name string) bool {
	if _, found := t.baseof[name]; found {
		return true
	}
	_, found := t.Lookup(name)
	return found
}

func (t *templateHandler) findLayout(d output.LayoutDescriptor, f output.Format) (tpl.Template, bool, error) {
	layouts, _ := t.layoutHandler.For(d, f)
	for _, name := range layouts {
		templ, found := t.main.Lookup(name)
		if found {
			return templ, true, nil
		}

		overlay, found := t.needsBaseof[name]

		if !found {
			continue
		}

		d.Baseof = true
		baseLayouts, _ := t.layoutHandler.For(d, f)
		var base templateInfo
		found = false
		for _, l := range baseLayouts {
			base, found = t.baseof[l]
			if found {
				break
			}
		}

		if !found {
			return nil, false, errors.Errorf("no baseof layout found for %q:", name)
		}

		templ, err := t.applyBaseTemplate(overlay, base)
		if err != nil {
			return nil, false, err
		}

		ts := newTemplateState(templ, overlay)
		ts.baseInfo = base

		// Add the base identity to detect changes
		ts.Add(identity.NewPathIdentity(files.ComponentFolderLayouts, base.name))

		t.applyTemplateTransformers(t.main, ts)

		return ts, true, nil

	}

	return nil, false, nil
}

func (t *templateHandler) findTemplate(name string) *templateState {
	if templ, found := t.Lookup(name); found {
		return templ.(*templateState)
	}
	return nil
}

func (t *templateHandler) newTemplateInfo(name, tpl string) templateInfo {
	var isText bool
	name, isText = t.nameIsText(name)
	return templateInfo{
		name:     name,
		isText:   isText,
		template: tpl,
	}
}

func (t *templateHandler) addFileContext(templ tpl.Template, inerr error) error {
	if strings.HasPrefix(templ.Name(), "_internal") {
		return inerr
	}

	ts, ok := templ.(*templateState)
	if !ok {
		return inerr
	}

	//lint:ignore ST1008 the error is the main result
	checkFilename := func(info templateInfo, inErr error) (error, bool) {
		if info.filename == "" {
			return inErr, false
		}

		lineMatcher := func(m herrors.LineMatcher) bool {
			if m.Position.LineNumber != m.LineNumber {
				return false
			}

			identifiers := t.extractIdentifiers(m.Error.Error())

			for _, id := range identifiers {
				if strings.Contains(m.Line, id) {
					return true
				}
			}
			return false
		}

		f, err := t.layoutsFs.Open(info.filename)
		if err != nil {
			return inErr, false
		}
		defer f.Close()

		fe, ok := herrors.WithFileContext(inErr, info.realFilename, f, lineMatcher)
		if ok {
			return fe, true
		}
		return inErr, false
	}

	inerr = errors.Wrap(inerr, "execute of template failed")

	if err, ok := checkFilename(ts.info, inerr); ok {
		return err
	}

	err, _ := checkFilename(ts.baseInfo, inerr)

	return err

}

func (t *templateHandler) addShortcodeVariant(ts *templateState) {
	name := ts.Name()
	base := templateBaseName(templateShortcode, name)

	shortcodename, variants := templateNameAndVariants(base)

	templs, found := t.shortcodes[shortcodename]
	if !found {
		templs = &shortcodeTemplates{}
		t.shortcodes[shortcodename] = templs
	}

	sv := shortcodeVariant{variants: variants, ts: ts}

	i := templs.indexOf(variants)

	if i != -1 {
		// Only replace if it's an override of an internal template.
		if !isInternal(name) {
			templs.variants[i] = sv
		}
	} else {
		templs.variants = append(templs.variants, sv)
	}
}

func (t *templateHandler) addTemplateFile(name, path string) error {
	getTemplate := func(filename string) (templateInfo, error) {
		fs := t.Layouts.Fs
		b, err := afero.ReadFile(fs, filename)
		if err != nil {
			return templateInfo{filename: filename, fs: fs}, err
		}

		s := removeLeadingBOM(string(b))

		realFilename := filename
		if fi, err := fs.Stat(filename); err == nil {
			if fim, ok := fi.(hugofs.FileMetaInfo); ok {
				realFilename = fim.Meta().Filename()
			}
		}

		var isText bool
		name, isText = t.nameIsText(name)

		return templateInfo{
			name:         name,
			isText:       isText,
			template:     s,
			filename:     filename,
			realFilename: realFilename,
			fs:           fs,
		}, nil
	}

	tinfo, err := getTemplate(path)
	if err != nil {
		return err
	}

	if isBaseTemplate(name) {
		// Store it for later.
		t.baseof[name] = tinfo
		return nil
	}

	needsBaseof := !t.noBaseNeeded(name) && defineRe.MatchString(tinfo.template)
	if needsBaseof {
		t.needsBaseof[name] = tinfo
		return nil
	}

	templ, err := t.addTemplateTo(tinfo, t.main)
	if err != nil {
		return tinfo.errWithFileContext("parse failed", err)
	}
	t.applyTemplateTransformers(t.main, templ)

	return nil

}

func (t *templateHandler) addTemplateTo(info templateInfo, to *templateNamespace) (*templateState, error) {
	return to.parse(info)
}

func (t *templateHandler) applyBaseTemplate(overlay, base templateInfo) (tpl.Template, error) {
	if overlay.isText {
		templ, err := t.main.prototypeTextClone.New(overlay.name).Parse(base.template)
		if err != nil {
			return nil, base.errWithFileContext("parse failed", err)
		}
		templ, err = templ.Parse(overlay.template)
		if err != nil {
			return nil, overlay.errWithFileContext("parse failed", err)
		}
		return templ, nil
	}

	templ, err := t.main.prototypeHTMLClone.New(overlay.name).Parse(base.template)
	if err != nil {
		return nil, base.errWithFileContext("parse failed", err)
	}

	templ, err = htmltemplate.Must(templ.Clone()).Parse(overlay.template)
	if err != nil {
		return nil, overlay.errWithFileContext("parse failed", err)
	}

	// The extra lookup is a workaround, see
	// * https://github.com/golang/go/issues/16101
	// * https://github.com/gohugoio/hugo/issues/2549
	templ = templ.Lookup(templ.Name())

	return templ, err
}

func (t *templateHandler) applyTemplateTransformers(ns *templateNamespace, ts *templateState) (*templateContext, error) {
	c, err := applyTemplateTransformers(ts, ns.newTemplateLookup(ts))
	if err != nil {
		return nil, err
	}

	for k := range c.templateNotFound {
		t.transformNotFound[k] = ts
		t.identityNotFound[k] = append(t.identityNotFound[k], c.t)
	}

	for k := range c.identityNotFound {
		t.identityNotFound[k] = append(t.identityNotFound[k], c.t)
	}

	return c, err
}

func (t *templateHandler) extractIdentifiers(line string) []string {
	m := identifiersRe.FindAllStringSubmatch(line, -1)
	identifiers := make([]string, len(m))
	for i := 0; i < len(m); i++ {
		identifiers[i] = m[i][1]
	}
	return identifiers
}

func (t *templateHandler) loadEmbedded() error {
	for _, kv := range embedded.EmbeddedTemplates {
		name, templ := kv[0], kv[1]
		if err := t.AddTemplate(internalPathPrefix+name, templ); err != nil {
			return err
		}
		if aliases, found := embeddedTemplatesAliases[name]; found {
			// TODO(bep) avoid reparsing these aliases
			for _, alias := range aliases {
				alias = internalPathPrefix + alias
				if err := t.AddTemplate(alias, templ); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func (t *templateHandler) loadTemplates() error {
	walker := func(path string, fi hugofs.FileMetaInfo, err error) error {
		if err != nil || fi.IsDir() {
			return err
		}

		if isDotFile(path) || isBackupFile(path) {
			return nil
		}

		name := strings.TrimPrefix(filepath.ToSlash(path), "/")
		filename := filepath.Base(path)
		outputFormat, found := t.OutputFormatsConfig.FromFilename(filename)

		if found && outputFormat.IsPlainText {
			name = textTmplNamePrefix + name
		}

		if err := t.addTemplateFile(name, path); err != nil {
			return err
		}

		return nil
	}

	if err := helpers.SymbolicWalk(t.Layouts.Fs, "", walker); err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		return nil
	}

	return nil

}

func (t *templateHandler) nameIsText(name string) (string, bool) {
	isText := strings.HasPrefix(name, textTmplNamePrefix)
	if isText {
		name = strings.TrimPrefix(name, textTmplNamePrefix)
	}
	return name, isText
}

func (t *templateHandler) noBaseNeeded(name string) bool {
	if strings.HasPrefix(name, "shortcodes/") || strings.HasPrefix(name, "partials/") {
		return true
	}
	return strings.Contains(name, "_markup/")
}

func (t *templateHandler) postTransform() error {
	for _, v := range t.main.templates {
		if v.typ == templateShortcode {
			t.addShortcodeVariant(v)
		}
	}

	for name, source := range t.transformNotFound {
		lookup := t.main.newTemplateLookup(source)
		templ := lookup(name)
		if templ != nil {
			_, err := applyTemplateTransformers(templ, lookup)
			if err != nil {
				return err
			}
		}
	}

	for k, v := range t.identityNotFound {
		ts := t.findTemplate(k)
		if ts != nil {
			for _, im := range v {
				im.Add(ts)
			}
		}
	}

	return nil
}

type templateNamespace struct {
	prototypeText      *texttemplate.Template
	prototypeHTML      *htmltemplate.Template
	prototypeTextClone *texttemplate.Template
	prototypeHTMLClone *htmltemplate.Template

	*templateStateMap
}

func (t templateNamespace) Clone(lock bool) *templateNamespace {
	if t.mu != nil {
		t.mu.Lock()
		defer t.mu.Unlock()
	}

	var mu *sync.RWMutex
	if lock {
		mu = &sync.RWMutex{}
	}

	t.templateStateMap = &templateStateMap{
		templates: make(map[string]*templateState),
		mu:        mu,
	}

	t.prototypeText = texttemplate.Must(t.prototypeText.Clone())
	t.prototypeHTML = htmltemplate.Must(t.prototypeHTML.Clone())

	return &t
}

func (t *templateNamespace) Lookup(name string) (tpl.Template, bool) {
	if t.mu != nil {
		t.mu.RLock()
		defer t.mu.RLock()
	}

	templ, found := t.templates[name]
	if !found {
		return nil, false
	}

	if t.mu != nil {
		return &templateWrapperWithLock{RWMutex: t.mu, Template: templ}, true
	}

	return templ, found
}

func (t *templateNamespace) createPrototypes() error {
	t.prototypeTextClone = texttemplate.Must(t.prototypeText.Clone())
	t.prototypeHTMLClone = htmltemplate.Must(t.prototypeHTML.Clone())

	return nil
}

func (t *templateNamespace) newTemplateLookup(in *templateState) func(name string) *templateState {
	return func(name string) *templateState {
		if templ, found := t.templates[name]; found {
			if templ.isText() != in.isText() {
				return nil
			}
			return templ
		}
		if templ, found := findTemplateIn(name, in); found {
			return newTemplateState(templ, templateInfo{name: templ.Name()})
		}
		return nil

	}
}

func (t *templateNamespace) parse(info templateInfo) (*templateState, error) {
	if t.mu != nil {
		t.mu.Lock()
		defer t.mu.Unlock()
	}

	if info.isText {
		prototype := t.prototypeText

		templ, err := prototype.New(info.name).Parse(info.template)
		if err != nil {
			return nil, err
		}

		ts := newTemplateState(templ, info)

		t.templates[info.name] = ts

		return ts, nil
	}

	prototype := t.prototypeHTML

	templ, err := prototype.New(info.name).Parse(info.template)
	if err != nil {
		return nil, err
	}

	ts := newTemplateState(templ, info)

	t.templates[info.name] = ts

	return ts, nil
}

type templateState struct {
	tpl.Template

	typ       templateType
	parseInfo tpl.ParseInfo
	identity.Manager

	info     templateInfo
	baseInfo templateInfo // Set when a base template is used.
}

func (t *templateState) ParseInfo() tpl.ParseInfo {
	return t.parseInfo
}

func (t *templateState) isText() bool {
	_, isText := t.Template.(*texttemplate.Template)
	return isText
}

type templateStateMap struct {
	mu        *sync.RWMutex // May be nil
	templates map[string]*templateState
}

type templateWrapperWithLock struct {
	*sync.RWMutex
	tpl.Template
}

type textTemplateWrapperWithLock struct {
	*sync.RWMutex
	*texttemplate.Template
}

func (t *textTemplateWrapperWithLock) Lookup(name string) (tpl.Template, bool) {
	t.RLock()
	templ := t.Template.Lookup(name)
	t.RUnlock()
	if templ == nil {
		return nil, false
	}
	return &textTemplateWrapperWithLock{
		RWMutex:  t.RWMutex,
		Template: templ,
	}, true
}

func (t *textTemplateWrapperWithLock) LookupVariant(name string, variants tpl.TemplateVariants) (tpl.Template, bool, bool) {
	panic("not supported")
}

func (t *textTemplateWrapperWithLock) Parse(name, tpl string) (tpl.Template, error) {
	t.Lock()
	defer t.Unlock()
	return t.Template.New(name).Parse(tpl)
}

func isBackupFile(path string) bool {
	return path[len(path)-1] == '~'
}

func isBaseTemplate(path string) bool {
	return strings.Contains(filepath.Base(path), baseFileBase)
}

func isDotFile(path string) bool {
	return filepath.Base(path)[0] == '.'
}

func removeLeadingBOM(s string) string {
	const bom = '\ufeff'

	for i, r := range s {
		if i == 0 && r != bom {
			return s
		}
		if i > 0 {
			return s[i:]
		}
	}

	return s

}

// resolves _internal/shortcodes/param.html => param.html etc.
func templateBaseName(typ templateType, name string) string {
	name = strings.TrimPrefix(name, internalPathPrefix)
	switch typ {
	case templateShortcode:
		return strings.TrimPrefix(name, shortcodesPathPrefix)
	default:
		panic("not implemented")
	}

}

func unwrap(templ tpl.Template) tpl.Template {
	if ts, ok := templ.(*templateState); ok {
		return ts.Template
	}
	return templ
}