tpl/internal: Synch Go templates fork with Go 1.16dev

This commit is contained in:
Bjørn Erik Pedersen 2020-12-03 13:50:17 +01:00
parent 66beac99c6
commit cf3e077da3
25 changed files with 2520 additions and 137 deletions

View file

@ -18,7 +18,7 @@ import (
func main() { func main() {
// TODO(bep) git checkout tag // TODO(bep) git checkout tag
// The current is built with Go version b68fa57c599720d33a2d735782969ce95eabf794 / go1.15dev // The current is built with Go version da54dfb6a1f3bef827b9ec3780c98fde77a97d11 / go1.16dev
fmt.Println("Forking ...") fmt.Println("Forking ...")
defer fmt.Println("Done ...") defer fmt.Println("Done ...")

View file

@ -58,6 +58,7 @@ const KnownEnv = `
GOSUMDB GOSUMDB
GOTMPDIR GOTMPDIR
GOTOOLDIR GOTOOLDIR
GOVCS
GOWASM GOWASM
GO_EXTLINK_ENABLED GO_EXTLINK_ENABLED
PKG_CONFIG PKG_CONFIG

View file

@ -130,7 +130,7 @@ func compare(aVal, bVal reflect.Value) int {
default: default:
return -1 return -1
} }
case reflect.Ptr: case reflect.Ptr, reflect.UnsafePointer:
a, b := aVal.Pointer(), bVal.Pointer() a, b := aVal.Pointer(), bVal.Pointer()
switch { switch {
case a < b: case a < b:

View file

@ -11,6 +11,7 @@ import (
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"unsafe"
) )
var compareTests = [][]reflect.Value{ var compareTests = [][]reflect.Value{
@ -32,6 +33,7 @@ var compareTests = [][]reflect.Value{
ct(reflect.TypeOf(complex128(0+1i)), -1-1i, -1+0i, -1+1i, 0-1i, 0+0i, 0+1i, 1-1i, 1+0i, 1+1i), ct(reflect.TypeOf(complex128(0+1i)), -1-1i, -1+0i, -1+1i, 0-1i, 0+0i, 0+1i, 1-1i, 1+0i, 1+1i),
ct(reflect.TypeOf(false), false, true), ct(reflect.TypeOf(false), false, true),
ct(reflect.TypeOf(&ints[0]), &ints[0], &ints[1], &ints[2]), ct(reflect.TypeOf(&ints[0]), &ints[0], &ints[1], &ints[2]),
ct(reflect.TypeOf(unsafe.Pointer(&ints[0])), unsafe.Pointer(&ints[0]), unsafe.Pointer(&ints[1]), unsafe.Pointer(&ints[2])),
ct(reflect.TypeOf(chans[0]), chans[0], chans[1], chans[2]), ct(reflect.TypeOf(chans[0]), chans[0], chans[1], chans[2]),
ct(reflect.TypeOf(toy{}), toy{0, 1}, toy{0, 2}, toy{1, -1}, toy{1, 1}), ct(reflect.TypeOf(toy{}), toy{0, 1}, toy{0, 2}, toy{1, -1}, toy{1, 1}),
ct(reflect.TypeOf([2]int{}), [2]int{1, 1}, [2]int{1, 2}, [2]int{2, 0}), ct(reflect.TypeOf([2]int{}), [2]int{1, 1}, [2]int{1, 2}, [2]int{2, 0}),
@ -118,6 +120,10 @@ var sortTests = []sortTest{
pointerMap(), pointerMap(),
"PTR0:0 PTR1:1 PTR2:2", "PTR0:0 PTR1:1 PTR2:2",
}, },
{
unsafePointerMap(),
"UNSAFEPTR0:0 UNSAFEPTR1:1 UNSAFEPTR2:2",
},
{ {
map[toy]string{{7, 2}: "72", {7, 1}: "71", {3, 4}: "34"}, map[toy]string{{7, 2}: "72", {7, 1}: "71", {3, 4}: "34"},
"{3 4}:34 {7 1}:71 {7 2}:72", "{3 4}:34 {7 1}:71 {7 2}:72",
@ -159,6 +165,14 @@ func sprintKey(key reflect.Value) string {
} }
} }
return "PTR???" return "PTR???"
case "unsafe.Pointer":
ptr := key.Interface().(unsafe.Pointer)
for i := range ints {
if ptr == unsafe.Pointer(&ints[i]) {
return fmt.Sprintf("UNSAFEPTR%d", i)
}
}
return "UNSAFEPTR???"
case "chan int": case "chan int":
c := key.Interface().(chan int) c := key.Interface().(chan int)
for i := range chans { for i := range chans {
@ -185,6 +199,14 @@ func pointerMap() map[*int]string {
return m return m
} }
func unsafePointerMap() map[unsafe.Pointer]string {
m := make(map[unsafe.Pointer]string)
for i := 2; i >= 0; i-- {
m[unsafe.Pointer(&ints[i])] = fmt.Sprint(i)
}
return m
}
func chanMap() map[chan int]string { func chanMap() map[chan int]string {
m := make(map[chan int]string) m := make(map[chan int]string)
for i := 2; i >= 0; i-- { for i := 2; i >= 0; i-- {

View file

@ -10,7 +10,7 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io"
"strings" "strings"
"sync" "sync"
"testing" "testing"
@ -18,7 +18,7 @@ import (
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
) )
func TestAddParseTree(t *testing.T) { func TestAddParseTreeHTML(t *testing.T) {
root := Must(New("root").Parse(`{{define "a"}} {{.}} {{template "b"}} {{.}} "></a>{{end}}`)) root := Must(New("root").Parse(`{{define "a"}} {{.}} {{template "b"}} {{.}} "></a>{{end}}`))
tree, err := parse.Parse("t", `{{define "b"}}<a href="{{end}}`, "", "", nil, nil) tree, err := parse.Parse("t", `{{define "b"}}<a href="{{end}}`, "", "", nil, nil)
if err != nil { if err != nil {
@ -174,7 +174,7 @@ func TestCloneThenParse(t *testing.T) {
t.Error("adding a template to a clone added it to the original") t.Error("adding a template to a clone added it to the original")
} }
// double check that the embedded template isn't available in the original // double check that the embedded template isn't available in the original
err := t0.ExecuteTemplate(ioutil.Discard, "a", nil) err := t0.ExecuteTemplate(io.Discard, "a", nil)
if err == nil { if err == nil {
t.Error("expected 'no such template' error") t.Error("expected 'no such template' error")
} }
@ -188,13 +188,13 @@ func TestFuncMapWorksAfterClone(t *testing.T) {
// get the expected error output (no clone) // get the expected error output (no clone)
uncloned := Must(New("").Funcs(funcs).Parse("{{customFunc}}")) uncloned := Must(New("").Funcs(funcs).Parse("{{customFunc}}"))
wantErr := uncloned.Execute(ioutil.Discard, nil) wantErr := uncloned.Execute(io.Discard, nil)
// toClone must be the same as uncloned. It has to be recreated from scratch, // toClone must be the same as uncloned. It has to be recreated from scratch,
// since cloning cannot occur after execution. // since cloning cannot occur after execution.
toClone := Must(New("").Funcs(funcs).Parse("{{customFunc}}")) toClone := Must(New("").Funcs(funcs).Parse("{{customFunc}}"))
cloned := Must(toClone.Clone()) cloned := Must(toClone.Clone())
gotErr := cloned.Execute(ioutil.Discard, nil) gotErr := cloned.Execute(io.Discard, nil)
if wantErr.Error() != gotErr.Error() { if wantErr.Error() != gotErr.Error() {
t.Errorf("clone error message mismatch want %q got %q", wantErr, gotErr) t.Errorf("clone error message mismatch want %q got %q", wantErr, gotErr)
@ -216,7 +216,7 @@ func TestTemplateCloneExecuteRace(t *testing.T) {
go func() { go func() {
defer wg.Done() defer wg.Done()
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
if err := tmpl.Execute(ioutil.Discard, "data"); err != nil { if err := tmpl.Execute(io.Discard, "data"); err != nil {
panic(err) panic(err)
} }
} }
@ -240,7 +240,7 @@ func TestCloneGrowth(t *testing.T) {
tmpl = Must(tmpl.Clone()) tmpl = Must(tmpl.Clone())
Must(tmpl.Parse(`{{define "B"}}Text{{end}}`)) Must(tmpl.Parse(`{{define "B"}}Text{{end}}`))
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
tmpl.Execute(ioutil.Discard, nil) tmpl.Execute(io.Discard, nil)
} }
if len(tmpl.DefinedTemplates()) > 200 { if len(tmpl.DefinedTemplates()) > 200 {
t.Fatalf("too many templates: %v", len(tmpl.DefinedTemplates())) t.Fatalf("too many templates: %v", len(tmpl.DefinedTemplates()))
@ -260,7 +260,7 @@ func TestCloneRedefinedName(t *testing.T) {
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
t2 := Must(t1.Clone()) t2 := Must(t1.Clone())
t2 = Must(t2.New(fmt.Sprintf("%d", i)).Parse(page)) t2 = Must(t2.New(fmt.Sprintf("%d", i)).Parse(page))
err := t2.Execute(ioutil.Discard, nil) err := t2.Execute(io.Discard, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View file

@ -404,11 +404,11 @@ func TestTypedContent(t *testing.T) {
} }
// Test that we print using the String method. Was issue 3073. // Test that we print using the String method. Was issue 3073.
type stringer struct { type myStringer struct {
v int v int
} }
func (s *stringer) String() string { func (s *myStringer) String() string {
return fmt.Sprintf("string=%d", s.v) return fmt.Sprintf("string=%d", s.v)
} }
@ -421,7 +421,7 @@ func (s *errorer) Error() string {
} }
func TestStringer(t *testing.T) { func TestStringer(t *testing.T) {
s := &stringer{3} s := &myStringer{3}
b := new(bytes.Buffer) b := new(bytes.Buffer)
tmpl := Must(New("x").Parse("{{.}}")) tmpl := Must(New("x").Parse("{{.}}"))
if err := tmpl.Execute(b, s); err != nil { if err := tmpl.Execute(b, s); err != nil {

View file

@ -125,6 +125,8 @@ func (e *escaper) escape(c context, n parse.Node) context {
switch n := n.(type) { switch n := n.(type) {
case *parse.ActionNode: case *parse.ActionNode:
return e.escapeAction(c, n) return e.escapeAction(c, n)
case *parse.CommentNode:
return c
case *parse.IfNode: case *parse.IfNode:
return e.escapeBranch(c, &n.BranchNode, "if") return e.escapeBranch(c, &n.BranchNode, "if")
case *parse.ListNode: case *parse.ListNode:

View file

@ -1825,7 +1825,7 @@ func TestIndirectPrint(t *testing.T) {
} }
// This is a test for issue 3272. // This is a test for issue 3272.
func TestEmptyTemplate(t *testing.T) { func TestEmptyTemplateHTML(t *testing.T) {
page := Must(New("page").ParseFiles(os.DevNull)) page := Must(New("page").ParseFiles(os.DevNull))
if err := page.ExecuteTemplate(os.Stdout, "page", "nothing"); err == nil { if err := page.ExecuteTemplate(os.Stdout, "page", "nothing"); err == nil {
t.Fatal("expected error") t.Fatal("expected error")

File diff suppressed because it is too large Load diff

View file

@ -240,8 +240,7 @@ func htmlNameFilter(args ...interface{}) string {
} }
s = strings.ToLower(s) s = strings.ToLower(s)
if t := attrType(s); t != contentTypePlain { if t := attrType(s); t != contentTypePlain {
// TODO: Split attr and element name part filters so we can whitelist // TODO: Split attr and element name part filters so we can recognize known attributes.
// attributes.
return filterFailsafe return filterFailsafe
} }
for _, r := range s { for _, r := range s {

View file

@ -0,0 +1,292 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Tests for multiple-template execution, copied from text/template.
// +build go1.13,!windows
package template
import (
"archive/zip"
"bytes"
"os"
"testing"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
)
var multiExecTests = []execTest{
{"empty", "", "", nil, true},
{"text", "some text", "some text", nil, true},
{"invoke x", `{{template "x" .SI}}`, "TEXT", tVal, true},
{"invoke x no args", `{{template "x"}}`, "TEXT", tVal, true},
{"invoke dot int", `{{template "dot" .I}}`, "17", tVal, true},
{"invoke dot []int", `{{template "dot" .SI}}`, "[3 4 5]", tVal, true},
{"invoke dotV", `{{template "dotV" .U}}`, "v", tVal, true},
{"invoke nested int", `{{template "nested" .I}}`, "17", tVal, true},
{"variable declared by template", `{{template "nested" $x:=.SI}},{{index $x 1}}`, "[3 4 5],4", tVal, true},
// User-defined function: test argument evaluator.
{"testFunc literal", `{{oneArg "joe"}}`, "oneArg=joe", tVal, true},
{"testFunc .", `{{oneArg .}}`, "oneArg=joe", "joe", true},
}
// These strings are also in testdata/*.
const multiText1 = `
{{define "x"}}TEXT{{end}}
{{define "dotV"}}{{.V}}{{end}}
`
const multiText2 = `
{{define "dot"}}{{.}}{{end}}
{{define "nested"}}{{template "dot" .}}{{end}}
`
func TestMultiExecute(t *testing.T) {
// Declare a couple of templates first.
template, err := New("root").Parse(multiText1)
if err != nil {
t.Fatalf("parse error for 1: %s", err)
}
_, err = template.Parse(multiText2)
if err != nil {
t.Fatalf("parse error for 2: %s", err)
}
testExecute(multiExecTests, template, t)
}
func TestParseFiles(t *testing.T) {
_, err := ParseFiles("DOES NOT EXIST")
if err == nil {
t.Error("expected error for non-existent file; got none")
}
template := New("root")
_, err = template.ParseFiles("testdata/file1.tmpl", "testdata/file2.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(multiExecTests, template, t)
}
func TestParseGlob(t *testing.T) {
_, err := ParseGlob("DOES NOT EXIST")
if err == nil {
t.Error("expected error for non-existent file; got none")
}
_, err = New("error").ParseGlob("[x")
if err == nil {
t.Error("expected error for bad pattern; got none")
}
template := New("root")
_, err = template.ParseGlob("testdata/file*.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(multiExecTests, template, t)
}
func TestParseFS(t *testing.T) {
fs := os.DirFS("testdata")
{
_, err := ParseFS(fs, "DOES NOT EXIST")
if err == nil {
t.Error("expected error for non-existent file; got none")
}
}
{
template := New("root")
_, err := template.ParseFS(fs, "file1.tmpl", "file2.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(multiExecTests, template, t)
}
{
template := New("root")
_, err := template.ParseFS(fs, "file*.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(multiExecTests, template, t)
}
}
// In these tests, actual content (not just template definitions) comes from the parsed files.
var templateFileExecTests = []execTest{
{"test", `{{template "tmpl1.tmpl"}}{{template "tmpl2.tmpl"}}`, "template1\n\ny\ntemplate2\n\nx\n", 0, true},
}
func TestParseFilesWithData(t *testing.T) {
template, err := New("root").ParseFiles("testdata/tmpl1.tmpl", "testdata/tmpl2.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(templateFileExecTests, template, t)
}
func TestParseGlobWithData(t *testing.T) {
template, err := New("root").ParseGlob("testdata/tmpl*.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(templateFileExecTests, template, t)
}
func TestParseZipFS(t *testing.T) {
z, err := zip.OpenReader("testdata/fs.zip")
if err != nil {
t.Fatalf("error parsing zip: %v", err)
}
template, err := New("root").ParseFS(z, "tmpl*.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(templateFileExecTests, template, t)
}
const (
cloneText1 = `{{define "a"}}{{template "b"}}{{template "c"}}{{end}}`
cloneText2 = `{{define "b"}}b{{end}}`
cloneText3 = `{{define "c"}}root{{end}}`
cloneText4 = `{{define "c"}}clone{{end}}`
)
// Issue 7032
func TestAddParseTreeToUnparsedTemplate(t *testing.T) {
master := "{{define \"master\"}}{{end}}"
tmpl := New("master")
tree, err := parse.Parse("master", master, "", "", nil)
if err != nil {
t.Fatalf("unexpected parse err: %v", err)
}
masterTree := tree["master"]
tmpl.AddParseTree("master", masterTree) // used to panic
}
func TestRedefinition(t *testing.T) {
var tmpl *Template
var err error
if tmpl, err = New("tmpl1").Parse(`{{define "test"}}foo{{end}}`); err != nil {
t.Fatalf("parse 1: %v", err)
}
if _, err = tmpl.Parse(`{{define "test"}}bar{{end}}`); err != nil {
t.Fatalf("got error %v, expected nil", err)
}
if _, err = tmpl.New("tmpl2").Parse(`{{define "test"}}bar{{end}}`); err != nil {
t.Fatalf("got error %v, expected nil", err)
}
}
// Issue 10879
func TestEmptyTemplateCloneCrash(t *testing.T) {
t1 := New("base")
t1.Clone() // used to panic
}
// Issue 10910, 10926
func TestTemplateLookUp(t *testing.T) {
t.Skip("broken on html/template") // TODO
t1 := New("foo")
if t1.Lookup("foo") != nil {
t.Error("Lookup returned non-nil value for undefined template foo")
}
t1.New("bar")
if t1.Lookup("bar") != nil {
t.Error("Lookup returned non-nil value for undefined template bar")
}
t1.Parse(`{{define "foo"}}test{{end}}`)
if t1.Lookup("foo") == nil {
t.Error("Lookup returned nil value for defined template")
}
}
func TestParse(t *testing.T) {
// In multiple calls to Parse with the same receiver template, only one call
// can contain text other than space, comments, and template definitions
t1 := New("test")
if _, err := t1.Parse(`{{define "test"}}{{end}}`); err != nil {
t.Fatalf("parsing test: %s", err)
}
if _, err := t1.Parse(`{{define "test"}}{{/* this is a comment */}}{{end}}`); err != nil {
t.Fatalf("parsing test: %s", err)
}
if _, err := t1.Parse(`{{define "test"}}foo{{end}}`); err != nil {
t.Fatalf("parsing test: %s", err)
}
}
func TestEmptyTemplate(t *testing.T) {
cases := []struct {
defn []string
in string
want string
}{
{[]string{"x", "y"}, "", "y"},
{[]string{""}, "once", ""},
{[]string{"", ""}, "twice", ""},
{[]string{"{{.}}", "{{.}}"}, "twice", "twice"},
{[]string{"{{/* a comment */}}", "{{/* a comment */}}"}, "comment", ""},
{[]string{"{{.}}", ""}, "twice", "twice"}, // TODO: should want "" not "twice"
}
for i, c := range cases {
root := New("root")
var (
m *Template
err error
)
for _, d := range c.defn {
m, err = root.New(c.in).Parse(d)
if err != nil {
t.Fatal(err)
}
}
buf := &bytes.Buffer{}
if err := m.Execute(buf, c.in); err != nil {
t.Error(i, err)
continue
}
if buf.String() != c.want {
t.Errorf("expected string %q: got %q", c.want, buf.String())
}
}
}
// Issue 19249 was a regression in 1.8 caused by the handling of empty
// templates added in that release, which got different answers depending
// on the order templates appeared in the internal map.
func TestIssue19294(t *testing.T) {
// The empty block in "xhtml" should be replaced during execution
// by the contents of "stylesheet", but if the internal map associating
// names with templates is built in the wrong order, the empty block
// looks non-empty and this doesn't happen.
var inlined = map[string]string{
"stylesheet": `{{define "stylesheet"}}stylesheet{{end}}`,
"xhtml": `{{block "stylesheet" .}}{{end}}`,
}
all := []string{"stylesheet", "xhtml"}
for i := 0; i < 100; i++ {
res, err := New("title.xhtml").Parse(`{{template "xhtml" .}}`)
if err != nil {
t.Fatal(err)
}
for _, name := range all {
_, err := res.New(name).Parse(inlined[name])
if err != nil {
t.Fatal(err)
}
}
var buf bytes.Buffer
res.Execute(&buf, 0)
if buf.String() != "stylesheet" {
t.Fatalf("iteration %d: got %q; expected %q", i, buf.String(), "stylesheet")
}
}
}

View file

@ -7,7 +7,9 @@ package template
import ( import (
"fmt" "fmt"
"io" "io"
"io/fs"
"io/ioutil" "io/ioutil"
"path"
"path/filepath" "path/filepath"
"sync" "sync"
@ -385,7 +387,7 @@ func Must(t *Template, err error) *Template {
// For instance, ParseFiles("a/foo", "b/foo") stores "b/foo" as the template // For instance, ParseFiles("a/foo", "b/foo") stores "b/foo" as the template
// named "foo", while "a/foo" is unavailable. // named "foo", while "a/foo" is unavailable.
func ParseFiles(filenames ...string) (*Template, error) { func ParseFiles(filenames ...string) (*Template, error) {
return parseFiles(nil, filenames...) return parseFiles(nil, readFileOS, filenames...)
} }
// ParseFiles parses the named files and associates the resulting templates with // ParseFiles parses the named files and associates the resulting templates with
@ -397,12 +399,12 @@ func ParseFiles(filenames ...string) (*Template, error) {
// //
// ParseFiles returns an error if t or any associated template has already been executed. // ParseFiles returns an error if t or any associated template has already been executed.
func (t *Template) ParseFiles(filenames ...string) (*Template, error) { func (t *Template) ParseFiles(filenames ...string) (*Template, error) {
return parseFiles(t, filenames...) return parseFiles(t, readFileOS, filenames...)
} }
// parseFiles is the helper for the method and function. If the argument // parseFiles is the helper for the method and function. If the argument
// template is nil, it is created from the first file. // template is nil, it is created from the first file.
func parseFiles(t *Template, filenames ...string) (*Template, error) { func parseFiles(t *Template, readFile func(string) (string, []byte, error), filenames ...string) (*Template, error) {
if err := t.checkCanParse(); err != nil { if err := t.checkCanParse(); err != nil {
return nil, err return nil, err
} }
@ -412,12 +414,11 @@ func parseFiles(t *Template, filenames ...string) (*Template, error) {
return nil, fmt.Errorf("html/template: no files named in call to ParseFiles") return nil, fmt.Errorf("html/template: no files named in call to ParseFiles")
} }
for _, filename := range filenames { for _, filename := range filenames {
b, err := ioutil.ReadFile(filename) name, b, err := readFile(filename)
if err != nil { if err != nil {
return nil, err return nil, err
} }
s := string(b) s := string(b)
name := filepath.Base(filename)
// First template becomes return value if not already defined, // First template becomes return value if not already defined,
// and we use that one for subsequent New calls to associate // and we use that one for subsequent New calls to associate
// all the templates together. Also, if this file has the same name // all the templates together. Also, if this file has the same name
@ -480,7 +481,7 @@ func parseGlob(t *Template, pattern string) (*Template, error) {
if len(filenames) == 0 { if len(filenames) == 0 {
return nil, fmt.Errorf("html/template: pattern matches no files: %#q", pattern) return nil, fmt.Errorf("html/template: pattern matches no files: %#q", pattern)
} }
return parseFiles(t, filenames...) return parseFiles(t, readFileOS, filenames...)
} }
// IsTrue reports whether the value is 'true', in the sense of not the zero of its type, // IsTrue reports whether the value is 'true', in the sense of not the zero of its type,
@ -489,3 +490,48 @@ func parseGlob(t *Template, pattern string) (*Template, error) {
func IsTrue(val interface{}) (truth, ok bool) { func IsTrue(val interface{}) (truth, ok bool) {
return template.IsTrue(val) return template.IsTrue(val)
} }
// ParseFS is like ParseFiles or ParseGlob but reads from the file system fs
// instead of the host operating system's file system.
// It accepts a list of glob patterns.
// (Note that most file names serve as glob patterns matching only themselves.)
func ParseFS(fs fs.FS, patterns ...string) (*Template, error) {
return parseFS(nil, fs, patterns)
}
// ParseFS is like ParseFiles or ParseGlob but reads from the file system fs
// instead of the host operating system's file system.
// It accepts a list of glob patterns.
// (Note that most file names serve as glob patterns matching only themselves.)
func (t *Template) ParseFS(fs fs.FS, patterns ...string) (*Template, error) {
return parseFS(t, fs, patterns)
}
func parseFS(t *Template, fsys fs.FS, patterns []string) (*Template, error) {
var filenames []string
for _, pattern := range patterns {
list, err := fs.Glob(fsys, pattern)
if err != nil {
return nil, err
}
if len(list) == 0 {
return nil, fmt.Errorf("template: pattern matches no files: %#q", pattern)
}
filenames = append(filenames, list...)
}
return parseFiles(t, readFileFS(fsys), filenames...)
}
func readFileOS(file string) (name string, b []byte, err error) {
name = filepath.Base(file)
b, err = ioutil.ReadFile(file)
return
}
func readFileFS(fsys fs.FS) func(string) (string, []byte, error) {
return func(file string) (name string, b []byte, err error) {
name = path.Base(file)
b, err = fs.ReadFile(fsys, file)
return
}
}

View file

@ -13,10 +13,11 @@ import (
"testing" "testing"
. "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate" . "github.com/gohugoio/hugo/tpl/internal/go_templates/htmltemplate"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" // https://golang.org/issue/12996
) )
func TestTemplateClone(t *testing.T) { func TestTemplateClone(t *testing.T) {
// https://golang.org/issue/12996
orig := New("name") orig := New("name")
clone, err := orig.Clone() clone, err := orig.Clone()
if err != nil { if err != nil {
@ -163,6 +164,21 @@ func TestStringsInScriptsWithJsonContentTypeAreCorrectlyEscaped(t *testing.T) {
} }
} }
func TestSkipEscapeComments(t *testing.T) {
c := newTestCase(t)
tr := parse.New("root")
tr.Mode = parse.ParseComments
newT, err := tr.Parse("{{/* A comment */}}{{ 1 }}{{/* Another comment */}}", "", "", make(map[string]*parse.Tree))
if err != nil {
t.Fatalf("Cannot parse template text: %v", err)
}
c.root, err = c.root.AddParseTree("root", newT)
if err != nil {
t.Fatalf("Cannot add parse tree to template: %v", err)
}
c.mustExecute(c.root, nil, "1")
}
type testCase struct { type testCase struct {
t *testing.T t *testing.T
root *Template root *Template

View file

@ -45,12 +45,8 @@ func HasGoBuild() bool {
return false return false
} }
switch runtime.GOOS { switch runtime.GOOS {
case "android", "js": case "android", "js", "ios":
return false return false
case "darwin":
if runtime.GOARCH == "arm64" {
return false
}
} }
return true return true
} }
@ -124,12 +120,8 @@ func GoTool() (string, error) {
// using os.StartProcess or (more commonly) exec.Command. // using os.StartProcess or (more commonly) exec.Command.
func HasExec() bool { func HasExec() bool {
switch runtime.GOOS { switch runtime.GOOS {
case "js": case "js", "ios":
return false return false
case "darwin":
if runtime.GOARCH == "arm64" {
return false
}
} }
return true return true
} }
@ -137,10 +129,8 @@ func HasExec() bool {
// HasSrc reports whether the entire source tree is available under GOROOT. // HasSrc reports whether the entire source tree is available under GOROOT.
func HasSrc() bool { func HasSrc() bool {
switch runtime.GOOS { switch runtime.GOOS {
case "darwin": case "ios":
if runtime.GOARCH == "arm64" { return false
return false
}
} }
return true return true
} }
@ -204,6 +194,32 @@ func MustHaveCGO(t testing.TB) {
} }
} }
// CanInternalLink reports whether the current system can link programs with
// internal linking.
// (This is the opposite of cmd/internal/sys.MustLinkExternal. Keep them in sync.)
func CanInternalLink() bool {
switch runtime.GOOS {
case "android":
if runtime.GOARCH != "arm64" {
return false
}
case "ios":
if runtime.GOARCH == "arm64" {
return false
}
}
return true
}
// MustInternalLink checks that the current system can link programs with internal
// linking.
// If not, MustInternalLink calls t.Skip with an explanation.
func MustInternalLink(t testing.TB) {
if !CanInternalLink() {
t.Skipf("skipping test: internal linking on %s/%s is not supported", runtime.GOOS, runtime.GOARCH)
}
}
// HasSymlink reports whether the current system can use os.Symlink. // HasSymlink reports whether the current system can use os.Symlink.
func HasSymlink() bool { func HasSymlink() bool {
ok, _ := hasSymlink() ok, _ := hasSymlink()
@ -272,3 +288,23 @@ func CleanCmdEnv(cmd *exec.Cmd) *exec.Cmd {
} }
return cmd return cmd
} }
// CPUIsSlow reports whether the CPU running the test is suspected to be slow.
func CPUIsSlow() bool {
switch runtime.GOARCH {
case "arm", "mips", "mipsle", "mips64", "mips64le":
return true
}
return false
}
// SkipIfShortAndSlow skips t if -short is set and the CPU running the test is
// suspected to be slow.
//
// (This is useful for CPU-intensive tests that otherwise complete quickly.)
func SkipIfShortAndSlow(t testing.TB) {
if testing.Short() && CPUIsSlow() {
t.Helper()
t.Skipf("skipping test in -short mode on %s", runtime.GOARCH)
}
}

View file

@ -40,16 +40,17 @@ More intricate examples appear below.
Text and spaces Text and spaces
By default, all text between actions is copied verbatim when the template is By default, all text between actions is copied verbatim when the template is
executed. For example, the string " items are made of " in the example above appears executed. For example, the string " items are made of " in the example above
on standard output when the program is run. appears on standard output when the program is run.
However, to aid in formatting template source code, if an action's left delimiter However, to aid in formatting template source code, if an action's left
(by default "{{") is followed immediately by a minus sign and ASCII space character delimiter (by default "{{") is followed immediately by a minus sign and white
("{{- "), all trailing white space is trimmed from the immediately preceding text. space, all trailing white space is trimmed from the immediately preceding text.
Similarly, if the right delimiter ("}}") is preceded by a space and minus sign Similarly, if the right delimiter ("}}") is preceded by white space and a minus
(" -}}"), all leading white space is trimmed from the immediately following text. sign, all leading white space is trimmed from the immediately following text.
In these trim markers, the ASCII space must be present; "{{-3}}" parses as an In these trim markers, the white space must be present:
action containing the number -3. "{{- 3}}" is like "{{3}}" but trims the immediately preceding text, while
"{{-3}}" parses as an action containing the number -3.
For instance, when executing the template whose source is For instance, when executing the template whose source is

View file

@ -256,6 +256,7 @@ func (s *state) walk(dot reflect.Value, node parse.Node) {
if len(node.Pipe.Decl) == 0 { if len(node.Pipe.Decl) == 0 {
s.printValue(node, val) s.printValue(node, val)
} }
case *parse.CommentNode:
case *parse.IfNode: case *parse.IfNode:
s.walkIfOrWith(parse.NodeIf, dot, node.Pipe, node.List, node.ElseList) s.walkIfOrWith(parse.NodeIf, dot, node.Pipe, node.List, node.ElseList)
case *parse.ListNode: case *parse.ListNode:

View file

@ -11,7 +11,7 @@ import (
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
"io/ioutil" "io"
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
@ -1297,7 +1297,7 @@ func TestUnterminatedStringError(t *testing.T) {
t.Fatal("expected error") t.Fatal("expected error")
} }
str := err.Error() str := err.Error()
if !strings.Contains(str, "X:3: unexpected unterminated raw quoted string") { if !strings.Contains(str, "X:3: unterminated raw quoted string") {
t.Fatalf("unexpected error: %s", str) t.Fatalf("unexpected error: %s", str)
} }
} }
@ -1330,7 +1330,7 @@ func TestExecuteGivesExecError(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
err = tmpl.Execute(ioutil.Discard, 0) err = tmpl.Execute(io.Discard, 0)
if err == nil { if err == nil {
t.Fatal("expected error; got none") t.Fatal("expected error; got none")
} }
@ -1476,7 +1476,7 @@ func TestEvalFieldErrors(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
tmpl := Must(New("tmpl").Parse(tc.src)) tmpl := Must(New("tmpl").Parse(tc.src))
err := tmpl.Execute(ioutil.Discard, tc.value) err := tmpl.Execute(io.Discard, tc.value)
got := "<nil>" got := "<nil>"
if err != nil { if err != nil {
got = err.Error() got = err.Error()
@ -1493,7 +1493,7 @@ func TestMaxExecDepth(t *testing.T) {
t.Skip("skipping in -short mode") t.Skip("skipping in -short mode")
} }
tmpl := Must(New("tmpl").Parse(`{{template "tmpl" .}}`)) tmpl := Must(New("tmpl").Parse(`{{template "tmpl" .}}`))
err := tmpl.Execute(ioutil.Discard, nil) err := tmpl.Execute(io.Discard, nil)
got := "<nil>" got := "<nil>"
if err != nil { if err != nil {
got = err.Error() got = err.Error()

View file

@ -8,7 +8,9 @@ package template
import ( import (
"fmt" "fmt"
"io/fs"
"io/ioutil" "io/ioutil"
"path"
"path/filepath" "path/filepath"
) )
@ -35,7 +37,7 @@ func Must(t *Template, err error) *Template {
// For instance, ParseFiles("a/foo", "b/foo") stores "b/foo" as the template // For instance, ParseFiles("a/foo", "b/foo") stores "b/foo" as the template
// named "foo", while "a/foo" is unavailable. // named "foo", while "a/foo" is unavailable.
func ParseFiles(filenames ...string) (*Template, error) { func ParseFiles(filenames ...string) (*Template, error) {
return parseFiles(nil, filenames...) return parseFiles(nil, readFileOS, filenames...)
} }
// ParseFiles parses the named files and associates the resulting templates with // ParseFiles parses the named files and associates the resulting templates with
@ -51,23 +53,22 @@ func ParseFiles(filenames ...string) (*Template, error) {
// the last one mentioned will be the one that results. // the last one mentioned will be the one that results.
func (t *Template) ParseFiles(filenames ...string) (*Template, error) { func (t *Template) ParseFiles(filenames ...string) (*Template, error) {
t.init() t.init()
return parseFiles(t, filenames...) return parseFiles(t, readFileOS, filenames...)
} }
// parseFiles is the helper for the method and function. If the argument // parseFiles is the helper for the method and function. If the argument
// template is nil, it is created from the first file. // template is nil, it is created from the first file.
func parseFiles(t *Template, filenames ...string) (*Template, error) { func parseFiles(t *Template, readFile func(string) (string, []byte, error), filenames ...string) (*Template, error) {
if len(filenames) == 0 { if len(filenames) == 0 {
// Not really a problem, but be consistent. // Not really a problem, but be consistent.
return nil, fmt.Errorf("template: no files named in call to ParseFiles") return nil, fmt.Errorf("template: no files named in call to ParseFiles")
} }
for _, filename := range filenames { for _, filename := range filenames {
b, err := ioutil.ReadFile(filename) name, b, err := readFile(filename)
if err != nil { if err != nil {
return nil, err return nil, err
} }
s := string(b) s := string(b)
name := filepath.Base(filename)
// First template becomes return value if not already defined, // First template becomes return value if not already defined,
// and we use that one for subsequent New calls to associate // and we use that one for subsequent New calls to associate
// all the templates together. Also, if this file has the same name // all the templates together. Also, if this file has the same name
@ -126,5 +127,51 @@ func parseGlob(t *Template, pattern string) (*Template, error) {
if len(filenames) == 0 { if len(filenames) == 0 {
return nil, fmt.Errorf("template: pattern matches no files: %#q", pattern) return nil, fmt.Errorf("template: pattern matches no files: %#q", pattern)
} }
return parseFiles(t, filenames...) return parseFiles(t, readFileOS, filenames...)
}
// ParseFS is like ParseFiles or ParseGlob but reads from the file system fsys
// instead of the host operating system's file system.
// It accepts a list of glob patterns.
// (Note that most file names serve as glob patterns matching only themselves.)
func ParseFS(fsys fs.FS, patterns ...string) (*Template, error) {
return parseFS(nil, fsys, patterns)
}
// ParseFS is like ParseFiles or ParseGlob but reads from the file system fsys
// instead of the host operating system's file system.
// It accepts a list of glob patterns.
// (Note that most file names serve as glob patterns matching only themselves.)
func (t *Template) ParseFS(fsys fs.FS, patterns ...string) (*Template, error) {
t.init()
return parseFS(t, fsys, patterns)
}
func parseFS(t *Template, fsys fs.FS, patterns []string) (*Template, error) {
var filenames []string
for _, pattern := range patterns {
list, err := fs.Glob(fsys, pattern)
if err != nil {
return nil, err
}
if len(list) == 0 {
return nil, fmt.Errorf("template: pattern matches no files: %#q", pattern)
}
filenames = append(filenames, list...)
}
return parseFiles(t, readFileFS(fsys), filenames...)
}
func readFileOS(file string) (name string, b []byte, err error) {
name = filepath.Base(file)
b, err = ioutil.ReadFile(file)
return
}
func readFileFS(fsys fs.FS) func(string) (string, []byte, error) {
return func(file string) (name string, b []byte, err error) {
name = path.Base(file)
b, err = fs.ReadFile(fsys, file)
return
}
} }

View file

@ -0,0 +1,66 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build go1.13
package template_test
import (
"bytes"
"github.com/gohugoio/hugo/tpl/internal/go_templates/testenv"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"testing"
)
// Issue 36021: verify that text/template doesn't prevent the linker from removing
// unused methods.
func _TestLinkerGC(t *testing.T) {
if testing.Short() {
t.Skip("skipping in short mode")
}
testenv.MustHaveGoBuild(t)
const prog = `package main
import (
_ "text/template"
)
type T struct{}
func (t *T) Unused() { println("THIS SHOULD BE ELIMINATED") }
func (t *T) Used() {}
var sink *T
func main() {
var t T
sink = &t
t.Used()
}
`
td, err := ioutil.TempDir("", "text_template_TestDeadCodeElimination")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(td)
if err := ioutil.WriteFile(filepath.Join(td, "x.go"), []byte(prog), 0644); err != nil {
t.Fatal(err)
}
cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", "x.exe", "x.go")
cmd.Dir = td
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("go build: %v, %s", err, out)
}
slurp, err := ioutil.ReadFile(filepath.Join(td, "x.exe"))
if err != nil {
t.Fatal(err)
}
if bytes.Contains(slurp, []byte("THIS SHOULD BE ELIMINATED")) {
t.Error("binary contains code that should be deadcode eliminated")
}
}

View file

@ -12,6 +12,7 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse" "github.com/gohugoio/hugo/tpl/internal/go_templates/texttemplate/parse"
"os"
"testing" "testing"
) )
@ -155,6 +156,35 @@ func TestParseGlob(t *testing.T) {
testExecute(multiExecTests, template, t) testExecute(multiExecTests, template, t)
} }
func TestParseFS(t *testing.T) {
fs := os.DirFS("testdata")
{
_, err := ParseFS(fs, "DOES NOT EXIST")
if err == nil {
t.Error("expected error for non-existent file; got none")
}
}
{
template := New("root")
_, err := template.ParseFS(fs, "file1.tmpl", "file2.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(multiExecTests, template, t)
}
{
template := New("root")
_, err := template.ParseFS(fs, "file*.tmpl")
if err != nil {
t.Fatalf("error parsing files: %v", err)
}
testExecute(multiExecTests, template, t)
}
}
// In these tests, actual content (not just template definitions) comes from the parsed files. // In these tests, actual content (not just template definitions) comes from the parsed files.
var templateFileExecTests = []execTest{ var templateFileExecTests = []execTest{
@ -361,6 +391,7 @@ func TestEmptyTemplate(t *testing.T) {
in string in string
want string want string
}{ }{
{[]string{"x", "y"}, "", "y"},
{[]string{""}, "once", ""}, {[]string{""}, "once", ""},
{[]string{"", ""}, "twice", ""}, {[]string{"", ""}, "twice", ""},
{[]string{"{{.}}", "{{.}}"}, "twice", "twice"}, {[]string{"{{.}}", "{{.}}"}, "twice", "twice"},

View file

@ -41,6 +41,7 @@ const (
itemBool // boolean constant itemBool // boolean constant
itemChar // printable ASCII character; grab bag for comma etc. itemChar // printable ASCII character; grab bag for comma etc.
itemCharConstant // character constant itemCharConstant // character constant
itemComment // comment text
itemComplex // complex constant (1+2i); imaginary is just a number itemComplex // complex constant (1+2i); imaginary is just a number
itemAssign // equals ('=') introducing an assignment itemAssign // equals ('=') introducing an assignment
itemDeclare // colon-equals (':=') introducing a declaration itemDeclare // colon-equals (':=') introducing a declaration
@ -91,15 +92,14 @@ const eof = -1
// If the action begins "{{- " rather than "{{", then all space/tab/newlines // If the action begins "{{- " rather than "{{", then all space/tab/newlines
// preceding the action are trimmed; conversely if it ends " -}}" the // preceding the action are trimmed; conversely if it ends " -}}" the
// leading spaces are trimmed. This is done entirely in the lexer; the // leading spaces are trimmed. This is done entirely in the lexer; the
// parser never sees it happen. We require an ASCII space to be // parser never sees it happen. We require an ASCII space (' ', \t, \r, \n)
// present to avoid ambiguity with things like "{{-3}}". It reads // to be present to avoid ambiguity with things like "{{-3}}". It reads
// better with the space present anyway. For simplicity, only ASCII // better with the space present anyway. For simplicity, only ASCII
// space does the job. // does the job.
const ( const (
spaceChars = " \t\r\n" // These are the space characters defined by Go itself. spaceChars = " \t\r\n" // These are the space characters defined by Go itself.
leftTrimMarker = "- " // Attached to left delimiter, trims trailing spaces from preceding text. trimMarker = '-' // Attached to left/right delimiter, trims trailing spaces from preceding/following text.
rightTrimMarker = " -" // Attached to right delimiter, trims leading spaces from following text. trimMarkerLen = Pos(1 + 1) // marker plus space before or after
trimMarkerLen = Pos(len(leftTrimMarker))
) )
// stateFn represents the state of the scanner as a function that returns the next state. // stateFn represents the state of the scanner as a function that returns the next state.
@ -107,18 +107,18 @@ type stateFn func(*lexer) stateFn
// lexer holds the state of the scanner. // lexer holds the state of the scanner.
type lexer struct { type lexer struct {
name string // the name of the input; used only for error reports name string // the name of the input; used only for error reports
input string // the string being scanned input string // the string being scanned
leftDelim string // start of action leftDelim string // start of action
rightDelim string // end of action rightDelim string // end of action
trimRightDelim string // end of action with trim marker emitComment bool // emit itemComment tokens.
pos Pos // current position in the input pos Pos // current position in the input
start Pos // start position of this item start Pos // start position of this item
width Pos // width of last rune read from input width Pos // width of last rune read from input
items chan item // channel of scanned items items chan item // channel of scanned items
parenDepth int // nesting depth of ( ) exprs parenDepth int // nesting depth of ( ) exprs
line int // 1+number of newlines seen line int // 1+number of newlines seen
startLine int // start line of this item startLine int // start line of this item
} }
// next returns the next rune in the input. // next returns the next rune in the input.
@ -203,7 +203,7 @@ func (l *lexer) drain() {
} }
// lex creates a new scanner for the input string. // lex creates a new scanner for the input string.
func lex(name, input, left, right string) *lexer { func lex(name, input, left, right string, emitComment bool) *lexer {
if left == "" { if left == "" {
left = leftDelim left = leftDelim
} }
@ -211,14 +211,14 @@ func lex(name, input, left, right string) *lexer {
right = rightDelim right = rightDelim
} }
l := &lexer{ l := &lexer{
name: name, name: name,
input: input, input: input,
leftDelim: left, leftDelim: left,
rightDelim: right, rightDelim: right,
trimRightDelim: rightTrimMarker + right, emitComment: emitComment,
items: make(chan item), items: make(chan item),
line: 1, line: 1,
startLine: 1, startLine: 1,
} }
go l.run() go l.run()
return l return l
@ -248,7 +248,7 @@ func lexText(l *lexer) stateFn {
ldn := Pos(len(l.leftDelim)) ldn := Pos(len(l.leftDelim))
l.pos += Pos(x) l.pos += Pos(x)
trimLength := Pos(0) trimLength := Pos(0)
if strings.HasPrefix(l.input[l.pos+ldn:], leftTrimMarker) { if hasLeftTrimMarker(l.input[l.pos+ldn:]) {
trimLength = rightTrimLength(l.input[l.start:l.pos]) trimLength = rightTrimLength(l.input[l.start:l.pos])
} }
l.pos -= trimLength l.pos -= trimLength
@ -277,7 +277,7 @@ func rightTrimLength(s string) Pos {
// atRightDelim reports whether the lexer is at a right delimiter, possibly preceded by a trim marker. // atRightDelim reports whether the lexer is at a right delimiter, possibly preceded by a trim marker.
func (l *lexer) atRightDelim() (delim, trimSpaces bool) { func (l *lexer) atRightDelim() (delim, trimSpaces bool) {
if strings.HasPrefix(l.input[l.pos:], l.trimRightDelim) { // With trim marker. if hasRightTrimMarker(l.input[l.pos:]) && strings.HasPrefix(l.input[l.pos+trimMarkerLen:], l.rightDelim) { // With trim marker.
return true, true return true, true
} }
if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim marker. if strings.HasPrefix(l.input[l.pos:], l.rightDelim) { // Without trim marker.
@ -294,7 +294,7 @@ func leftTrimLength(s string) Pos {
// lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker. // lexLeftDelim scans the left delimiter, which is known to be present, possibly with a trim marker.
func lexLeftDelim(l *lexer) stateFn { func lexLeftDelim(l *lexer) stateFn {
l.pos += Pos(len(l.leftDelim)) l.pos += Pos(len(l.leftDelim))
trimSpace := strings.HasPrefix(l.input[l.pos:], leftTrimMarker) trimSpace := hasLeftTrimMarker(l.input[l.pos:])
afterMarker := Pos(0) afterMarker := Pos(0)
if trimSpace { if trimSpace {
afterMarker = trimMarkerLen afterMarker = trimMarkerLen
@ -323,6 +323,9 @@ func lexComment(l *lexer) stateFn {
if !delim { if !delim {
return l.errorf("comment ends before closing delimiter") return l.errorf("comment ends before closing delimiter")
} }
if l.emitComment {
l.emit(itemComment)
}
if trimSpace { if trimSpace {
l.pos += trimMarkerLen l.pos += trimMarkerLen
} }
@ -336,7 +339,7 @@ func lexComment(l *lexer) stateFn {
// lexRightDelim scans the right delimiter, which is known to be present, possibly with a trim marker. // lexRightDelim scans the right delimiter, which is known to be present, possibly with a trim marker.
func lexRightDelim(l *lexer) stateFn { func lexRightDelim(l *lexer) stateFn {
trimSpace := strings.HasPrefix(l.input[l.pos:], rightTrimMarker) trimSpace := hasRightTrimMarker(l.input[l.pos:])
if trimSpace { if trimSpace {
l.pos += trimMarkerLen l.pos += trimMarkerLen
l.ignore() l.ignore()
@ -363,7 +366,7 @@ func lexInsideAction(l *lexer) stateFn {
return l.errorf("unclosed left paren") return l.errorf("unclosed left paren")
} }
switch r := l.next(); { switch r := l.next(); {
case r == eof || isEndOfLine(r): case r == eof:
return l.errorf("unclosed action") return l.errorf("unclosed action")
case isSpace(r): case isSpace(r):
l.backup() // Put space back in case we have " -}}". l.backup() // Put space back in case we have " -}}".
@ -433,7 +436,7 @@ func lexSpace(l *lexer) stateFn {
} }
// Be careful about a trim-marked closing delimiter, which has a minus // Be careful about a trim-marked closing delimiter, which has a minus
// after a space. We know there is a space, so check for the '-' that might follow. // after a space. We know there is a space, so check for the '-' that might follow.
if strings.HasPrefix(l.input[l.pos-1:], l.trimRightDelim) { if hasRightTrimMarker(l.input[l.pos-1:]) && strings.HasPrefix(l.input[l.pos-1+trimMarkerLen:], l.rightDelim) {
l.backup() // Before the space. l.backup() // Before the space.
if numSpaces == 1 { if numSpaces == 1 {
return lexRightDelim // On the delim, so go right to that. return lexRightDelim // On the delim, so go right to that.
@ -520,7 +523,7 @@ func lexFieldOrVariable(l *lexer, typ itemType) stateFn {
// day to implement arithmetic. // day to implement arithmetic.
func (l *lexer) atTerminator() bool { func (l *lexer) atTerminator() bool {
r := l.peek() r := l.peek()
if isSpace(r) || isEndOfLine(r) { if isSpace(r) {
return true return true
} }
switch r { switch r {
@ -651,15 +654,18 @@ Loop:
// isSpace reports whether r is a space character. // isSpace reports whether r is a space character.
func isSpace(r rune) bool { func isSpace(r rune) bool {
return r == ' ' || r == '\t' return r == ' ' || r == '\t' || r == '\r' || r == '\n'
}
// isEndOfLine reports whether r is an end-of-line character.
func isEndOfLine(r rune) bool {
return r == '\r' || r == '\n'
} }
// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore. // isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
func isAlphaNumeric(r rune) bool { func isAlphaNumeric(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
} }
func hasLeftTrimMarker(s string) bool {
return len(s) >= 2 && s[0] == trimMarker && isSpace(rune(s[1]))
}
func hasRightTrimMarker(s string) bool {
return len(s) >= 2 && isSpace(rune(s[0])) && s[1] == trimMarker
}

View file

@ -17,6 +17,7 @@ var itemName = map[itemType]string{
itemBool: "bool", itemBool: "bool",
itemChar: "char", itemChar: "char",
itemCharConstant: "charconst", itemCharConstant: "charconst",
itemComment: "comment",
itemComplex: "complex", itemComplex: "complex",
itemDeclare: ":=", itemDeclare: ":=",
itemEOF: "EOF", itemEOF: "EOF",
@ -92,6 +93,7 @@ var lexTests = []lexTest{
{"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}}, {"text", `now is the time`, []item{mkItem(itemText, "now is the time"), tEOF}},
{"text with comment", "hello-{{/* this is a comment */}}-world", []item{ {"text with comment", "hello-{{/* this is a comment */}}-world", []item{
mkItem(itemText, "hello-"), mkItem(itemText, "hello-"),
mkItem(itemComment, "/* this is a comment */"),
mkItem(itemText, "-world"), mkItem(itemText, "-world"),
tEOF, tEOF,
}}, }},
@ -313,6 +315,7 @@ var lexTests = []lexTest{
}}, }},
{"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{ {"trimming spaces before and after comment", "hello- {{- /* hello */ -}} -world", []item{
mkItem(itemText, "hello-"), mkItem(itemText, "hello-"),
mkItem(itemComment, "/* hello */"),
mkItem(itemText, "-world"), mkItem(itemText, "-world"),
tEOF, tEOF,
}}, }},
@ -322,7 +325,7 @@ var lexTests = []lexTest{
tLeft, tLeft,
mkItem(itemError, "unrecognized character in action: U+0001"), mkItem(itemError, "unrecognized character in action: U+0001"),
}}, }},
{"unclosed action", "{{\n}}", []item{ {"unclosed action", "{{", []item{
tLeft, tLeft,
mkItem(itemError, "unclosed action"), mkItem(itemError, "unclosed action"),
}}, }},
@ -391,7 +394,7 @@ var lexTests = []lexTest{
// collect gathers the emitted items into a slice. // collect gathers the emitted items into a slice.
func collect(t *lexTest, left, right string) (items []item) { func collect(t *lexTest, left, right string) (items []item) {
l := lex(t.name, t.input, left, right) l := lex(t.name, t.input, left, right, true)
for { for {
item := l.nextItem() item := l.nextItem()
items = append(items, item) items = append(items, item)
@ -531,7 +534,7 @@ func TestPos(t *testing.T) {
func TestShutdown(t *testing.T) { func TestShutdown(t *testing.T) {
// We need to duplicate template.Parse here to hold on to the lexer. // We need to duplicate template.Parse here to hold on to the lexer.
const text = "erroneous{{define}}{{else}}1234" const text = "erroneous{{define}}{{else}}1234"
lexer := lex("foo", text, "{{", "}}") lexer := lex("foo", text, "{{", "}}", false)
_, err := New("root").parseLexer(lexer) _, err := New("root").parseLexer(lexer)
if err == nil { if err == nil {
t.Fatalf("expected error") t.Fatalf("expected error")

View file

@ -70,6 +70,7 @@ const (
NodeTemplate // A template invocation action. NodeTemplate // A template invocation action.
NodeVariable // A $ variable. NodeVariable // A $ variable.
NodeWith // A with action. NodeWith // A with action.
NodeComment // A comment.
) )
// Nodes. // Nodes.
@ -149,6 +150,38 @@ func (t *TextNode) Copy() Node {
return &TextNode{tr: t.tr, NodeType: NodeText, Pos: t.Pos, Text: append([]byte{}, t.Text...)} return &TextNode{tr: t.tr, NodeType: NodeText, Pos: t.Pos, Text: append([]byte{}, t.Text...)}
} }
// CommentNode holds a comment.
type CommentNode struct {
NodeType
Pos
tr *Tree
Text string // Comment text.
}
func (t *Tree) newComment(pos Pos, text string) *CommentNode {
return &CommentNode{tr: t, NodeType: NodeComment, Pos: pos, Text: text}
}
func (c *CommentNode) String() string {
var sb strings.Builder
c.writeTo(&sb)
return sb.String()
}
func (c *CommentNode) writeTo(sb *strings.Builder) {
sb.WriteString("{{")
sb.WriteString(c.Text)
sb.WriteString("}}")
}
func (c *CommentNode) tree() *Tree {
return c.tr
}
func (c *CommentNode) Copy() Node {
return &CommentNode{tr: c.tr, NodeType: NodeComment, Pos: c.Pos, Text: c.Text}
}
// PipeNode holds a pipeline with optional declaration // PipeNode holds a pipeline with optional declaration
type PipeNode struct { type PipeNode struct {
NodeType NodeType
@ -349,7 +382,7 @@ func (i *IdentifierNode) Copy() Node {
return NewIdentifier(i.Ident).SetTree(i.tr).SetPos(i.Pos) return NewIdentifier(i.Ident).SetTree(i.tr).SetPos(i.Pos)
} }
// AssignNode holds a list of variable names, possibly with chained field // VariableNode holds a list of variable names, possibly with chained field
// accesses. The dollar sign is part of the (first) name. // accesses. The dollar sign is part of the (first) name.
type VariableNode struct { type VariableNode struct {
NodeType NodeType

View file

@ -21,16 +21,26 @@ type Tree struct {
Name string // name of the template represented by the tree. Name string // name of the template represented by the tree.
ParseName string // name of the top-level template during parsing, for error messages. ParseName string // name of the top-level template during parsing, for error messages.
Root *ListNode // top-level root of the tree. Root *ListNode // top-level root of the tree.
Mode Mode // parsing mode.
text string // text parsed to create the template (or its parent) text string // text parsed to create the template (or its parent)
// Parsing only; cleared after parse. // Parsing only; cleared after parse.
funcs []map[string]interface{} funcs []map[string]interface{}
lex *lexer lex *lexer
token [3]item // three-token lookahead for parser. token [3]item // three-token lookahead for parser.
peekCount int peekCount int
vars []string // variables defined at the moment. vars []string // variables defined at the moment.
treeSet map[string]*Tree treeSet map[string]*Tree
actionLine int // line of left delim starting action
mode Mode
} }
// A mode value is a set of flags (or 0). Modes control parser behavior.
type Mode uint
const (
ParseComments Mode = 1 << iota // parse comments and add them to AST
)
// Copy returns a copy of the Tree. Any parsing state is discarded. // Copy returns a copy of the Tree. Any parsing state is discarded.
func (t *Tree) Copy() *Tree { func (t *Tree) Copy() *Tree {
if t == nil { if t == nil {
@ -178,6 +188,16 @@ func (t *Tree) expectOneOf(expected1, expected2 itemType, context string) item {
// unexpected complains about the token and terminates processing. // unexpected complains about the token and terminates processing.
func (t *Tree) unexpected(token item, context string) { func (t *Tree) unexpected(token item, context string) {
if token.typ == itemError {
extra := ""
if t.actionLine != 0 && t.actionLine != token.line {
extra = fmt.Sprintf(" in action started at %s:%d", t.ParseName, t.actionLine)
if strings.HasSuffix(token.val, " action") {
extra = extra[len(" in action"):] // avoid "action in action"
}
}
t.errorf("%s%s", token, extra)
}
t.errorf("unexpected %s in %s", token, context) t.errorf("unexpected %s in %s", token, context)
} }
@ -220,7 +240,8 @@ func (t *Tree) stopParse() {
func (t *Tree) Parse(text, leftDelim, rightDelim string, treeSet map[string]*Tree, funcs ...map[string]interface{}) (tree *Tree, err error) { func (t *Tree) Parse(text, leftDelim, rightDelim string, treeSet map[string]*Tree, funcs ...map[string]interface{}) (tree *Tree, err error) {
defer t.recover(&err) defer t.recover(&err)
t.ParseName = t.Name t.ParseName = t.Name
t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim), treeSet) emitComment := t.Mode&ParseComments != 0
t.startParse(funcs, lex(t.Name, text, leftDelim, rightDelim, emitComment), treeSet)
t.text = text t.text = text
t.parse() t.parse()
t.add() t.add()
@ -240,12 +261,14 @@ func (t *Tree) add() {
} }
} }
// IsEmptyTree reports whether this tree (node) is empty of everything but space. // IsEmptyTree reports whether this tree (node) is empty of everything but space or comments.
func IsEmptyTree(n Node) bool { func IsEmptyTree(n Node) bool {
switch n := n.(type) { switch n := n.(type) {
case nil: case nil:
return true return true
case *ActionNode: case *ActionNode:
case *CommentNode:
return true
case *IfNode: case *IfNode:
case *ListNode: case *ListNode:
for _, node := range n.Nodes { for _, node := range n.Nodes {
@ -276,6 +299,7 @@ func (t *Tree) parse() {
if t.nextNonSpace().typ == itemDefine { if t.nextNonSpace().typ == itemDefine {
newT := New("definition") // name will be updated once we know it. newT := New("definition") // name will be updated once we know it.
newT.text = t.text newT.text = t.text
newT.Mode = t.Mode
newT.ParseName = t.ParseName newT.ParseName = t.ParseName
newT.startParse(t.funcs, t.lex, t.treeSet) newT.startParse(t.funcs, t.lex, t.treeSet)
newT.parseDefinition() newT.parseDefinition()
@ -331,19 +355,27 @@ func (t *Tree) itemList() (list *ListNode, next Node) {
} }
// textOrAction: // textOrAction:
// text | action // text | comment | action
func (t *Tree) textOrAction() Node { func (t *Tree) textOrAction() Node {
switch token := t.nextNonSpace(); token.typ { switch token := t.nextNonSpace(); token.typ {
case itemText: case itemText:
return t.newText(token.pos, token.val) return t.newText(token.pos, token.val)
case itemLeftDelim: case itemLeftDelim:
t.actionLine = token.line
defer t.clearActionLine()
return t.action() return t.action()
case itemComment:
return t.newComment(token.pos, token.val)
default: default:
t.unexpected(token, "input") t.unexpected(token, "input")
} }
return nil return nil
} }
func (t *Tree) clearActionLine() {
t.actionLine = 0
}
// Action: // Action:
// control // control
// command ("|" command)* // command ("|" command)*
@ -369,12 +401,12 @@ func (t *Tree) action() (n Node) {
t.backup() t.backup()
token := t.peek() token := t.peek()
// Do not pop variables; they persist until "end". // Do not pop variables; they persist until "end".
return t.newAction(token.pos, token.line, t.pipeline("command")) return t.newAction(token.pos, token.line, t.pipeline("command", itemRightDelim))
} }
// Pipeline: // Pipeline:
// declarations? command ('|' command)* // declarations? command ('|' command)*
func (t *Tree) pipeline(context string) (pipe *PipeNode) { func (t *Tree) pipeline(context string, end itemType) (pipe *PipeNode) {
token := t.peekNonSpace() token := t.peekNonSpace()
pipe = t.newPipeline(token.pos, token.line, nil) pipe = t.newPipeline(token.pos, token.line, nil)
// Are there declarations or assignments? // Are there declarations or assignments?
@ -415,12 +447,9 @@ decls:
} }
for { for {
switch token := t.nextNonSpace(); token.typ { switch token := t.nextNonSpace(); token.typ {
case itemRightDelim, itemRightParen: case end:
// At this point, the pipeline is complete // At this point, the pipeline is complete
t.checkPipeline(pipe, context) t.checkPipeline(pipe, context)
if token.typ == itemRightParen {
t.backup()
}
return return
case itemBool, itemCharConstant, itemComplex, itemDot, itemField, itemIdentifier, case itemBool, itemCharConstant, itemComplex, itemDot, itemField, itemIdentifier,
itemNumber, itemNil, itemRawString, itemString, itemVariable, itemLeftParen: itemNumber, itemNil, itemRawString, itemString, itemVariable, itemLeftParen:
@ -449,7 +478,7 @@ func (t *Tree) checkPipeline(pipe *PipeNode, context string) {
func (t *Tree) parseControl(allowElseIf bool, context string) (pos Pos, line int, pipe *PipeNode, list, elseList *ListNode) { func (t *Tree) parseControl(allowElseIf bool, context string) (pos Pos, line int, pipe *PipeNode, list, elseList *ListNode) {
defer t.popVars(len(t.vars)) defer t.popVars(len(t.vars))
pipe = t.pipeline(context) pipe = t.pipeline(context, itemRightDelim)
var next Node var next Node
list, next = t.itemList() list, next = t.itemList()
switch next.Type() { switch next.Type() {
@ -535,10 +564,11 @@ func (t *Tree) blockControl() Node {
token := t.nextNonSpace() token := t.nextNonSpace()
name := t.parseTemplateName(token, context) name := t.parseTemplateName(token, context)
pipe := t.pipeline(context) pipe := t.pipeline(context, itemRightDelim)
block := New(name) // name will be updated once we know it. block := New(name) // name will be updated once we know it.
block.text = t.text block.text = t.text
block.Mode = t.Mode
block.ParseName = t.ParseName block.ParseName = t.ParseName
block.startParse(t.funcs, t.lex, t.treeSet) block.startParse(t.funcs, t.lex, t.treeSet)
var end Node var end Node
@ -564,7 +594,7 @@ func (t *Tree) templateControl() Node {
if t.nextNonSpace().typ != itemRightDelim { if t.nextNonSpace().typ != itemRightDelim {
t.backup() t.backup()
// Do not pop variables; they persist until "end". // Do not pop variables; they persist until "end".
pipe = t.pipeline(context) pipe = t.pipeline(context, itemRightDelim)
} }
return t.newTemplate(token.pos, token.line, name, pipe) return t.newTemplate(token.pos, token.line, name, pipe)
} }
@ -598,13 +628,12 @@ func (t *Tree) command() *CommandNode {
switch token := t.next(); token.typ { switch token := t.next(); token.typ {
case itemSpace: case itemSpace:
continue continue
case itemError:
t.errorf("%s", token.val)
case itemRightDelim, itemRightParen: case itemRightDelim, itemRightParen:
t.backup() t.backup()
case itemPipe: case itemPipe:
// nothing here; break loop below
default: default:
t.errorf("unexpected %s in operand", token) t.unexpected(token, "operand")
} }
break break
} }
@ -659,8 +688,6 @@ func (t *Tree) operand() Node {
// A nil return means the next item is not a term. // A nil return means the next item is not a term.
func (t *Tree) term() Node { func (t *Tree) term() Node {
switch token := t.nextNonSpace(); token.typ { switch token := t.nextNonSpace(); token.typ {
case itemError:
t.errorf("%s", token.val)
case itemIdentifier: case itemIdentifier:
if !t.hasFunction(token.val) { if !t.hasFunction(token.val) {
t.errorf("function %q not defined", token.val) t.errorf("function %q not defined", token.val)
@ -683,11 +710,7 @@ func (t *Tree) term() Node {
} }
return number return number
case itemLeftParen: case itemLeftParen:
pipe := t.pipeline("parenthesized pipeline") return t.pipeline("parenthesized pipeline", itemRightParen)
if token := t.next(); token.typ != itemRightParen {
t.errorf("unclosed right paren: unexpected %s", token)
}
return pipe
case itemString, itemRawString: case itemString, itemRawString:
s, err := strconv.Unquote(token.val) s, err := strconv.Unquote(token.val)
if err != nil { if err != nil {

View file

@ -252,6 +252,13 @@ var parseTests = []parseTest{
{"comment trim left and right", "x \r\n\t{{- /* */ -}}\n\n\ty", noError, `"x""y"`}, {"comment trim left and right", "x \r\n\t{{- /* */ -}}\n\n\ty", noError, `"x""y"`},
{"block definition", `{{block "foo" .}}hello{{end}}`, noError, {"block definition", `{{block "foo" .}}hello{{end}}`, noError,
`{{template "foo" .}}`}, `{{template "foo" .}}`},
{"newline in assignment", "{{ $x \n := \n 1 \n }}", noError, "{{$x := 1}}"},
{"newline in empty action", "{{\n}}", hasError, "{{\n}}"},
{"newline in pipeline", "{{\n\"x\"\n|\nprintf\n}}", noError, `{{"x" | printf}}`},
{"newline in comment", "{{/*\nhello\n*/}}", noError, ""},
{"newline in comment", "{{-\n/*\nhello\n*/\n-}}", noError, ""},
// Errors. // Errors.
{"unclosed action", "hello{{range", hasError, ""}, {"unclosed action", "hello{{range", hasError, ""},
{"unmatched end", "{{end}}", hasError, ""}, {"unmatched end", "{{end}}", hasError, ""},
@ -350,6 +357,30 @@ func TestParseCopy(t *testing.T) {
testParse(true, t) testParse(true, t)
} }
func TestParseWithComments(t *testing.T) {
textFormat = "%q"
defer func() { textFormat = "%s" }()
tests := [...]parseTest{
{"comment", "{{/*\n\n\n*/}}", noError, "{{/*\n\n\n*/}}"},
{"comment trim left", "x \r\n\t{{- /* hi */}}", noError, `"x"{{/* hi */}}`},
{"comment trim right", "{{/* hi */ -}}\n\n\ty", noError, `{{/* hi */}}"y"`},
{"comment trim left and right", "x \r\n\t{{- /* */ -}}\n\n\ty", noError, `"x"{{/* */}}"y"`},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
tr := New(test.name)
tr.Mode = ParseComments
tmpl, err := tr.Parse(test.input, "", "", make(map[string]*Tree))
if err != nil {
t.Errorf("%q: expected error; got none", test.name)
}
if result := tmpl.Root.String(); result != test.result {
t.Errorf("%s=(%q): got\n\t%v\nexpected\n\t%v", test.name, test.input, result, test.result)
}
})
}
}
type isEmptyTest struct { type isEmptyTest struct {
name string name string
input string input string
@ -360,6 +391,7 @@ var isEmptyTests = []isEmptyTest{
{"empty", ``, true}, {"empty", ``, true},
{"nonempty", `hello`, false}, {"nonempty", `hello`, false},
{"spaces only", " \t\n \t\n", true}, {"spaces only", " \t\n \t\n", true},
{"comment only", "{{/* comment */}}", true},
{"definition", `{{define "x"}}something{{end}}`, true}, {"definition", `{{define "x"}}something{{end}}`, true},
{"definitions and space", "{{define `x`}}something{{end}}\n\n{{define `y`}}something{{end}}\n\n", true}, {"definitions and space", "{{define `x`}}something{{end}}\n\n{{define `y`}}something{{end}}\n\n", true},
{"definitions and text", "{{define `x`}}something{{end}}\nx\n{{define `y`}}something{{end}}\ny\n", false}, {"definitions and text", "{{define `x`}}something{{end}}\nx\n{{define `y`}}something{{end}}\ny\n", false},
@ -403,23 +435,38 @@ var errorTests = []parseTest{
// Check line numbers are accurate. // Check line numbers are accurate.
{"unclosed1", {"unclosed1",
"line1\n{{", "line1\n{{",
hasError, `unclosed1:2: unexpected unclosed action in command`}, hasError, `unclosed1:2: unclosed action`},
{"unclosed2", {"unclosed2",
"line1\n{{define `x`}}line2\n{{", "line1\n{{define `x`}}line2\n{{",
hasError, `unclosed2:3: unexpected unclosed action in command`}, hasError, `unclosed2:3: unclosed action`},
{"unclosed3",
"line1\n{{\"x\"\n\"y\"\n",
hasError, `unclosed3:4: unclosed action started at unclosed3:2`},
{"unclosed4",
"{{\n\n\n\n\n",
hasError, `unclosed4:6: unclosed action started at unclosed4:1`},
{"var1",
"line1\n{{\nx\n}}",
hasError, `var1:3: function "x" not defined`},
// Specific errors. // Specific errors.
{"function", {"function",
"{{foo}}", "{{foo}}",
hasError, `function "foo" not defined`}, hasError, `function "foo" not defined`},
{"comment", {"comment1",
"{{/*}}", "{{/*}}",
hasError, `unclosed comment`}, hasError, `comment1:1: unclosed comment`},
{"comment2",
"{{/*\nhello\n}}",
hasError, `comment2:1: unclosed comment`},
{"lparen", {"lparen",
"{{.X (1 2 3}}", "{{.X (1 2 3}}",
hasError, `unclosed left paren`}, hasError, `unclosed left paren`},
{"rparen", {"rparen",
"{{.X 1 2 3)}}", "{{.X 1 2 3 ) }}",
hasError, `unexpected ")"`}, hasError, `unexpected ")" in command`},
{"rparen2",
"{{(.X 1 2 3",
hasError, `unclosed action`},
{"space", {"space",
"{{`x`3}}", "{{`x`3}}",
hasError, `in operand`}, hasError, `in operand`},
@ -465,7 +512,7 @@ var errorTests = []parseTest{
hasError, `missing value for parenthesized pipeline`}, hasError, `missing value for parenthesized pipeline`},
{"multilinerawstring", {"multilinerawstring",
"{{ $v := `\n` }} {{", "{{ $v := `\n` }} {{",
hasError, `multilinerawstring:2: unexpected unclosed action`}, hasError, `multilinerawstring:2: unclosed action`},
{"rangeundefvar", {"rangeundefvar",
"{{range $k}}{{end}}", "{{range $k}}{{end}}",
hasError, `undefined variable`}, hasError, `undefined variable`},