Make replaceShortcodeTokens rewrite the input slice

Currently a `[]byte` copy is returned. In most cases this is the safe thing to do, but we should just modify/grow the slice as needed.

This is faster and consumes less memory:

```
benchmark                             old ns/op     new ns/op     delta
BenchmarkReplaceShortcodeTokens-4     7350          4419          -39.88%

benchmark                             old allocs     new allocs     delta
BenchmarkReplaceShortcodeTokens-4     5              1              -80.00%

benchmark                             old bytes     new bytes     delta
BenchmarkReplaceShortcodeTokens-4     4816          1152          -76.08%
```

This commit is aso a small spring cleaning of duplicated code in the different `PageConvert` methods.

Fixes #1516

hugolib/handler_page.go

@@ -56,25 +56,7 @@ type markdownHandler struct {
 
 func (h markdownHandler) Extensions() []string { return []string{"mdown", "markdown", "md"} }
 func (h markdownHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
-	p.ProcessShortcodes(t)
+	return commonConvert(p, t)
-
-	tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
-	if len(p.contentShortCodes) > 0 {
-		replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
-			tmpContent, tmpTableOfContents)
-		if err != nil {
-			jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-		tmpContent = replaced[0]
-		tmpTableOfContents = replaced[1]
-	}
-
-	p.Content = helpers.BytesToHTML(tmpContent)
-	p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
-	return HandledResult{err: nil}
 }
 
 type htmlHandler struct {
@@ -84,21 +66,18 @@ type htmlHandler struct {
 func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
 func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
 	p.ProcessShortcodes(t)
-	var content []byte
 	var err error
 
 	if len(p.contentShortCodes) > 0 {
-		content, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
+		p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
 
 		if err != nil {
-			jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
+			jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
 			return HandledResult{err: err}
 		}
-	} else {
-		content = p.rawContent
 	}
 
-	p.Content = helpers.BytesToHTML(content)
+	p.Content = helpers.BytesToHTML(p.rawContent)
 	return HandledResult{err: nil}
 }
 
@@ -108,27 +87,7 @@ type asciidocHandler struct {
 
 func (h asciidocHandler) Extensions() []string { return []string{"asciidoc", "adoc", "ad"} }
 func (h asciidocHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
-	p.ProcessShortcodes(t)
+	return commonConvert(p, t)
-
-	// TODO(spf13) Add/Refactor AsciiDoc Logic here
-	tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
-	if len(p.contentShortCodes) > 0 {
-		replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
-			tmpContent, tmpTableOfContents)
-		if err != nil {
-			jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-		tmpContent = replaced[0]
-		tmpTableOfContents = replaced[1]
-	}
-
-	p.Content = helpers.BytesToHTML(tmpContent)
-	p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
-	//err := p.Convert()
-	return HandledResult{page: p, err: nil}
 }
 
 type rstHandler struct {
@@ -137,25 +96,7 @@ type rstHandler struct {
 
 func (h rstHandler) Extensions() []string { return []string{"rest", "rst"} }
 func (h rstHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
-	p.ProcessShortcodes(t)
+	return commonConvert(p, t)
-
-	tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
-
-	if len(p.contentShortCodes) > 0 {
-		replaced, err := replaceShortcodeTokensInsources(shortcodePlaceholderPrefix, p.contentShortCodes,
-			tmpContent, tmpTableOfContents)
-		if err != nil {
-			jww.FATAL.Printf("Fail to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-		tmpContent = replaced[0]
-		tmpTableOfContents = replaced[1]
-	}
-
-	p.Content = helpers.BytesToHTML(tmpContent)
-	p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-
-	return HandledResult{err: nil}
 }
 
 type mmarkHandler struct {
@@ -164,21 +105,27 @@ type mmarkHandler struct {
 
 func (h mmarkHandler) Extensions() []string { return []string{"mmark"} }
 func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
+	return commonConvert(p, t)
+}
+
+func commonConvert(p *Page, t tpl.Template) HandledResult {
 	p.ProcessShortcodes(t)
 
-	tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)))
+	var err error
+
+	renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
 
 	if len(p.contentShortCodes) > 0 {
-		tmpContentWithTokensReplaced, err := replaceShortcodeTokens(tmpContent, shortcodePlaceholderPrefix, p.contentShortCodes)
+		renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes)
 
 		if err != nil {
-			jww.FATAL.Printf("Fail to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
+			jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
 			return HandledResult{err: err}
-		} else {
-			tmpContent = tmpContentWithTokensReplaced
 		}
 	}
 
+	tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent)
+
 	p.Content = helpers.BytesToHTML(tmpContent)
 	p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
 

hugolib/page.go

@@ -204,6 +204,7 @@ func (p *Page) setSummary() {
 			p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0
 		}
 
+		// TODO(bep) consider doing this once only
 		renderedHeader := p.renderBytes(header)
 		if len(p.contentShortCodes) > 0 {
 			tmpContentWithTokensReplaced, err :=

hugolib/shortcode.go

@@ -453,30 +453,14 @@ Loop:
 
 }
 
-// replaceShortcodeTokensInsources calls replaceShortcodeTokens for every source given.
-func replaceShortcodeTokensInsources(prefix string, replacements map[string]string, sources ...[]byte) (b [][]byte, err error) {
-	result := make([][]byte, len(sources))
-	for i, s := range sources {
-		b, err := replaceShortcodeTokens(s, prefix, replacements)
-
-		if err != nil {
-			return nil, err
-		}
-		result[i] = b
-	}
-	return result, nil
-}
-
 // Replace prefixed shortcode tokens (HUGOSHORTCODE-1, HUGOSHORTCODE-2) with the real content.
+// Note: This function will rewrite the input slice.
 func replaceShortcodeTokens(source []byte, prefix string, replacements map[string]string) ([]byte, error) {
 
 	if len(replacements) == 0 {
 		return source, nil
 	}
 
-	buff := bp.GetBuffer()
-	defer bp.PutBuffer(buff)
-
 	sourceLen := len(source)
 	start := 0
 
@@ -507,28 +491,14 @@ func replaceShortcodeTokens(source []byte, prefix string, replacements map[strin
 			}
 		}
 
-		oldVal := source[j:end]
+		// This and other cool slice tricks: https://github.com/golang/go/wiki/SliceTricks
-		_, err := buff.Write(source[start:j])
+		source = append(source[:j], append(newVal, source[end:]...)...)
-		if err != nil {
-			return nil, errors.New("buff write failed")
-		}
-		_, err = buff.Write(newVal)
-		if err != nil {
-			return nil, errors.New("buff write failed")
-		}
-		start = j + len(oldVal)
 
 		k = bytes.Index(source[start:], pre)
-	}
+
-	_, err := buff.Write(source[start:])
-	if err != nil {
-		return nil, errors.New("buff write failed")
 	}
 
-	bc := make([]byte, buff.Len(), buff.Len())
+	return source, nil
-	copy(bc, buff.Bytes())
-
-	return bc, nil
 }
 
 func getShortcodeTemplate(name string, t tpl.Template) *template.Template {

hugolib/shortcode_test.go

@@ -405,6 +405,14 @@ func TestReplaceShortcodeTokens(t *testing.T) {
 		expect       interface{}
 	}{
 		{"Hello {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello World."},
+		{"Hello {@{@PREFIX-1@}@.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, false},
+		{"{@{@PREFIX2-1@}@}", "PREFIX2", map[string]string{"{@{@PREFIX2-1@}@}": "World"}, "World"},
+		{"Hello World!", "PREFIX2", map[string]string{}, "Hello World!"},
+		{"!{@{@PREFIX-1@}@}", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "!World"},
+		{"{@{@PREFIX-1@}@}!", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "World!"},
+		{"!{@{@PREFIX-1@}@}!", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "!World!"},
+		{"@{@PREFIX-1@}@}", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "@{@PREFIX-1@}@}"},
+		{"Hello {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "To You My Old Friend Who Told Me This Fantastic Story"}, "Hello To You My Old Friend Who Told Me This Fantastic Story."},
 		{"A {@{@A-1@}@} asdf {@{@A-2@}@}.", "A", map[string]string{"{@{@A-1@}@}": "v1", "{@{@A-2@}@}": "v2"}, "A v1 asdf v2."},
 		{"Hello {@{@PREFIX2-1@}@}. Go {@{@PREFIX2-2@}@}, Go, Go {@{@PREFIX2-3@}@} Go Go!.", "PREFIX2", map[string]string{"{@{@PREFIX2-1@}@}": "Europe", "{@{@PREFIX2-2@}@}": "Jonny", "{@{@PREFIX2-3@}@}": "Johnny"}, "Hello Europe. Go Jonny, Go, Go Johnny Go Go!."},
 		{"A {@{@PREFIX-2@}@} {@{@PREFIX-1@}@}.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "A", "{@{@PREFIX-2@}@}": "B"}, "A B A."},
@@ -420,11 +428,11 @@ func TestReplaceShortcodeTokens(t *testing.T) {
 		{"Hello <p>{@{@PREFIX-1@}@}. END</p>.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello <p>World. END</p>."},
 		{"<p>Hello {@{@PREFIX-1@}@}</p>. END.", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "<p>Hello World</p>. END."},
 		{"Hello <p>{@{@PREFIX-1@}@}12", "PREFIX", map[string]string{"{@{@PREFIX-1@}@}": "World"}, "Hello <p>World12"},
-		// Make sure the buffering expands as needed
 		{"Hello {@{@P-1@}@}. {@{@P-1@}@}-{@{@P-1@}@} {@{@P-1@}@} {@{@P-1@}@} {@{@P-1@}@} END", "P", map[string]string{"{@{@P-1@}@}": strings.Repeat("BC", 100)},
 			fmt.Sprintf("Hello %s. %s-%s %s %s %s END",
 				strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100), strings.Repeat("BC", 100))},
 	} {
+
 		results, err := replaceShortcodeTokens([]byte(this.input), this.prefix, this.replacements)
 
 		if b, ok := this.expect.(bool); ok && !b {