hugolib: Process and render shortcodes in their order of appearance

Fixes #3359

hugolib/hugo_sites.go

@@ -602,8 +602,8 @@ func (h *HugoSites) Pages() Pages {
 }
 
 func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) {
-	if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {
+	if p.shortcodeState != nil && p.shortcodeState.contentShortcodes.Len() > 0 {
-		p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())
+		p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", p.shortcodeState.contentShortcodes.Len(), p.BaseFileName())
 		err := p.shortcodeState.executeShortcodesForDelta(p)
 
 		if err != nil {

hugolib/orderedMap.go

@@ -0,0 +1,100 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"fmt"
+	"sync"
+)
+
+type orderedMap struct {
+	sync.RWMutex
+	keys []interface{}
+	m    map[interface{}]interface{}
+}
+
+func newOrderedMap() *orderedMap {
+	return &orderedMap{m: make(map[interface{}]interface{})}
+}
+
+func newOrderedMapFromStringMapString(m map[string]string) *orderedMap {
+	om := newOrderedMap()
+	for k, v := range m {
+		om.Add(k, v)
+	}
+	return om
+}
+
+func (m *orderedMap) Add(k, v interface{}) {
+	m.Lock()
+	_, found := m.m[k]
+	if found {
+		panic(fmt.Sprintf("%v already added", v))
+	}
+	m.m[k] = v
+	m.keys = append(m.keys, k)
+	m.Unlock()
+
+}
+
+func (m *orderedMap) Get(k interface{}) (interface{}, bool) {
+	m.RLock()
+	defer m.RUnlock()
+	v, found := m.m[k]
+	return v, found
+}
+
+func (m *orderedMap) Contains(k interface{}) bool {
+	m.RLock()
+	defer m.RUnlock()
+	_, found := m.m[k]
+	return found
+}
+
+func (m *orderedMap) Keys() []interface{} {
+	m.RLock()
+	defer m.RUnlock()
+	return m.keys
+}
+
+func (m *orderedMap) Len() int {
+	m.RLock()
+	defer m.RUnlock()
+	return len(m.keys)
+}
+
+// Some shortcuts for known types.
+func (m *orderedMap) getShortcode(k interface{}) *shortcode {
+	v, found := m.Get(k)
+	if !found {
+		return nil
+	}
+	return v.(*shortcode)
+}
+
+func (m *orderedMap) getShortcodeRenderer(k interface{}) func() (string, error) {
+	v, found := m.Get(k)
+	if !found {
+		return nil
+	}
+	return v.(func() (string, error))
+}
+
+func (m *orderedMap) getString(k interface{}) string {
+	v, found := m.Get(k)
+	if !found {
+		return ""
+	}
+	return v.(string)
+}

hugolib/orderedMap_test.go

@@ -0,0 +1,69 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugolib
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOrderedMap(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	m := newOrderedMap()
+	m.Add("b", "vb")
+	m.Add("c", "vc")
+	m.Add("a", "va")
+	b, f1 := m.Get("b")
+
+	assert.True(f1)
+	assert.Equal(b, "vb")
+	assert.True(m.Contains("b"))
+	assert.False(m.Contains("e"))
+
+	assert.Equal([]interface{}{"b", "c", "a"}, m.Keys())
+
+}
+
+func TestOrderedMapConcurrent(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	var wg sync.WaitGroup
+
+	m := newOrderedMap()
+
+	for i := 1; i < 20; i++ {
+		wg.Add(1)
+		go func(id int) {
+			defer wg.Done()
+			key := fmt.Sprintf("key%d", id)
+			val := key + "val"
+			m.Add(key, val)
+			v, found := m.Get(key)
+			assert.True(found)
+			assert.Equal(v, val)
+			assert.True(m.Contains(key))
+			assert.True(m.Len() > 0)
+			assert.True(len(m.Keys()) > 0)
+		}(i)
+
+	}
+
+	wg.Wait()
+}

hugolib/shortcode.go

@@ -180,11 +180,11 @@ type shortcodeHandler struct {
 	p *PageWithoutContent
 
 	// This is all shortcode rendering funcs for all potential output formats.
-	contentShortcodes map[scKey]func() (string, error)
+	contentShortcodes *orderedMap
 
 	// This map contains the new or changed set of shortcodes that need
 	// to be rendered for the current output format.
-	contentShortcodesDelta map[scKey]func() (string, error)
+	contentShortcodesDelta *orderedMap
 
 	// This maps the shorcode placeholders with the rendered content.
 	// We will do (potential) partial re-rendering per output format,
@@ -192,7 +192,7 @@ type shortcodeHandler struct {
 	renderedShortcodes map[string]string
 
 	// Maps the shortcodeplaceholder with the actual shortcode.
-	shortcodes map[string]shortcode
+	shortcodes *orderedMap
 
 	// All the shortcode names in this set.
 	nameSet map[string]bool
@@ -216,8 +216,8 @@ func (s *shortcodeHandler) createShortcodePlaceholder() string {
 func newShortcodeHandler(p *Page) *shortcodeHandler {
 	return &shortcodeHandler{
 		p:                  p.withoutContent(),
-		contentShortcodes:  make(map[scKey]func() (string, error)),
+		contentShortcodes:  newOrderedMap(),
-		shortcodes:         make(map[string]shortcode),
+		shortcodes:         newOrderedMap(),
 		nameSet:            make(map[string]bool),
 		renderedShortcodes: make(map[string]string),
 	}
@@ -259,7 +259,7 @@ const innerNewlineRegexp = "\n"
 const innerCleanupRegexp = `\A<p>(.*)</p>\n\z`
 const innerCleanupExpand = "$1"
 
-func prepareShortcodeForPage(placeholder string, sc shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) {
+func prepareShortcodeForPage(placeholder string, sc *shortcode, parent *ShortcodeWithPage, p *PageWithoutContent) map[scKey]func() (string, error) {
 
 	m := make(map[scKey]func() (string, error))
 	lang := p.Lang()
@@ -277,7 +277,7 @@ func prepareShortcodeForPage(placeholder string, sc shortcode, parent *Shortcode
 
 func renderShortcode(
 	tmplKey scKey,
-	sc shortcode,
+	sc *shortcode,
 	parent *ShortcodeWithPage,
 	p *PageWithoutContent) string {
 
@@ -298,8 +298,8 @@ func renderShortcode(
 			switch innerData.(type) {
 			case string:
 				inner += innerData.(string)
-			case shortcode:
+			case *shortcode:
-				inner += renderShortcode(tmplKey, innerData.(shortcode), data, p)
+				inner += renderShortcode(tmplKey, innerData.(*shortcode), data, p)
 			default:
 				p.s.Log.ERROR.Printf("Illegal state on shortcode rendering of %q in page %q. Illegal type in inner data: %s ",
 					sc.name, p.Path(), reflect.TypeOf(innerData))
@@ -363,48 +363,51 @@ func (s *shortcodeHandler) updateDelta() bool {
 
 	contentShortcodes := s.contentShortcodesForOutputFormat(s.p.s.rc.Format)
 
-	if s.contentShortcodesDelta == nil || len(s.contentShortcodesDelta) == 0 {
+	if s.contentShortcodesDelta == nil || s.contentShortcodesDelta.Len() == 0 {
 		s.contentShortcodesDelta = contentShortcodes
 		return true
 	}
 
-	delta := make(map[scKey]func() (string, error))
+	delta := newOrderedMap()
 
-	for k, v := range contentShortcodes {
+	for _, k := range contentShortcodes.Keys() {
-		if _, found := s.contentShortcodesDelta[k]; !found {
+		if !s.contentShortcodesDelta.Contains(k) {
-			delta[k] = v
+			v, _ := contentShortcodes.Get(k)
+			delta.Add(k, v)
 		}
 	}
 
 	s.contentShortcodesDelta = delta
 
-	return len(delta) > 0
+	return delta.Len() > 0
 }
 
-func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) map[scKey]func() (string, error) {
+func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) *orderedMap {
-	contentShortcodesForOuputFormat := make(map[scKey]func() (string, error))
+	contentShortcodesForOuputFormat := newOrderedMap()
 	lang := s.p.Lang()
 
-	for shortcodePlaceholder := range s.shortcodes {
+	for _, key := range s.shortcodes.Keys() {
+		shortcodePlaceholder := key.(string)
+		//	shortcodePlaceholder := s.shortcodes.getShortcode(key)
 
 		key := newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder)
-		renderFn, found := s.contentShortcodes[key]
+		renderFn, found := s.contentShortcodes.Get(key)
 
 		if !found {
 			key.OutputFormat = ""
-			renderFn, found = s.contentShortcodes[key]
+			renderFn, found = s.contentShortcodes.Get(key)
 		}
 
 		// Fall back to HTML
 		if !found && key.Suffix != "html" {
 			key.Suffix = "html"
-			renderFn, found = s.contentShortcodes[key]
+			renderFn, found = s.contentShortcodes.Get(key)
 		}
 
 		if !found {
 			panic(fmt.Sprintf("Shortcode %q could not be found", shortcodePlaceholder))
 		}
-		contentShortcodesForOuputFormat[newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder)] = renderFn
+		contentShortcodesForOuputFormat.Add(newScKeyFromLangAndOutputFormat(lang, f, shortcodePlaceholder), renderFn)
 	}
 
 	return contentShortcodesForOuputFormat
@@ -412,27 +415,29 @@ func (s *shortcodeHandler) contentShortcodesForOutputFormat(f output.Format) map
 
 func (s *shortcodeHandler) executeShortcodesForDelta(p *PageWithoutContent) error {
 
-	for k, render := range s.contentShortcodesDelta {
+	for _, k := range s.contentShortcodesDelta.Keys() {
+		render := s.contentShortcodesDelta.getShortcodeRenderer(k)
 		renderedShortcode, err := render()
 		if err != nil {
 			return fmt.Errorf("Failed to execute shortcode in page %q: %s", p.Path(), err)
 		}
 
-		s.renderedShortcodes[k.ShortcodePlaceholder] = renderedShortcode
+		s.renderedShortcodes[k.(scKey).ShortcodePlaceholder] = renderedShortcode
 	}
 
 	return nil
 
 }
 
-func createShortcodeRenderers(shortcodes map[string]shortcode, p *PageWithoutContent) map[scKey]func() (string, error) {
+func createShortcodeRenderers(shortcodes *orderedMap, p *PageWithoutContent) *orderedMap {
 
-	shortcodeRenderers := make(map[scKey]func() (string, error))
+	shortcodeRenderers := newOrderedMap()
 
-	for k, v := range shortcodes {
+	for _, k := range shortcodes.Keys() {
-		prepared := prepareShortcodeForPage(k, v, nil, p)
+		v := shortcodes.getShortcode(k)
+		prepared := prepareShortcodeForPage(k.(string), v, nil, p)
 		for kk, vv := range prepared {
-			shortcodeRenderers[kk] = vv
+			shortcodeRenderers.Add(kk, vv)
 		}
 	}
 
@@ -444,8 +449,8 @@ var errShortCodeIllegalState = errors.New("Illegal shortcode state")
 // pageTokens state:
 // - before: positioned just before the shortcode start
 // - after: shortcode(s) consumed (plural when they are nested)
-func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (shortcode, error) {
+func (s *shortcodeHandler) extractShortcode(pt *pageTokens, p *PageWithoutContent) (*shortcode, error) {
-	sc := shortcode{}
+	sc := &shortcode{}
 	var isInner = false
 
 	var currItem item
@@ -616,7 +621,7 @@ Loop:
 
 			placeHolder := s.createShortcodePlaceholder()
 			result.WriteString(placeHolder)
-			s.shortcodes[placeHolder] = currShortcode
+			s.shortcodes.Add(placeHolder, currShortcode)
 		case tEOF:
 			break Loop
 		case tError:

hugolib/shortcode_test.go

@@ -433,16 +433,17 @@ func TestExtractShortcodes(t *testing.T) {
 			t.Fatalf("[%d] %s: Failed to compile regexp %q: %q", i, this.name, expected, err)
 		}
 
-		if strings.Count(content, shortcodePlaceholderPrefix) != len(shortCodes) {
+		if strings.Count(content, shortcodePlaceholderPrefix) != shortCodes.Len() {
-			t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, len(shortCodes))
+			t.Fatalf("[%d] %s: Not enough placeholders, found %d", i, this.name, shortCodes.Len())
 		}
 
 		if !r.MatchString(content) {
 			t.Fatalf("[%d] %s: Shortcode extract didn't match. got %q but expected %q", i, this.name, content, expected)
 		}
 
-		for placeHolder, sc := range shortCodes {
+		for _, placeHolder := range shortCodes.Keys() {
-			if !strings.Contains(content, placeHolder) {
+			sc := shortCodes.getShortcode(placeHolder)
+			if !strings.Contains(content, placeHolder.(string)) {
 				t.Fatalf("[%d] %s: Output does not contain placeholder %q", i, this.name, placeHolder)
 			}
 
@@ -753,10 +754,11 @@ NotFound: {{< thisDoesNotExist >}}
 
 }
 
-func collectAndSortShortcodes(shortcodes map[string]shortcode) []string {
+func collectAndSortShortcodes(shortcodes *orderedMap) []string {
 	var asArray []string
 
-	for key, sc := range shortcodes {
+	for _, key := range shortcodes.Keys() {
+		sc := shortcodes.getShortcode(key)
 		asArray = append(asArray, fmt.Sprintf("%s:%s", key, sc))
 	}
 
@@ -881,3 +883,48 @@ func TestScKey(t *testing.T) {
 		newDefaultScKey("IJKL"))
 
 }
+
+func TestPreserveShortcodeOrder(t *testing.T) {
+	t.Parallel()
+	assert := require.New(t)
+
+	contentTemplate := `---
+title: doc%d
+weight: %d
+---
+# doc
+
+{{< increment >}}{{< s1 >}}{{< increment >}}{{< s2 >}}{{< increment >}}{{< s3 >}}{{< increment >}}{{< s4 >}}{{< increment >}}{{< s5 >}}
+
+
+`
+
+	shortCodeTemplate := `v%d: {{ .Page.Scratch.Get "v" }}|`
+
+	var shortcodes []string
+	var content []string
+
+	shortcodes = append(shortcodes, []string{"shortcodes/increment.html", `{{ .Page.Scratch.Add "v" 1}}`}...)
+
+	for i := 1; i <= 5; i++ {
+		shortcodes = append(shortcodes, []string{fmt.Sprintf("shortcodes/s%d.html", i), fmt.Sprintf(shortCodeTemplate, i)}...)
+	}
+
+	for i := 1; i <= 3; i++ {
+		content = append(content, []string{fmt.Sprintf("p%d.md", i), fmt.Sprintf(contentTemplate, i, i)}...)
+	}
+
+	builder := newTestSitesBuilder(t).WithDefaultMultiSiteConfig()
+
+	builder.WithContent(content...).WithTemplatesAdded(shortcodes...).CreateSites().Build(BuildCfg{})
+
+	s := builder.H.Sites[0]
+	assert.Equal(3, len(s.RegularPages))
+
+	p1 := s.RegularPages[0]
+
+	if !strings.Contains(string(p1.content()), `v1: 1|v2: 2|v3: 3|v4: 4|v5: 5`) {
+		t.Fatal(p1.content())
+	}
+
+}