mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-21 20:46:30 -05:00
hugolib: Fix possible .Content cut
There have been one report of a site with truncated `.Content` after the Hugo `0.40.1` release. This commit fixes this so that race should not be possible anymore. It also adds a stress test with focus on content rendering and multiple output formats. Fixes #4706
This commit is contained in:
Bjørn Erik Pedersen
parent
5a9d1324d8
commit
90d0d83097
8 changed files with 191 additions and 56 deletions
|
|
@ -559,14 +559,22 @@ func (h *HugoSites) setupTranslations() {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *Site) preparePagesForRender(start bool) {
|
||||
func (s *Site) preparePagesForRender(start bool) error {
|
||||
for _, p := range s.Pages {
|
||||
p.setContentInit(start)
|
||||
if err := p.initMainOutputFormat(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range s.headlessPages {
|
||||
p.setContentInit(start)
|
||||
if err := p.initMainOutputFormat(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Pages returns all pages for all sites.
|
||||
|
|
|
|||
|
|
@ -234,7 +234,9 @@ func (h *HugoSites) render(config *BuildCfg) error {
|
|||
|
||||
isRenderingSite := s == s2
|
||||
|
||||
s2.preparePagesForRender(isRenderingSite && i == 0)
|
||||
if err := s2.preparePagesForRender(isRenderingSite && i == 0); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -734,6 +734,119 @@ func TestChangeDefaultLanguage(t *testing.T) {
|
|||
b.AssertFileContent("public/sect/doc2/index.html", "Single", "Hello")
|
||||
}
|
||||
|
||||
// https://github.com/gohugoio/hugo/issues/4706
|
||||
func TestContentStressTest(t *testing.T) {
|
||||
b := newTestSitesBuilder(t)
|
||||
|
||||
numPages := 500
|
||||
|
||||
contentTempl := `
|
||||
---
|
||||
%s
|
||||
title: %q
|
||||
weight: %d
|
||||
multioutput: %t
|
||||
---
|
||||
|
||||
# Header
|
||||
|
||||
CONTENT
|
||||
|
||||
The End.
|
||||
`
|
||||
|
||||
contentTempl = strings.Replace(contentTempl, "CONTENT", strings.Repeat(`
|
||||
|
||||
## Another header
|
||||
|
||||
Some text. Some more text.
|
||||
|
||||
`, 100), -1)
|
||||
|
||||
var content []string
|
||||
defaultOutputs := `outputs: ["html", "json", "rss" ]`
|
||||
|
||||
for i := 1; i <= numPages; i++ {
|
||||
outputs := defaultOutputs
|
||||
multioutput := true
|
||||
if i%3 == 0 {
|
||||
outputs = `outputs: ["json"]`
|
||||
multioutput = false
|
||||
}
|
||||
section := "s1"
|
||||
if i%10 == 0 {
|
||||
section = "s2"
|
||||
}
|
||||
content = append(content, []string{fmt.Sprintf("%s/page%d.md", section, i), fmt.Sprintf(contentTempl, outputs, fmt.Sprintf("Title %d", i), i, multioutput)}...)
|
||||
}
|
||||
|
||||
content = append(content, []string{"_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("Home %d", 0), 0, true)}...)
|
||||
content = append(content, []string{"s1/_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("S %d", 1), 1, true)}...)
|
||||
content = append(content, []string{"s2/_index.md", fmt.Sprintf(contentTempl, defaultOutputs, fmt.Sprintf("S %d", 2), 2, true)}...)
|
||||
|
||||
b.WithSimpleConfigFile()
|
||||
b.WithTemplates("layouts/_default/single.html", `Single: {{ .Content }}`)
|
||||
b.WithTemplates("layouts/_default/myview.html", `View: {{ len .Content }}`)
|
||||
b.WithTemplates("layouts/_default/single.json", `Single JSON: {{ .Content }}`)
|
||||
b.WithTemplates("layouts/_default/list.html", `
|
||||
Page: {{ .Paginator.PageNumber }}
|
||||
P: {{ path.Join .Path }}
|
||||
List: {{ len .Paginator.Pages }}|List Content: {{ len .Content }}
|
||||
{{ $shuffled := where .Site.RegularPages "Params.multioutput" true | shuffle }}
|
||||
{{ $first5 := $shuffled | first 5 }}
|
||||
L1: {{ len .Site.RegularPages }} L2: {{ len $first5 }}
|
||||
{{ range $i, $e := $first5 }}
|
||||
Render {{ $i }}: {{ .Render "myview" }}
|
||||
{{ end }}
|
||||
END
|
||||
`)
|
||||
|
||||
b.WithContent(content...)
|
||||
|
||||
b.CreateSites().Build(BuildCfg{})
|
||||
|
||||
contentMatchers := []string{"<h2 id=\"another-header\">Another header</h2>", "<h2 id=\"another-header-99\">Another header</h2>", "<p>The End.</p>"}
|
||||
|
||||
for i := 1; i <= numPages; i++ {
|
||||
if i%3 != 0 {
|
||||
section := "s1"
|
||||
if i%10 == 0 {
|
||||
section = "s2"
|
||||
}
|
||||
checkContent(b, fmt.Sprintf("public/%s/page%d/index.html", section, i), 8343, contentMatchers...)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 1; i <= numPages; i++ {
|
||||
section := "s1"
|
||||
if i%10 == 0 {
|
||||
section = "s2"
|
||||
}
|
||||
checkContent(b, fmt.Sprintf("public/%s/page%d/index.json", section, i), 8348, contentMatchers...)
|
||||
}
|
||||
|
||||
checkContent(b, "public/s1/index.html", 184, "P: s1/_index.md\nList: 10|List Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335\n\nRender 1: View: 8335\n\nRender 2: View: 8335\n\nRender 3: View: 8335\n\nRender 4: View: 8335\n\nEND\n")
|
||||
checkContent(b, "public/s2/index.html", 184, "P: s2/_index.md\nList: 10|List Content: 8335", "Render 4: View: 8335\n\nEND")
|
||||
checkContent(b, "public/index.html", 181, "P: _index.md\nList: 10|List Content: 8335", "4: View: 8335\n\nEND")
|
||||
|
||||
// Chek paginated pages
|
||||
for i := 2; i <= 9; i++ {
|
||||
checkContent(b, fmt.Sprintf("public/page/%d/index.html", i), 181, fmt.Sprintf("Page: %d", i), "Content: 8335\n\n\nL1: 500 L2: 5\n\nRender 0: View: 8335", "Render 4: View: 8335\n\nEND")
|
||||
}
|
||||
}
|
||||
|
||||
func checkContent(s *sitesBuilder, filename string, length int, matches ...string) {
|
||||
content := readDestination(s.T, s.Fs, filename)
|
||||
for _, match := range matches {
|
||||
if !strings.Contains(content, match) {
|
||||
s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
|
||||
}
|
||||
}
|
||||
if len(content) != length {
|
||||
s.Fatalf("got %d expected %d", len(content), length)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableOfContentsInShortcodes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
|
|
|||
|
|
@ -416,7 +416,6 @@ type pageInit struct {
|
|||
languageInit sync.Once
|
||||
pageMenusInit sync.Once
|
||||
pageMetaInit sync.Once
|
||||
pageOutputInit sync.Once
|
||||
renderingConfigInit sync.Once
|
||||
withoutContentInit sync.Once
|
||||
}
|
||||
|
|
@ -1178,6 +1177,24 @@ func (p *Page) subResourceTargetPathFactory(base string) string {
|
|||
return path.Join(p.relTargetPathBase, base)
|
||||
}
|
||||
|
||||
func (p *Page) initMainOutputFormat() error {
|
||||
if p.mainPageOutput != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
outFormat := p.outputFormats[0]
|
||||
pageOutput, err := newPageOutput(p, false, false, outFormat)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to create output page for type %q for page %q: %s", outFormat.Name, p.pathOrTitle(), err)
|
||||
}
|
||||
|
||||
p.mainPageOutput = pageOutput
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func (p *Page) setContentInit(start bool) error {
|
||||
|
||||
if start {
|
||||
|
|
@ -1962,12 +1979,17 @@ func (p *Page) updatePageDates() {
|
|||
|
||||
// copy creates a copy of this page with the lazy sync.Once vars reset
|
||||
// so they will be evaluated again, for word count calculations etc.
|
||||
func (p *Page) copy() *Page {
|
||||
func (p *Page) copy(initContent bool) *Page {
|
||||
p.contentInitMu.Lock()
|
||||
c := *p
|
||||
p.contentInitMu.Unlock()
|
||||
c.pageInit = &pageInit{}
|
||||
c.pageContentInit = &pageContentInit{}
|
||||
if initContent {
|
||||
if len(p.outputFormats) < 2 {
|
||||
panic(fmt.Sprintf("programming error: page %q should not need to rebuild content as it has only %d outputs", p.Path(), len(p.outputFormats)))
|
||||
}
|
||||
c.pageContentInit = &pageContentInit{}
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ func (p *PageOutput) targetPath(addends ...string) (string, error) {
|
|||
return tp, nil
|
||||
}
|
||||
|
||||
func newPageOutput(p *Page, createCopy bool, f output.Format) (*PageOutput, error) {
|
||||
func newPageOutput(p *Page, createCopy, initContent bool, f output.Format) (*PageOutput, error) {
|
||||
// TODO(bep) This is only needed for tests and we should get rid of it.
|
||||
if p.targetPathDescriptorPrototype == nil {
|
||||
if err := p.initPaths(); err != nil {
|
||||
|
|
@ -64,7 +64,7 @@ func newPageOutput(p *Page, createCopy bool, f output.Format) (*PageOutput, erro
|
|||
}
|
||||
|
||||
if createCopy {
|
||||
p = p.copy()
|
||||
p = p.copy(initContent)
|
||||
}
|
||||
|
||||
td, err := p.createTargetPathDescriptor(f)
|
||||
|
|
@ -82,8 +82,8 @@ func newPageOutput(p *Page, createCopy bool, f output.Format) (*PageOutput, erro
|
|||
|
||||
// copy creates a copy of this PageOutput with the lazy sync.Once vars reset
|
||||
// so they will be evaluated again, for word count calculations etc.
|
||||
func (p *PageOutput) copyWithFormat(f output.Format) (*PageOutput, error) {
|
||||
c, err := newPageOutput(p.Page, true, f)
|
||||
func (p *PageOutput) copyWithFormat(f output.Format, initContent bool) (*PageOutput, error) {
|
||||
c, err := newPageOutput(p.Page, true, initContent, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -92,7 +92,7 @@ func (p *PageOutput) copyWithFormat(f output.Format) (*PageOutput, error) {
|
|||
}
|
||||
|
||||
func (p *PageOutput) copy() (*PageOutput, error) {
|
||||
return p.copyWithFormat(p.outputFormat)
|
||||
return p.copyWithFormat(p.outputFormat, false)
|
||||
}
|
||||
|
||||
func (p *PageOutput) layouts(layouts ...string) ([]string, error) {
|
||||
|
|
@ -142,24 +142,6 @@ func (p *PageOutput) Render(layout ...string) template.HTML {
|
|||
}
|
||||
|
||||
func (p *Page) Render(layout ...string) template.HTML {
|
||||
p.pageOutputInit.Do(func() {
|
||||
if p.mainPageOutput != nil {
|
||||
return
|
||||
}
|
||||
// If Render is called in a range loop, the page output isn't available.
|
||||
// So, create one.
|
||||
outFormat := p.outputFormats[0]
|
||||
pageOutput, err := newPageOutput(p, true, outFormat)
|
||||
|
||||
if err != nil {
|
||||
p.s.Log.ERROR.Printf("Failed to create output page for type %q for page %q: %s", outFormat.Name, p.pathOrTitle(), err)
|
||||
return
|
||||
}
|
||||
|
||||
p.mainPageOutput = pageOutput
|
||||
|
||||
})
|
||||
|
||||
return p.mainPageOutput.Render(layout...)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -279,8 +279,8 @@ func doTestPaginator(t *testing.T, useViper bool) {
|
|||
require.NoError(t, err)
|
||||
|
||||
pages := createTestPages(s, 12)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
n1.Data["Pages"] = pages
|
||||
|
||||
var paginator1 *Pager
|
||||
|
|
@ -306,7 +306,7 @@ func doTestPaginator(t *testing.T, useViper bool) {
|
|||
require.Equal(t, paginator1, samePaginator)
|
||||
|
||||
pp, _ := s.NewPage("test")
|
||||
p, _ := newPageOutput(pp, false, output.HTMLFormat)
|
||||
p, _ := newPageOutput(pp, false, false, output.HTMLFormat)
|
||||
|
||||
_, err = p.Paginator()
|
||||
require.NotNil(t, err)
|
||||
|
|
@ -315,7 +315,7 @@ func doTestPaginator(t *testing.T, useViper bool) {
|
|||
func TestPaginatorWithNegativePaginate(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := newTestSite(t, "paginate", -1)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
_, err := n1.Paginator()
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
|
@ -378,8 +378,8 @@ func doTestPaginate(t *testing.T, useViper bool) {
|
|||
}
|
||||
|
||||
pages := createTestPages(s, 6)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
|
||||
var paginator1, paginator2 *Pager
|
||||
|
||||
|
|
@ -404,7 +404,7 @@ func doTestPaginate(t *testing.T, useViper bool) {
|
|||
require.Equal(t, paginator2, paginator1.Next())
|
||||
|
||||
pp, err := s.NewPage("test")
|
||||
p, _ := newPageOutput(pp, false, output.HTMLFormat)
|
||||
p, _ := newPageOutput(pp, false, false, output.HTMLFormat)
|
||||
|
||||
_, err = p.Paginate(pages)
|
||||
require.NotNil(t, err)
|
||||
|
|
@ -413,7 +413,7 @@ func doTestPaginate(t *testing.T, useViper bool) {
|
|||
func TestInvalidOptions(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := newTestSite(t)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
|
||||
_, err := n1.Paginate(createTestPages(s, 1), 1, 2)
|
||||
require.NotNil(t, err)
|
||||
|
|
@ -431,7 +431,7 @@ func TestPaginateWithNegativePaginate(t *testing.T) {
|
|||
s, err := NewSiteForCfg(deps.DepsCfg{Cfg: cfg, Fs: fs})
|
||||
require.NoError(t, err)
|
||||
|
||||
n, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
|
||||
_, err = n.Paginate(createTestPages(s, 2))
|
||||
require.NotNil(t, err)
|
||||
|
|
@ -458,8 +458,8 @@ func TestPaginatePages(t *testing.T) {
|
|||
func TestPaginatorFollowedByPaginateShouldFail(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := newTestSite(t, "paginate", 10)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
|
||||
_, err := n1.Paginator()
|
||||
require.Nil(t, err)
|
||||
|
|
@ -475,8 +475,8 @@ func TestPaginateFollowedByDifferentPaginateShouldFail(t *testing.T) {
|
|||
t.Parallel()
|
||||
s := newTestSite(t, "paginate", 10)
|
||||
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, output.HTMLFormat)
|
||||
n1, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
n2, _ := newPageOutput(s.newHomePage(), false, false, output.HTMLFormat)
|
||||
|
||||
p1 := createTestPages(s, 2)
|
||||
p2 := createTestPages(s, 10)
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ func headlessPagesPublisher(s *Site, wg *sync.WaitGroup) {
|
|||
// Avoid double work.
|
||||
continue
|
||||
}
|
||||
pageOutput, err := newPageOutput(page, false, outFormat)
|
||||
pageOutput, err := newPageOutput(page, false, false, outFormat)
|
||||
if err == nil {
|
||||
page.mainPageOutput = pageOutput
|
||||
err = pageOutput.renderResources()
|
||||
|
|
@ -92,23 +92,20 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
|
|||
|
||||
for i, outFormat := range page.outputFormats {
|
||||
|
||||
if outFormat != page.s.rc.Format {
|
||||
// Will be rendered ... later.
|
||||
continue
|
||||
}
|
||||
|
||||
var (
|
||||
pageOutput *PageOutput
|
||||
err error
|
||||
)
|
||||
|
||||
if i == 0 {
|
||||
pageOutput, err = newPageOutput(page, false, outFormat)
|
||||
page.mainPageOutput = pageOutput
|
||||
}
|
||||
|
||||
if outFormat != page.s.rc.Format {
|
||||
// Will be rendered ... later.
|
||||
continue
|
||||
}
|
||||
|
||||
if pageOutput == nil {
|
||||
pageOutput, err = page.mainPageOutput.copyWithFormat(outFormat)
|
||||
pageOutput = page.mainPageOutput
|
||||
} else {
|
||||
pageOutput, err = page.mainPageOutput.copyWithFormat(outFormat, true)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -116,6 +113,10 @@ func pageRenderer(s *Site, pages <-chan *Page, results chan<- error, wg *sync.Wa
|
|||
continue
|
||||
}
|
||||
|
||||
if pageOutput == nil {
|
||||
panic("no pageOutput")
|
||||
}
|
||||
|
||||
// We only need to re-publish the resources if the output format is different
|
||||
// from all of the previous (e.g. the "amp" use case).
|
||||
shouldRender := i == 0
|
||||
|
|
@ -291,7 +292,7 @@ func (s *Site) render404() error {
|
|||
htmlOut := output.HTMLFormat
|
||||
htmlOut.BaseName = "404"
|
||||
|
||||
pageOutput, err := newPageOutput(p, false, htmlOut)
|
||||
pageOutput, err := newPageOutput(p, false, false, htmlOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -373,7 +374,7 @@ func (s *Site) renderRobotsTXT() error {
|
|||
|
||||
rLayouts := []string{"robots.txt", "_default/robots.txt", "_internal/_default/robots.txt"}
|
||||
|
||||
pageOutput, err := newPageOutput(p, false, output.RobotsTxtFormat)
|
||||
pageOutput, err := newPageOutput(p, false, false, output.RobotsTxtFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ type sitesBuilder struct {
|
|||
Fs *hugofs.Fs
|
||||
T testing.TB
|
||||
|
||||
logger *jww.Notepad
|
||||
|
||||
dumper litter.Options
|
||||
|
||||
// Aka the Hugo server mode.
|
||||
|
|
@ -88,6 +90,11 @@ func (s *sitesBuilder) Running() *sitesBuilder {
|
|||
return s
|
||||
}
|
||||
|
||||
func (s *sitesBuilder) WithLogger(logger *jww.Notepad) *sitesBuilder {
|
||||
s.logger = logger
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {
|
||||
s.workingDir = dir
|
||||
return s
|
||||
|
|
@ -282,7 +289,7 @@ func (s *sitesBuilder) CreateSites() *sitesBuilder {
|
|||
s.Cfg = cfg
|
||||
}
|
||||
|
||||
sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Running: s.running})
|
||||
sites, err := NewHugoSites(deps.DepsCfg{Fs: s.Fs, Cfg: s.Cfg, Logger: s.logger, Running: s.running})
|
||||
if err != nil {
|
||||
s.Fatalf("Failed to create sites: %s", err)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue