hugo/hugolib/testhelpers_test.go

1002 lines
25 KiB
Go
Raw Normal View History

package hugolib
import (
"bytes"
"context"
"fmt"
"image/jpeg"
"io"
2022-02-09 10:37:21 -05:00
"io/fs"
2020-02-10 06:03:41 -05:00
"math/rand"
"os"
"path/filepath"
"regexp"
"strings"
"testing"
"text/template"
2020-02-10 06:03:41 -05:00
"time"
"github.com/gohugoio/hugo/config/allconfig"
"github.com/gohugoio/hugo/config/security"
Introduce a tree map for all content This commit introduces a new data structure to store pages and their resources. This data structure is backed by radix trees. This simplies tree operations, makes all pages a bundle, and paves the way for #6310. It also solves a set of annoying issues (see list below). Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use. ``` name old time/op new time/op delta SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4) SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4) SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4) SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4) SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4) SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4) ``` Fixes #6312 Fixes #6087 Fixes #6738 Fixes #6412 Fixes #6743 Fixes #6875 Fixes #6034 Fixes #6902 Fixes #6173 Fixes #6590
2019-09-10 05:26:34 -04:00
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/google/go-cmp/cmp"
"github.com/gohugoio/hugo/parser"
"github.com/fsnotify/fsnotify"
"github.com/gohugoio/hugo/common/hexec"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/deps"
"github.com/gohugoio/hugo/resources/page"
"github.com/sanity-io/litter"
"github.com/spf13/afero"
2019-03-24 05:11:16 -04:00
"github.com/spf13/cast"
"github.com/gohugoio/hugo/helpers"
2019-03-24 05:11:16 -04:00
"github.com/gohugoio/hugo/resources/resource"
qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/hugofs"
)
var (
deepEqualsPages = qt.CmpEquals(cmp.Comparer(func(p1, p2 *pageState) bool { return p1 == p2 }))
deepEqualsOutputFormats = qt.CmpEquals(cmp.Comparer(func(o1, o2 output.Format) bool {
return o1.Name == o2.Name && o1.MediaType.Type == o2.MediaType.Type
}))
)
type sitesBuilder struct {
Cfg config.Provider
Configs *allconfig.Configs
environ []string
Fs *hugofs.Fs
T testing.TB
depsCfg deps.DepsCfg
*qt.C
logger loggers.Logger
2020-02-10 06:03:41 -05:00
rnd *rand.Rand
dumper litter.Options
// Used to test partial rebuilds.
changedFiles []string
removedFiles []string
2018-03-16 13:27:44 -04:00
// Aka the Hugo server mode.
running bool
H *HugoSites
theme string
2018-03-16 13:27:44 -04:00
// Default toml
configFormat string
configFileSet bool
2021-07-14 15:47:26 -04:00
configSet bool
2018-03-16 13:27:44 -04:00
2018-03-21 12:21:46 -04:00
// Default is empty.
// TODO(bep) revisit this and consider always setting it to something.
// Consider this in relation to using the BaseFs.PublishFs to all publishing.
workingDir string
addNothing bool
2018-03-17 14:24:02 -04:00
// Base data/content
2020-02-10 06:03:41 -05:00
contentFilePairs []filenameContent
templateFilePairs []filenameContent
i18nFilePairs []filenameContent
dataFilePairs []filenameContent
2018-03-17 14:24:02 -04:00
// Additional data/content.
// As in "use the base, but add these on top".
2020-02-10 06:03:41 -05:00
contentFilePairsAdded []filenameContent
templateFilePairsAdded []filenameContent
i18nFilePairsAdded []filenameContent
dataFilePairsAdded []filenameContent
}
type filenameContent struct {
filename string
content string
}
func newTestSitesBuilder(t testing.TB) *sitesBuilder {
v := config.New()
v.Set("publishDir", "public")
all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaning There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324
2023-12-24 13:11:05 -05:00
v.Set("disableLiveReload", true)
fs := hugofs.NewFromOld(afero.NewMemMapFs(), v)
litterOptions := litter.Options{
HidePrivateFields: true,
StripPackageNames: true,
Separator: " ",
}
return &sitesBuilder{
T: t, C: qt.New(t), Fs: fs, configFormat: "toml",
dumper: litterOptions, rnd: rand.New(rand.NewSource(time.Now().Unix())),
}
2018-03-16 13:27:44 -04:00
}
func newTestSitesBuilderFromDepsCfg(t testing.TB, d deps.DepsCfg) *sitesBuilder {
c := qt.New(t)
litterOptions := litter.Options{
HidePrivateFields: true,
StripPackageNames: true,
Separator: " ",
}
2020-02-10 06:03:41 -05:00
b := &sitesBuilder{T: t, C: c, depsCfg: d, Fs: d.Fs, dumper: litterOptions, rnd: rand.New(rand.NewSource(time.Now().Unix()))}
workingDir := d.Configs.LoadingInfo.BaseConfig.WorkingDir
b.WithWorkingDir(workingDir)
return b
}
2018-03-16 13:27:44 -04:00
func (s *sitesBuilder) Running() *sitesBuilder {
s.running = true
return s
}
func (s *sitesBuilder) WithNothingAdded() *sitesBuilder {
s.addNothing = true
return s
}
func (s *sitesBuilder) WithLogger(logger loggers.Logger) *sitesBuilder {
s.logger = logger
return s
}
2018-03-21 12:21:46 -04:00
func (s *sitesBuilder) WithWorkingDir(dir string) *sitesBuilder {
s.workingDir = filepath.FromSlash(dir)
return s
}
func (s *sitesBuilder) WithEnviron(env ...string) *sitesBuilder {
for i := 0; i < len(env); i += 2 {
s.environ = append(s.environ, fmt.Sprintf("%s=%s", env[i], env[i+1]))
}
2018-03-21 12:21:46 -04:00
return s
}
func (s *sitesBuilder) WithConfigTemplate(data any, format, configTemplate string) *sitesBuilder {
s.T.Helper()
2018-03-16 13:27:44 -04:00
if format == "" {
format = "toml"
}
templ, err := template.New("test").Parse(configTemplate)
if err != nil {
2018-03-17 14:24:02 -04:00
s.Fatalf("Template parse failed: %s", err)
2018-03-16 13:27:44 -04:00
}
var b bytes.Buffer
templ.Execute(&b, data)
2018-03-17 14:24:02 -04:00
return s.WithConfigFile(format, b.String())
}
func (s *sitesBuilder) WithViper(v config.Provider) *sitesBuilder {
s.T.Helper()
if s.configFileSet {
s.T.Fatal("WithViper: use Viper or config.toml, not both")
}
defer func() {
s.configSet = true
}()
// Write to a config file to make sure the tests follow the same code path.
var buff bytes.Buffer
m := v.Get("").(maps.Params)
s.Assert(parser.InterfaceToConfig(m, metadecoders.TOML, &buff), qt.IsNil)
return s.WithConfigFile("toml", buff.String())
2018-03-17 14:24:02 -04:00
}
func (s *sitesBuilder) WithConfigFile(format, conf string) *sitesBuilder {
s.T.Helper()
if s.configSet {
s.T.Fatal("WithConfigFile: use config.Config or config.toml, not both")
}
s.configFileSet = true
filename := s.absFilename("config." + format)
writeSource(s.T, s.Fs, filename, conf)
2018-03-16 13:27:44 -04:00
s.configFormat = format
return s
}
func (s *sitesBuilder) WithThemeConfigFile(format, conf string) *sitesBuilder {
s.T.Helper()
if s.theme == "" {
s.theme = "test-theme"
}
filename := filepath.Join("themes", s.theme, "config."+format)
writeSource(s.T, s.Fs, s.absFilename(filename), conf)
return s
}
func (s *sitesBuilder) WithSourceFile(filenameContent ...string) *sitesBuilder {
s.T.Helper()
for i := 0; i < len(filenameContent); i += 2 {
writeSource(s.T, s.Fs, s.absFilename(filenameContent[i]), filenameContent[i+1])
}
Add support for theme composition and inheritance This commit adds support for theme composition and inheritance in Hugo. With this, it helps thinking about a theme as a set of ordered components: ```toml theme = ["my-shortcodes", "base-theme", "hyde"] ``` The theme definition example above in `config.toml` creates a theme with the 3 components with presedence from left to right. So, Hugo will, for any given file, data entry etc., look first in the project, and then in `my-shortcode`, `base-theme` and lastly `hyde`. Hugo uses two different algorithms to merge the filesystems, depending on the file type: * For `i18n` and `data` files, Hugo merges deeply using the translation id and data key inside the files. * For `static`, `layouts` (templates) and `archetypes` files, these are merged on file level. So the left-most file will be chosen. The name used in the `theme` definition above must match a folder in `/your-site/themes`, e.g. `/your-site/themes/my-shortcodes`. There are plans to improve on this and get a URL scheme so this can be resolved automatically. Also note that a component that is part of a theme can have its own configuration file, e.g. `config.toml`. There are currently some restrictions to what a theme component can configure: * `params` (global and per language) * `menu` (global and per language) * `outputformats` and `mediatypes` The same rules apply here: The left-most param/menu etc. with the same ID will win. There are some hidden and experimental namespace support in the above, which we will work to improve in the future, but theme authors are encouraged to create their own namespaces to avoid naming conflicts. A final note: Themes/components can also have a `theme` definition in their `config.toml` and similar, which is the "inheritance" part of this commit's title. This is currently not supported by the Hugo theme site. We will have to wait for some "auto dependency" feature to be implemented for that to happen, but this can be a powerful feature if you want to create your own theme-variant based on others. Fixes #4460 Fixes #4450
2018-03-01 09:01:25 -05:00
return s
}
func (s *sitesBuilder) absFilename(filename string) string {
filename = filepath.FromSlash(filename)
if filepath.IsAbs(filename) {
return filename
}
if s.workingDir != "" && !strings.HasPrefix(filename, s.workingDir) {
filename = filepath.Join(s.workingDir, filename)
}
return filename
}
const commonConfigSections = `
[services]
[services.disqus]
shortname = "disqus_shortname"
[services.googleAnalytics]
2021-03-03 07:30:06 -05:00
id = "UA-ga_id"
[privacy]
[privacy.disqus]
disable = false
[privacy.googleAnalytics]
respectDoNotTrack = true
[privacy.instagram]
simple = true
[privacy.twitter]
enableDNT = true
[privacy.vimeo]
disable = false
[privacy.youtube]
disable = false
privacyEnhanced = true
`
2018-03-17 14:24:02 -04:00
func (s *sitesBuilder) WithSimpleConfigFile() *sitesBuilder {
s.T.Helper()
return s.WithSimpleConfigFileAndBaseURL("http://example.com/")
}
func (s *sitesBuilder) WithSimpleConfigFileAndBaseURL(baseURL string) *sitesBuilder {
s.T.Helper()
return s.WithSimpleConfigFileAndSettings(map[string]any{"baseURL": baseURL})
2020-06-14 05:14:56 -04:00
}
func (s *sitesBuilder) WithSimpleConfigFileAndSettings(settings any) *sitesBuilder {
2020-06-14 05:14:56 -04:00
s.T.Helper()
var buf bytes.Buffer
parser.InterfaceToConfig(settings, metadecoders.TOML, &buf)
config := buf.String() + commonConfigSections
2018-03-17 14:24:02 -04:00
return s.WithConfigFile("toml", config)
}
func (s *sitesBuilder) WithDefaultMultiSiteConfig() *sitesBuilder {
defaultMultiSiteConfig := `
baseURL = "http://example.com/blog"
paginate = 1
disablePathToLower = true
defaultContentLanguage = "en"
defaultContentLanguageInSubdir = true
[permalinks]
other = "/somewhere/else/:filename"
[Taxonomies]
tag = "tags"
[Languages]
[Languages.en]
weight = 10
title = "In English"
languageName = "English"
[[Languages.en.menu.main]]
url = "/"
name = "Home"
weight = 0
[Languages.fr]
weight = 20
title = "Le Français"
languageName = "Français"
[Languages.fr.Taxonomies]
plaque = "plaques"
[Languages.nn]
weight = 30
title = "På nynorsk"
languageName = "Nynorsk"
paginatePath = "side"
[Languages.nn.Taxonomies]
lag = "lag"
[[Languages.nn.menu.main]]
url = "/"
name = "Heim"
weight = 1
[Languages.nb]
weight = 40
title = "På bokmål"
languageName = "Bokmål"
paginatePath = "side"
[Languages.nb.Taxonomies]
lag = "lag"
` + commonConfigSections
2018-03-17 14:24:02 -04:00
return s.WithConfigFile("toml", defaultMultiSiteConfig)
}
func (s *sitesBuilder) WithSunset(in string) {
// Write a real image into one of the bundle above.
src, err := os.Open(filepath.FromSlash("testdata/sunset.jpg"))
s.Assert(err, qt.IsNil)
out, err := s.Fs.Source.Create(filepath.FromSlash(filepath.Join(s.workingDir, in)))
s.Assert(err, qt.IsNil)
_, err = io.Copy(out, src)
s.Assert(err, qt.IsNil)
out.Close()
src.Close()
}
2020-02-10 06:03:41 -05:00
func (s *sitesBuilder) createFilenameContent(pairs []string) []filenameContent {
var slice []filenameContent
s.appendFilenameContent(&slice, pairs...)
return slice
}
func (s *sitesBuilder) appendFilenameContent(slice *[]filenameContent, pairs ...string) {
if len(pairs)%2 != 0 {
panic("file content mismatch")
}
for i := 0; i < len(pairs); i += 2 {
c := filenameContent{
filename: pairs[i],
content: pairs[i+1],
}
*slice = append(*slice, c)
}
}
func (s *sitesBuilder) WithContent(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.contentFilePairs, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
2018-03-16 13:27:44 -04:00
}
func (s *sitesBuilder) WithContentAdded(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.contentFilePairsAdded, filenameContent...)
return s
}
func (s *sitesBuilder) WithTemplates(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.templateFilePairs, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
2018-03-16 13:27:44 -04:00
}
func (s *sitesBuilder) WithTemplatesAdded(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.templateFilePairsAdded, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
}
func (s *sitesBuilder) WithData(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.dataFilePairs, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
}
func (s *sitesBuilder) WithDataAdded(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.dataFilePairsAdded, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
}
func (s *sitesBuilder) WithI18n(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.i18nFilePairs, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
}
func (s *sitesBuilder) WithI18nAdded(filenameContent ...string) *sitesBuilder {
2020-02-10 06:03:41 -05:00
s.appendFilenameContent(&s.i18nFilePairsAdded, filenameContent...)
2018-03-17 14:24:02 -04:00
return s
}
func (s *sitesBuilder) EditFiles(filenameContent ...string) *sitesBuilder {
for i := 0; i < len(filenameContent); i += 2 {
filename, content := filepath.FromSlash(filenameContent[i]), filenameContent[i+1]
absFilename := s.absFilename(filename)
s.changedFiles = append(s.changedFiles, absFilename)
writeSource(s.T, s.Fs, absFilename, content)
}
return s
}
func (s *sitesBuilder) RemoveFiles(filenames ...string) *sitesBuilder {
for _, filename := range filenames {
absFilename := s.absFilename(filename)
s.removedFiles = append(s.removedFiles, absFilename)
s.Assert(s.Fs.Source.Remove(absFilename), qt.IsNil)
}
return s
}
2020-02-10 06:03:41 -05:00
func (s *sitesBuilder) writeFilePairs(folder string, files []filenameContent) *sitesBuilder {
// We have had some "filesystem ordering" bugs that we have not discovered in
// our tests running with the in memory filesystem.
// That file system is backed by a map so not sure how this helps, but some
// randomness in tests doesn't hurt.
// TODO(bep) this turns out to be more confusing than helpful.
// s.rnd.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
2020-02-10 06:03:41 -05:00
for _, fc := range files {
2018-03-21 12:21:46 -04:00
target := folder
// TODO(bep) clean up this magic.
2020-02-10 06:03:41 -05:00
if strings.HasPrefix(fc.filename, folder) {
2018-03-21 12:21:46 -04:00
target = ""
}
if s.workingDir != "" {
target = filepath.Join(s.workingDir, target)
}
2020-02-10 06:03:41 -05:00
writeSource(s.T, s.Fs, filepath.Join(target, fc.filename), fc.content)
}
return s
}
func (s *sitesBuilder) CreateSites() *sitesBuilder {
if err := s.CreateSitesE(); err != nil {
s.Fatalf("Failed to create sites: %s", err)
}
s.Assert(s.Fs.PublishDir, qt.IsNotNil)
s.Assert(s.Fs.WorkingDirReadOnly, qt.IsNotNil)
return s
}
Add /config dir support This commit adds support for a configuration directory (default `config`). The different pieces in this puzzle are: * A new `--environment` (or `-e`) flag. This can also be set with the `HUGO_ENVIRONMENT` OS environment variable. The value for `environment` defaults to `production` when running `hugo` and `development` when running `hugo server`. You can set it to any value you want (e.g. `hugo server -e "Sensible Environment"`), but as it is used to load configuration from the file system, the letter case may be important. You can get this value in your templates with `{{ hugo.Environment }}`. * A new `--configDir` flag (defaults to `config` below your project). This can also be set with `HUGO_CONFIGDIR` OS environment variable. If the `configDir` exists, the configuration files will be read and merged on top of each other from left to right; the right-most value will win on duplicates. Given the example tree below: If `environment` is `production`, the left-most `config.toml` would be the one directly below the project (this can now be omitted if you want), and then `_default/config.toml` and finally `production/config.toml`. And since these will be merged, you can just provide the environment specific configuration setting in you production config, e.g. `enableGitInfo = true`. The order within the directories will be lexical (`config.toml` and then `params.toml`). ```bash config ├── _default │   ├── config.toml │   ├── languages.toml │   ├── menus │   │   ├── menus.en.toml │   │   └── menus.zh.toml │   └── params.toml ├── development │   └── params.toml └── production ├── config.toml └── params.toml ``` Some configuration maps support the language code in the filename (e.g. `menus.en.toml`): `menus` (`menu` also works) and `params`. Also note that the only folders with "a meaning" in the above listing is the top level directories below `config`. The `menus` sub folder is just added for better organization. We use `TOML` in the example above, but Hugo also supports `JSON` and `YAML` as configuration formats. These can be mixed. Fixes #5422
2018-11-15 03:28:02 -05:00
func (s *sitesBuilder) LoadConfig() error {
if !s.configFileSet {
s.WithSimpleConfigFile()
}
flags := config.New()
flags.Set("internal", map[string]any{
"running": s.running,
"watch": s.running,
})
if s.workingDir != "" {
flags.Set("workingDir", s.workingDir)
}
res, err := allconfig.LoadConfig(allconfig.ConfigSourceDescriptor{
Fs: s.Fs.Source,
Logger: s.logger,
Flags: flags,
Environ: s.environ,
Filename: "config." + s.configFormat,
})
Add /config dir support This commit adds support for a configuration directory (default `config`). The different pieces in this puzzle are: * A new `--environment` (or `-e`) flag. This can also be set with the `HUGO_ENVIRONMENT` OS environment variable. The value for `environment` defaults to `production` when running `hugo` and `development` when running `hugo server`. You can set it to any value you want (e.g. `hugo server -e "Sensible Environment"`), but as it is used to load configuration from the file system, the letter case may be important. You can get this value in your templates with `{{ hugo.Environment }}`. * A new `--configDir` flag (defaults to `config` below your project). This can also be set with `HUGO_CONFIGDIR` OS environment variable. If the `configDir` exists, the configuration files will be read and merged on top of each other from left to right; the right-most value will win on duplicates. Given the example tree below: If `environment` is `production`, the left-most `config.toml` would be the one directly below the project (this can now be omitted if you want), and then `_default/config.toml` and finally `production/config.toml`. And since these will be merged, you can just provide the environment specific configuration setting in you production config, e.g. `enableGitInfo = true`. The order within the directories will be lexical (`config.toml` and then `params.toml`). ```bash config ├── _default │   ├── config.toml │   ├── languages.toml │   ├── menus │   │   ├── menus.en.toml │   │   └── menus.zh.toml │   └── params.toml ├── development │   └── params.toml └── production ├── config.toml └── params.toml ``` Some configuration maps support the language code in the filename (e.g. `menus.en.toml`): `menus` (`menu` also works) and `params`. Also note that the only folders with "a meaning" in the above listing is the top level directories below `config`. The `menus` sub folder is just added for better organization. We use `TOML` in the example above, but Hugo also supports `JSON` and `YAML` as configuration formats. These can be mixed. Fixes #5422
2018-11-15 03:28:02 -05:00
if err != nil {
return err
}
s.Cfg = res.LoadingInfo.Cfg
s.Configs = res
Add /config dir support This commit adds support for a configuration directory (default `config`). The different pieces in this puzzle are: * A new `--environment` (or `-e`) flag. This can also be set with the `HUGO_ENVIRONMENT` OS environment variable. The value for `environment` defaults to `production` when running `hugo` and `development` when running `hugo server`. You can set it to any value you want (e.g. `hugo server -e "Sensible Environment"`), but as it is used to load configuration from the file system, the letter case may be important. You can get this value in your templates with `{{ hugo.Environment }}`. * A new `--configDir` flag (defaults to `config` below your project). This can also be set with `HUGO_CONFIGDIR` OS environment variable. If the `configDir` exists, the configuration files will be read and merged on top of each other from left to right; the right-most value will win on duplicates. Given the example tree below: If `environment` is `production`, the left-most `config.toml` would be the one directly below the project (this can now be omitted if you want), and then `_default/config.toml` and finally `production/config.toml`. And since these will be merged, you can just provide the environment specific configuration setting in you production config, e.g. `enableGitInfo = true`. The order within the directories will be lexical (`config.toml` and then `params.toml`). ```bash config ├── _default │   ├── config.toml │   ├── languages.toml │   ├── menus │   │   ├── menus.en.toml │   │   └── menus.zh.toml │   └── params.toml ├── development │   └── params.toml └── production ├── config.toml └── params.toml ``` Some configuration maps support the language code in the filename (e.g. `menus.en.toml`): `menus` (`menu` also works) and `params`. Also note that the only folders with "a meaning" in the above listing is the top level directories below `config`. The `menus` sub folder is just added for better organization. We use `TOML` in the example above, but Hugo also supports `JSON` and `YAML` as configuration formats. These can be mixed. Fixes #5422
2018-11-15 03:28:02 -05:00
return nil
}
func (s *sitesBuilder) CreateSitesE() error {
if !s.addNothing {
if _, ok := s.Fs.Source.(*afero.OsFs); ok {
for _, dir := range []string{
"content/sect",
"layouts/_default",
"layouts/_default/_markup",
"layouts/partials",
"layouts/shortcodes",
"data",
"i18n",
} {
if err := os.MkdirAll(filepath.Join(s.workingDir, dir), 0o777); err != nil {
return fmt.Errorf("failed to create %q: %w", dir, err)
}
}
}
s.addDefaults()
s.writeFilePairs("content", s.contentFilePairsAdded)
s.writeFilePairs("layouts", s.templateFilePairsAdded)
s.writeFilePairs("data", s.dataFilePairsAdded)
s.writeFilePairs("i18n", s.i18nFilePairsAdded)
s.writeFilePairs("i18n", s.i18nFilePairs)
s.writeFilePairs("data", s.dataFilePairs)
s.writeFilePairs("content", s.contentFilePairs)
s.writeFilePairs("layouts", s.templateFilePairs)
}
if err := s.LoadConfig(); err != nil {
return fmt.Errorf("failed to load config: %w", err)
}
s.Fs.PublishDir = hugofs.NewCreateCountingFs(s.Fs.PublishDir)
depsCfg := s.depsCfg
depsCfg.Fs = s.Fs
if depsCfg.Configs.IsZero() {
depsCfg.Configs = s.Configs
}
depsCfg.TestLogger = s.logger
sites, err := NewHugoSites(depsCfg)
if err != nil {
return fmt.Errorf("failed to create sites: %w", err)
}
s.H = sites
return nil
}
func (s *sitesBuilder) BuildE(cfg BuildCfg) error {
if s.H == nil {
s.CreateSites()
}
return s.H.Build(cfg)
}
func (s *sitesBuilder) Build(cfg BuildCfg) *sitesBuilder {
s.T.Helper()
return s.build(cfg, false)
}
func (s *sitesBuilder) BuildFail(cfg BuildCfg) *sitesBuilder {
s.T.Helper()
return s.build(cfg, true)
}
func (s *sitesBuilder) changeEvents() []fsnotify.Event {
var events []fsnotify.Event
for _, v := range s.changedFiles {
events = append(events, fsnotify.Event{
Name: v,
Op: fsnotify.Write,
})
}
for _, v := range s.removedFiles {
events = append(events, fsnotify.Event{
Name: v,
Op: fsnotify.Remove,
})
}
return events
}
func (s *sitesBuilder) build(cfg BuildCfg, shouldFail bool) *sitesBuilder {
s.Helper()
defer func() {
s.changedFiles = nil
}()
if s.H == nil {
2018-03-17 14:24:02 -04:00
s.CreateSites()
}
Add support for theme composition and inheritance This commit adds support for theme composition and inheritance in Hugo. With this, it helps thinking about a theme as a set of ordered components: ```toml theme = ["my-shortcodes", "base-theme", "hyde"] ``` The theme definition example above in `config.toml` creates a theme with the 3 components with presedence from left to right. So, Hugo will, for any given file, data entry etc., look first in the project, and then in `my-shortcode`, `base-theme` and lastly `hyde`. Hugo uses two different algorithms to merge the filesystems, depending on the file type: * For `i18n` and `data` files, Hugo merges deeply using the translation id and data key inside the files. * For `static`, `layouts` (templates) and `archetypes` files, these are merged on file level. So the left-most file will be chosen. The name used in the `theme` definition above must match a folder in `/your-site/themes`, e.g. `/your-site/themes/my-shortcodes`. There are plans to improve on this and get a URL scheme so this can be resolved automatically. Also note that a component that is part of a theme can have its own configuration file, e.g. `config.toml`. There are currently some restrictions to what a theme component can configure: * `params` (global and per language) * `menu` (global and per language) * `outputformats` and `mediatypes` The same rules apply here: The left-most param/menu etc. with the same ID will win. There are some hidden and experimental namespace support in the above, which we will work to improve in the future, but theme authors are encouraged to create their own namespaces to avoid naming conflicts. A final note: Themes/components can also have a `theme` definition in their `config.toml` and similar, which is the "inheritance" part of this commit's title. This is currently not supported by the Hugo theme site. We will have to wait for some "auto dependency" feature to be implemented for that to happen, but this can be a powerful feature if you want to create your own theme-variant based on others. Fixes #4460 Fixes #4450
2018-03-01 09:01:25 -05:00
err := s.H.Build(cfg, s.changeEvents()...)
if err == nil {
logErrorCount := s.H.NumLogErrors()
if logErrorCount > 0 {
err = fmt.Errorf("logged %d errors", logErrorCount)
}
}
if err != nil && !shouldFail {
2018-03-17 14:24:02 -04:00
s.Fatalf("Build failed: %s", err)
} else if err == nil && shouldFail {
s.Fatalf("Expected error")
}
return s
}
2018-03-17 14:24:02 -04:00
func (s *sitesBuilder) addDefaults() {
var (
contentTemplate = `---
title: doc1
weight: 1
tags:
- tag1
date: "2018-02-28"
---
# doc1
*some "content"*
{{< shortcode >}}
{{< lingo >}}
`
2018-03-17 14:24:02 -04:00
defaultContent = []string{
"content/sect/doc1.en.md", contentTemplate,
"content/sect/doc1.fr.md", contentTemplate,
"content/sect/doc1.nb.md", contentTemplate,
"content/sect/doc1.nn.md", contentTemplate,
}
2019-10-28 05:52:47 -04:00
listTemplateCommon = "{{ $p := .Paginator }}{{ $p.PageNumber }}|{{ .Title }}|{{ i18n \"hello\" }}|{{ .Permalink }}|Pager: {{ template \"_internal/pagination.html\" . }}|Kind: {{ .Kind }}|Content: {{ .Content }}|Len Pages: {{ len .Pages }}|Len RegularPages: {{ len .RegularPages }}| HasParent: {{ if .Parent }}YES{{ else }}NO{{ end }}"
2018-03-17 14:24:02 -04:00
defaultTemplates = []string{
2019-10-28 05:52:47 -04:00
"_default/single.html", "Single: {{ .Title }}|{{ i18n \"hello\" }}|{{.Language.Lang}}|RelPermalink: {{ .RelPermalink }}|Permalink: {{ .Permalink }}|{{ .Content }}|Resources: {{ range .Resources }}{{ .MediaType }}: {{ .RelPermalink}} -- {{ end }}|Summary: {{ .Summary }}|Truncated: {{ .Truncated }}|Parent: {{ .Parent.Title }}",
"_default/list.html", "List Page " + listTemplateCommon,
"index.html", "{{ $p := .Paginator }}Default Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink }}|String Resource Permalink: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").Permalink }}",
"index.fr.html", "{{ $p := .Paginator }}French Home Page {{ $p.PageNumber }}: {{ .Title }}|{{ .IsHome }}|{{ i18n \"hello\" }}|{{ .Permalink }}|{{ .Site.Data.hugo.slogan }}|String Resource: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").RelPermalink }}|String Resource Permalink: {{ ( \"Hugo Pipes\" | resources.FromString \"text/pipes.txt\").Permalink }}",
"_default/terms.html", "Taxonomy Term Page " + listTemplateCommon,
"_default/taxonomy.html", "Taxonomy List Page " + listTemplateCommon,
2018-03-17 14:24:02 -04:00
// Shortcodes
"shortcodes/shortcode.html", "Shortcode: {{ i18n \"hello\" }}",
// A shortcode in multiple languages
"shortcodes/lingo.html", "LingoDefault",
"shortcodes/lingo.fr.html", "LingoFrench",
// Special templates
"404.html", "404|{{ .Lang }}|{{ .Title }}",
"robots.txt", "robots|{{ .Lang }}|{{ .Title }}",
2018-03-17 14:24:02 -04:00
}
defaultI18n = []string{
"en.yaml", `
hello:
other: "Hello"
`,
"fr.yaml", `
hello:
other: "Bonjour"
`,
}
defaultData = []string{
"hugo.toml", "slogan = \"Hugo Rocks!\"",
}
)
if len(s.contentFilePairs) == 0 {
2020-02-10 06:03:41 -05:00
s.writeFilePairs("content", s.createFilenameContent(defaultContent))
2018-03-17 14:24:02 -04:00
}
2020-02-10 06:03:41 -05:00
2018-03-17 14:24:02 -04:00
if len(s.templateFilePairs) == 0 {
2020-02-10 06:03:41 -05:00
s.writeFilePairs("layouts", s.createFilenameContent(defaultTemplates))
2018-03-17 14:24:02 -04:00
}
if len(s.dataFilePairs) == 0 {
2020-02-10 06:03:41 -05:00
s.writeFilePairs("data", s.createFilenameContent(defaultData))
2018-03-17 14:24:02 -04:00
}
if len(s.i18nFilePairs) == 0 {
2020-02-10 06:03:41 -05:00
s.writeFilePairs("i18n", s.createFilenameContent(defaultI18n))
2018-03-17 14:24:02 -04:00
}
}
func (s *sitesBuilder) Fatalf(format string, args ...any) {
s.T.Helper()
s.T.Fatalf(format, args...)
2018-03-16 13:27:44 -04:00
}
func (s *sitesBuilder) AssertFileContentFn(filename string, f func(s string) bool) {
s.T.Helper()
content := s.FileContent(filename)
if !f(content) {
s.Fatalf("Assert failed for %q in content\n%s", filename, content)
}
}
2022-02-09 10:37:21 -05:00
// Helper to migrate tests to new format.
func (s *sitesBuilder) DumpTxtar() string {
var sb strings.Builder
skipRe := regexp.MustCompile(`^(public|resources|package-lock.json|go.sum)`)
afero.Walk(s.Fs.Source, s.workingDir, func(path string, info fs.FileInfo, err error) error {
all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaning There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324
2023-12-24 13:11:05 -05:00
if err != nil {
return err
}
2022-02-09 10:37:21 -05:00
rel := strings.TrimPrefix(path, s.workingDir+"/")
if skipRe.MatchString(rel) {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info == nil || info.IsDir() {
return nil
}
sb.WriteString(fmt.Sprintf("-- %s --\n", rel))
b, err := afero.ReadFile(s.Fs.Source, path)
s.Assert(err, qt.IsNil)
sb.WriteString(strings.TrimSpace(string(b)))
sb.WriteString("\n")
return nil
})
return sb.String()
}
func (s *sitesBuilder) AssertHome(matches ...string) {
s.AssertFileContent("public/index.html", matches...)
}
func (s *sitesBuilder) AssertFileContent(filename string, matches ...string) {
s.T.Helper()
content := s.FileContent(filename)
for _, m := range matches {
lines := strings.Split(m, "\n")
for _, match := range lines {
match = strings.TrimSpace(match)
if match == "" {
continue
}
if !strings.Contains(content, match) {
all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaning There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324
2023-12-24 13:11:05 -05:00
s.Assert(content, qt.Contains, match, qt.Commentf(match+" not in: \n"+content))
}
}
}
}
func (s *sitesBuilder) AssertFileDoesNotExist(filename string) {
if s.CheckExists(filename) {
s.Fatalf("File %q exists but must not exist.", filename)
}
}
func (s *sitesBuilder) AssertImage(width, height int, filename string) {
f, err := s.Fs.WorkingDirReadOnly.Open(filename)
s.Assert(err, qt.IsNil)
defer f.Close()
cfg, err := jpeg.DecodeConfig(f)
tpl/tplimpl: Rework template management to get rid of concurrency issues This more or less completes the simplification of the template handling code in Hugo started in v0.62. The main motivation was to fix a long lasting issue about a crash in HTML content files without front matter. But this commit also comes with a big functional improvement. As we now have moved the base template evaluation to the build stage we now use the same lookup rules for `baseof` as for `list` etc. type of templates. This means that in this simple example you can have a `baseof` template for the `blog` section without having to duplicate the others: ``` layouts ├── _default │   ├── baseof.html │   ├── list.html │   └── single.html └── blog └── baseof.html ``` Also, when simplifying code, you often get rid of some double work, as shown in the "site building" benchmarks below. These benchmarks looks suspiciously good, but I have repeated the below with ca. the same result. Compared to master: ``` name old time/op new time/op delta SiteNew/Bundle_with_image-16 13.1ms ± 1% 10.5ms ± 1% -19.34% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 13.0ms ± 0% 10.7ms ± 1% -18.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 46.4ms ± 2% 43.1ms ± 1% -7.15% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 52.2ms ± 2% 47.8ms ± 1% -8.30% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 77.9ms ± 1% 70.9ms ± 1% -9.01% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 43.0ms ± 0% 37.2ms ± 1% -13.54% (p=0.029 n=4+4) SiteNew/Page_collections-16 58.2ms ± 1% 52.4ms ± 1% -9.95% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image-16 3.81MB ± 0% 2.22MB ± 0% -41.70% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 3.60MB ± 0% 2.01MB ± 0% -44.20% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 19.3MB ± 1% 14.1MB ± 0% -26.91% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 70.7MB ± 0% 69.0MB ± 0% -2.40% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 37.1MB ± 0% 31.2MB ± 0% -15.94% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 17.6MB ± 0% 10.6MB ± 0% -39.92% (p=0.029 n=4+4) SiteNew/Page_collections-16 25.9MB ± 0% 21.2MB ± 0% -17.99% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image-16 52.3k ± 0% 26.1k ± 0% -50.18% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 52.3k ± 0% 26.1k ± 0% -50.16% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 336k ± 1% 269k ± 0% -19.90% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 422k ± 0% 395k ± 0% -6.43% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 401k ± 0% 313k ± 0% -21.79% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 247k ± 0% 143k ± 0% -42.17% (p=0.029 n=4+4) SiteNew/Page_collections-16 282k ± 0% 207k ± 0% -26.55% (p=0.029 n=4+4) ``` Fixes #6716 Fixes #6760 Fixes #6768 Fixes #6778
2020-01-15 09:59:56 -05:00
s.Assert(err, qt.IsNil)
s.Assert(cfg.Width, qt.Equals, width)
s.Assert(cfg.Height, qt.Equals, height)
}
func (s *sitesBuilder) AssertNoDuplicateWrites() {
s.Helper()
hugofs.WalkFilesystems(s.Fs.PublishDir, func(fs afero.Fs) bool {
if dfs, ok := fs.(hugofs.DuplicatesReporter); ok {
s.Assert(dfs.ReportDuplicates(), qt.Equals, "")
}
return false
})
}
func (s *sitesBuilder) FileContent(filename string) string {
s.Helper()
filename = filepath.FromSlash(filename)
return readWorkingDir(s.T, s.Fs, filename)
}
func (s *sitesBuilder) AssertObject(expected string, object any) {
s.T.Helper()
got := s.dumper.Sdump(object)
expected = strings.TrimSpace(expected)
if expected != got {
fmt.Println(got)
Introduce a tree map for all content This commit introduces a new data structure to store pages and their resources. This data structure is backed by radix trees. This simplies tree operations, makes all pages a bundle, and paves the way for #6310. It also solves a set of annoying issues (see list below). Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use. ``` name old time/op new time/op delta SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4) SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4) SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4) SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4) SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4) SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4) ``` Fixes #6312 Fixes #6087 Fixes #6738 Fixes #6412 Fixes #6743 Fixes #6875 Fixes #6034 Fixes #6902 Fixes #6173 Fixes #6590
2019-09-10 05:26:34 -04:00
diff := htesting.DiffStrings(expected, got)
s.Fatalf("diff:\n%s\nexpected\n%s\ngot\n%s", diff, expected, got)
}
}
func (s *sitesBuilder) AssertFileContentRe(filename string, matches ...string) {
content := readWorkingDir(s.T, s.Fs, filename)
for _, match := range matches {
r := regexp.MustCompile("(?s)" + match)
if !r.MatchString(content) {
2018-03-16 13:27:44 -04:00
s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
}
}
}
2018-03-17 14:24:02 -04:00
func (s *sitesBuilder) CheckExists(filename string) bool {
return workingDirExists(s.Fs, filepath.Clean(filename))
2018-03-17 14:24:02 -04:00
}
func (s *sitesBuilder) GetPage(ref string) page.Page {
all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaning There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324
2023-12-24 13:11:05 -05:00
p, err := s.H.Sites[0].getPage(nil, ref)
s.Assert(err, qt.IsNil)
return p
}
Introduce a tree map for all content This commit introduces a new data structure to store pages and their resources. This data structure is backed by radix trees. This simplies tree operations, makes all pages a bundle, and paves the way for #6310. It also solves a set of annoying issues (see list below). Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use. ``` name old time/op new time/op delta SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4) SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4) SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4) SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4) SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4) SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4) ``` Fixes #6312 Fixes #6087 Fixes #6738 Fixes #6412 Fixes #6743 Fixes #6875 Fixes #6034 Fixes #6902 Fixes #6173 Fixes #6590
2019-09-10 05:26:34 -04:00
func (s *sitesBuilder) GetPageRel(p page.Page, ref string) page.Page {
all: Rework page store, add a dynacache, improve partial rebuilds, and some general spring cleaning There are some breaking changes in this commit, see #11455. Closes #11455 Closes #11549 This fixes a set of bugs (see issue list) and it is also paying some technical debt accumulated over the years. We now build with Staticcheck enabled in the CI build. The performance should be about the same as before for regular sized Hugo sites, but it should perform and scale much better to larger data sets, as objects that uses lots of memory (e.g. rendered Markdown, big JSON files read into maps with transform.Unmarshal etc.) will now get automatically garbage collected if needed. Performance on partial rebuilds when running the server in fast render mode should be the same, but the change detection should be much more accurate. A list of the notable new features: * A new dependency tracker that covers (almost) all of Hugo's API and is used to do fine grained partial rebuilds when running the server. * A new and simpler tree document store which allows fast lookups and prefix-walking in all dimensions (e.g. language) concurrently. * You can now configure an upper memory limit allowing for much larger data sets and/or running on lower specced PCs. We have lifted the "no resources in sub folders" restriction for branch bundles (e.g. sections). Memory Limit * Hugos will, by default, set aside a quarter of the total system memory, but you can set this via the OS environment variable HUGO_MEMORYLIMIT (in gigabytes). This is backed by a partitioned LRU cache used throughout Hugo. A cache that gets dynamically resized in low memory situations, allowing Go's Garbage Collector to free the memory. New Dependency Tracker: Hugo has had a rule based coarse grained approach to server rebuilds that has worked mostly pretty well, but there have been some surprises (e.g. stale content). This is now revamped with a new dependency tracker that can quickly calculate the delta given a changed resource (e.g. a content file, template, JS file etc.). This handles transitive relations, e.g. $page -> js.Build -> JS import, or $page1.Content -> render hook -> site.GetPage -> $page2.Title, or $page1.Content -> shortcode -> partial -> site.RegularPages -> $page2.Content -> shortcode ..., and should also handle changes to aggregated values (e.g. site.Lastmod) effectively. This covers all of Hugo's API with 2 known exceptions (a list that may not be fully exhaustive): Changes to files loaded with template func os.ReadFile may not be handled correctly. We recommend loading resources with resources.Get Changes to Hugo objects (e.g. Page) passed in the template context to lang.Translate may not be detected correctly. We recommend having simple i18n templates without too much data context passed in other than simple types such as strings and numbers. Note that the cachebuster configuration (when A changes then rebuild B) works well with the above, but we recommend that you revise that configuration, as it in most situations should not be needed. One example where it is still needed is with TailwindCSS and using changes to hugo_stats.json to trigger new CSS rebuilds. Document Store: Previously, a little simplified, we split the document store (where we store pages and resources) in a tree per language. This worked pretty well, but the structure made some operations harder than they needed to be. We have now restructured it into one Radix tree for all languages. Internally the language is considered to be a dimension of that tree, and the tree can be viewed in all dimensions concurrently. This makes some operations re. language simpler (e.g. finding translations is just a slice range), but the idea is that it should also be relatively inexpensive to add more dimensions if needed (e.g. role). Fixes #10169 Fixes #10364 Fixes #10482 Fixes #10630 Fixes #10656 Fixes #10694 Fixes #10918 Fixes #11262 Fixes #11439 Fixes #11453 Fixes #11457 Fixes #11466 Fixes #11540 Fixes #11551 Fixes #11556 Fixes #11654 Fixes #11661 Fixes #11663 Fixes #11664 Fixes #11669 Fixes #11671 Fixes #11807 Fixes #11808 Fixes #11809 Fixes #11815 Fixes #11840 Fixes #11853 Fixes #11860 Fixes #11883 Fixes #11904 Fixes #7388 Fixes #7425 Fixes #7436 Fixes #7544 Fixes #7882 Fixes #7960 Fixes #8255 Fixes #8307 Fixes #8863 Fixes #8927 Fixes #9192 Fixes #9324
2023-12-24 13:11:05 -05:00
p, err := s.H.Sites[0].getPage(p, ref)
Introduce a tree map for all content This commit introduces a new data structure to store pages and their resources. This data structure is backed by radix trees. This simplies tree operations, makes all pages a bundle, and paves the way for #6310. It also solves a set of annoying issues (see list below). Not a motivation behind this, but this commit also makes Hugo in general a little bit faster and more memory effective (see benchmarks). Especially for partial rebuilds on content edits, but also when taxonomies is in use. ``` name old time/op new time/op delta SiteNew/Bundle_with_image/Edit-16 1.32ms ± 8% 1.00ms ± 9% -24.42% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 1.28ms ± 0% 0.94ms ± 0% -26.26% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 33.9ms ± 2% 21.8ms ± 1% -35.67% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 40.6ms ± 1% 37.7ms ± 3% -7.20% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 56.7ms ± 0% 51.7ms ± 1% -8.82% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 19.9ms ± 2% 18.3ms ± 3% -7.64% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 37.9ms ± 4% 34.0ms ± 2% -10.28% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 10.7ms ± 0% 10.6ms ± 0% -1.15% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 10.8ms ± 0% 10.7ms ± 0% -1.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 43.2ms ± 1% 39.6ms ± 1% -8.35% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 47.6ms ± 1% 47.3ms ± 0% ~ (p=0.057 n=4+4) SiteNew/Deep_content_tree-16 73.0ms ± 1% 74.2ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Many_HTML_templates-16 37.9ms ± 0% 38.1ms ± 1% ~ (p=0.114 n=4+4) SiteNew/Page_collections-16 53.6ms ± 1% 54.7ms ± 1% +2.09% (p=0.029 n=4+4) name old alloc/op new alloc/op delta SiteNew/Bundle_with_image/Edit-16 486kB ± 0% 430kB ± 0% -11.47% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 265kB ± 0% 209kB ± 0% -21.06% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 13.6MB ± 0% 8.8MB ± 0% -34.93% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 66.5MB ± 0% 63.9MB ± 0% -3.95% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 28.8MB ± 0% 25.8MB ± 0% -10.55% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 6.16MB ± 0% 5.56MB ± 0% -9.86% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 16.9MB ± 0% 16.0MB ± 0% -5.19% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 2.28MB ± 0% 2.29MB ± 0% +0.35% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 2.07MB ± 0% 2.07MB ± 0% ~ (p=0.114 n=4+4) SiteNew/Tags_and_categories-16 14.3MB ± 0% 13.2MB ± 0% -7.30% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 69.1MB ± 0% 69.0MB ± 0% ~ (p=0.343 n=4+4) SiteNew/Deep_content_tree-16 31.3MB ± 0% 31.8MB ± 0% +1.49% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 10.8MB ± 0% 10.9MB ± 0% +1.11% (p=0.029 n=4+4) SiteNew/Page_collections-16 21.4MB ± 0% 21.6MB ± 0% +1.15% (p=0.029 n=4+4) name old allocs/op new allocs/op delta SiteNew/Bundle_with_image/Edit-16 4.74k ± 0% 3.86k ± 0% -18.57% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file/Edit-16 4.73k ± 0% 3.85k ± 0% -18.58% (p=0.029 n=4+4) SiteNew/Tags_and_categories/Edit-16 301k ± 0% 198k ± 0% -34.14% (p=0.029 n=4+4) SiteNew/Canonify_URLs/Edit-16 389k ± 0% 373k ± 0% -4.07% (p=0.029 n=4+4) SiteNew/Deep_content_tree/Edit-16 338k ± 0% 262k ± 0% -22.63% (p=0.029 n=4+4) SiteNew/Many_HTML_templates/Edit-16 102k ± 0% 88k ± 0% -13.81% (p=0.029 n=4+4) SiteNew/Page_collections/Edit-16 176k ± 0% 152k ± 0% -13.32% (p=0.029 n=4+4) SiteNew/Bundle_with_image-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Bundle_with_JSON_file-16 26.8k ± 0% 26.8k ± 0% +0.05% (p=0.029 n=4+4) SiteNew/Tags_and_categories-16 273k ± 0% 245k ± 0% -10.36% (p=0.029 n=4+4) SiteNew/Canonify_URLs-16 396k ± 0% 398k ± 0% +0.39% (p=0.029 n=4+4) SiteNew/Deep_content_tree-16 317k ± 0% 325k ± 0% +2.53% (p=0.029 n=4+4) SiteNew/Many_HTML_templates-16 146k ± 0% 147k ± 0% +0.98% (p=0.029 n=4+4) SiteNew/Page_collections-16 210k ± 0% 215k ± 0% +2.44% (p=0.029 n=4+4) ``` Fixes #6312 Fixes #6087 Fixes #6738 Fixes #6412 Fixes #6743 Fixes #6875 Fixes #6034 Fixes #6902 Fixes #6173 Fixes #6590
2019-09-10 05:26:34 -04:00
s.Assert(err, qt.IsNil)
return p
}
2018-03-17 14:24:02 -04:00
func (s *sitesBuilder) NpmInstall() hexec.Runner {
sc := security.DefaultConfig
var err error
sc.Exec.Allow, err = security.NewWhitelist("npm")
s.Assert(err, qt.IsNil)
ex := hexec.New(sc)
command, err := ex.New("npm", "install")
s.Assert(err, qt.IsNil)
return command
}
func newTestHelperFromProvider(cfg config.Provider, fs *hugofs.Fs, t testing.TB) (testHelper, *allconfig.Configs) {
res, err := allconfig.LoadConfig(allconfig.ConfigSourceDescriptor{
Flags: cfg,
Fs: fs.Source,
})
if err != nil {
t.Fatal(err)
}
return newTestHelper(res.Base, fs, t), res
}
func newTestHelper(cfg *allconfig.Config, fs *hugofs.Fs, t testing.TB) testHelper {
return testHelper{
Cfg: cfg,
Fs: fs,
C: qt.New(t),
}
}
2017-02-17 14:52:50 -05:00
type testHelper struct {
Cfg *allconfig.Config
2017-02-17 14:52:50 -05:00
Fs *hugofs.Fs
*qt.C
2017-02-17 14:52:50 -05:00
}
2017-02-17 15:14:52 -05:00
func (th testHelper) assertFileContent(filename string, matches ...string) {
th.Helper()
2017-02-17 15:14:52 -05:00
filename = th.replaceDefaultContentLanguageValue(filename)
content := readWorkingDir(th, th.Fs, filename)
2017-02-17 14:52:50 -05:00
for _, match := range matches {
2017-02-17 15:14:52 -05:00
match = th.replaceDefaultContentLanguageValue(match)
th.Assert(strings.Contains(content, match), qt.Equals, true, qt.Commentf(match+" not in: \n"+content))
2017-02-17 14:52:50 -05:00
}
}
func (th testHelper) assertFileNotExist(filename string) {
exists, err := helpers.Exists(filename, th.Fs.PublishDir)
th.Assert(err, qt.IsNil)
th.Assert(exists, qt.Equals, false)
}
2017-02-17 15:14:52 -05:00
func (th testHelper) replaceDefaultContentLanguageValue(value string) string {
defaultInSubDir := th.Cfg.DefaultContentLanguageInSubdir
replace := th.Cfg.DefaultContentLanguage + "/"
2017-02-17 15:14:52 -05:00
if !defaultInSubDir {
value = strings.Replace(value, replace, "", 1)
}
return value
}
func loadTestConfigFromProvider(cfg config.Provider) (*allconfig.Configs, error) {
workingDir := cfg.GetString("workingDir")
fs := afero.NewMemMapFs()
if workingDir != "" {
fs.MkdirAll(workingDir, 0o755)
}
res, err := allconfig.LoadConfig(allconfig.ConfigSourceDescriptor{Flags: cfg, Fs: fs})
return res, err
}
func newTestCfg(withConfig ...func(cfg config.Provider) error) (config.Provider, *hugofs.Fs) {
mm := afero.NewMemMapFs()
cfg := config.New()
cfg.Set("defaultContentLanguageInSubdir", false)
cfg.Set("publishDir", "public")
fs := hugofs.NewFromOld(hugofs.NewBaseFileDecorator(mm), cfg)
return cfg, fs
}
func newTestSitesFromConfig(t testing.TB, afs afero.Fs, tomlConfig string, layoutPathContentPairs ...string) (testHelper, *HugoSites) {
if len(layoutPathContentPairs)%2 != 0 {
t.Fatalf("Layouts must be provided in pairs")
}
c := qt.New(t)
writeToFs(t, afs, filepath.Join("content", ".gitkeep"), "")
writeToFs(t, afs, "config.toml", tomlConfig)
cfg, err := allconfig.LoadConfig(allconfig.ConfigSourceDescriptor{Fs: afs})
c.Assert(err, qt.IsNil)
fs := hugofs.NewFrom(afs, cfg.LoadingInfo.BaseConfig)
th := newTestHelper(cfg.Base, fs, t)
for i := 0; i < len(layoutPathContentPairs); i += 2 {
writeSource(t, fs, layoutPathContentPairs[i], layoutPathContentPairs[i+1])
}
h, err := NewHugoSites(deps.DepsCfg{Fs: fs, Configs: cfg})
c.Assert(err, qt.IsNil)
return th, h
}
// TODO(bep) replace these with the builder
2017-05-23 05:10:15 -04:00
func buildSingleSite(t testing.TB, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site {
t.Helper()
return buildSingleSiteExpected(t, false, false, depsCfg, buildCfg)
}
2020-12-16 06:11:32 -05:00
func buildSingleSiteExpected(t testing.TB, expectSiteInitError, expectBuildError bool, depsCfg deps.DepsCfg, buildCfg BuildCfg) *Site {
t.Helper()
b := newTestSitesBuilderFromDepsCfg(t, depsCfg).WithNothingAdded()
err := b.CreateSitesE()
2020-12-16 06:11:32 -05:00
if expectSiteInitError {
b.Assert(err, qt.Not(qt.IsNil))
return nil
} else {
b.Assert(err, qt.IsNil)
}
h := b.H
b.Assert(len(h.Sites), qt.Equals, 1)
if expectBuildError {
b.Assert(h.Build(buildCfg), qt.Not(qt.IsNil))
return nil
}
b.Assert(h.Build(buildCfg), qt.IsNil)
return h.Sites[0]
}
:sparkles: Implement Page bundling and image handling This commit is not the smallest in Hugo's history. Some hightlights include: * Page bundles (for complete articles, keeping images and content together etc.). * Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`. * Processed images are cached inside `resources/_gen/images` (default) in your project. * Symbolic links (both files and dirs) are now allowed anywhere inside /content * A new table based build summary * The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below). A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory: ```bash ▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render" benchmark old ns/op new ns/op delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86% benchmark old allocs new allocs delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30% benchmark old bytes new bytes delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64% ``` Fixes #3651 Closes #3158 Fixes #1014 Closes #2021 Fixes #1240 Updates #3757
2017-07-24 03:00:23 -04:00
func writeSourcesToSource(t *testing.T, base string, fs *hugofs.Fs, sources ...[2]string) {
for _, src := range sources {
:sparkles: Implement Page bundling and image handling This commit is not the smallest in Hugo's history. Some hightlights include: * Page bundles (for complete articles, keeping images and content together etc.). * Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`. * Processed images are cached inside `resources/_gen/images` (default) in your project. * Symbolic links (both files and dirs) are now allowed anywhere inside /content * A new table based build summary * The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below). A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory: ```bash ▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render" benchmark old ns/op new ns/op delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86% benchmark old allocs new allocs delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30% benchmark old bytes new bytes delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64% ``` Fixes #3651 Closes #3158 Fixes #1014 Closes #2021 Fixes #1240 Updates #3757
2017-07-24 03:00:23 -04:00
writeSource(t, fs, filepath.Join(base, src[0]), src[1])
}
}
func getPage(in page.Page, ref string) page.Page {
p, err := in.GetPage(ref)
if err != nil {
panic(err)
}
return p
}
2019-03-24 05:11:16 -04:00
func content(c resource.ContentProvider) string {
cc, err := c.Content(context.Background())
2019-03-24 05:11:16 -04:00
if err != nil {
panic(err)
}
ccs, err := cast.ToStringE(cc)
if err != nil {
panic(err)
}
return ccs
}