Misc resource fixes/improvements

* Add --pprof flag to server to enable profile debugging.
* Don't cache the resource content, it seem to eat memory on bigger sites.
* Keep --printMemoryUsag running in server

Fixes #11974
This commit is contained in:
Bjørn Erik Pedersen 2024-02-02 16:00:48 +01:00
parent d0788b96ae
commit 2873324898
7 changed files with 58 additions and 65 deletions

View file

@ -22,6 +22,7 @@ import (
"os"
"os/signal"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
@ -341,8 +342,12 @@ func (r *rootCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args
if r.buildWatch {
defer r.timeTrack(time.Now(), "Built")
}
err := b.build()
return err
close, err := b.build()
if err != nil {
return err
}
close()
return nil
}()
if err != nil {
return err
@ -411,6 +416,7 @@ func (r *rootCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
MaxEntries: 1,
OnEvict: func(key int32, value *hugolib.HugoSites) {
value.Close()
runtime.GC()
},
})

View file

@ -361,34 +361,32 @@ func (c *hugoBuilder) newWatcher(pollIntervalStr string, dirList ...string) (*wa
return watcher, nil
}
func (c *hugoBuilder) build() error {
func (c *hugoBuilder) build() (func(), error) {
stopProfiling, err := c.initProfiling()
if err != nil {
return err
return nil, err
}
defer func() {
if stopProfiling != nil {
stopProfiling()
}
}()
if err := c.fullBuild(false); err != nil {
return err
return nil, err
}
if !c.r.quiet {
c.r.Println()
h, err := c.hugo()
if err != nil {
return err
return nil, err
}
h.PrintProcessingStats(os.Stdout)
c.r.Println()
}
return nil
return func() {
if stopProfiling != nil {
stopProfiling()
}
}, nil
}
func (c *hugoBuilder) buildSites(noBuildLock bool) (err error) {

View file

@ -25,6 +25,7 @@ import (
"io"
"net"
"net/http"
_ "net/http/pprof"
"net/url"
"os"
"os/signal"
@ -451,6 +452,7 @@ type serverCommand struct {
tlsCertFile string
tlsKeyFile string
tlsAuto bool
pprof bool
serverPort int
liveReloadPort int
serverWatch bool
@ -465,6 +467,11 @@ func (c *serverCommand) Name() string {
}
func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
if c.pprof {
go func() {
http.ListenAndServe("localhost:8080", nil)
}()
}
// Watch runs its own server as part of the routine
if c.serverWatch {
@ -487,15 +494,19 @@ func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, arg
}
var close func()
err := func() error {
defer c.r.timeTrack(time.Now(), "Built")
err := c.build()
var err error
close, err = c.build()
return err
}()
if err != nil {
return err
}
defer close()
return c.serve()
}
@ -520,6 +531,7 @@ of a second, you will be able to save and see your changes nearly instantly.`
cmd.Flags().StringVarP(&c.tlsCertFile, "tlsCertFile", "", "", "path to TLS certificate file")
cmd.Flags().StringVarP(&c.tlsKeyFile, "tlsKeyFile", "", "", "path to TLS key file")
cmd.Flags().BoolVar(&c.tlsAuto, "tlsAuto", false, "generate and use locally-trusted certificates.")
cmd.Flags().BoolVar(&c.pprof, "pprof", false, "enable the pprof server (port 8080)")
cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL")

View file

@ -720,9 +720,11 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
h.pageTrees.treeTaxonomyEntries.DeletePrefix("")
if delete {
_, ok := h.pageTrees.treePages.LongestPrefixAll(pathInfo.Base())
if ok {
h.pageTrees.treePages.DeleteAll(pathInfo.Base())
h.pageTrees.resourceTrees.DeleteAll(pathInfo.Base())
if pathInfo.IsBundle() {
// Assume directory removed.
h.pageTrees.treePages.DeletePrefixAll(pathInfo.Base() + "/")

View file

@ -101,10 +101,10 @@ func TestRebuildEditTextFileInBranchBundle(t *testing.T) {
func TestRebuildRenameTextFileInLeafBundle(t *testing.T) {
b := TestRunning(t, rebuildFilesSimple)
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.")
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
b.RenameFile("content/mysection/mysectionbundle/mysectionbundletext.txt", "content/mysection/mysectionbundle/mysectionbundletext2.txt").Build()
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.")
b.AssertFileContent("public/mysection/mysectionbundle/index.html", "mysectionbundletext2", "My Section Bundle Text 2 Content.", "Len Resources: 2|")
b.AssertRenderCountPage(3)
b.AssertRenderCountContent(3)
}

View file

@ -343,7 +343,7 @@ func GetTestInfoForResource(r resource.Resource) GenericResourceTestInfo {
// genericResource represents a generic linkable resource.
type genericResource struct {
*resourceContent
publishInit *sync.Once
sd ResourceSourceDescriptor
paths internal.ResourcePaths
@ -412,11 +412,18 @@ func (l *genericResource) cloneTo(targetPath string) resource.Resource {
}
func (l *genericResource) Content(context.Context) (any, error) {
if err := l.initContent(); err != nil {
return nil, err
r, err := l.ReadSeekCloser()
if err != nil {
return "", err
}
defer r.Close()
return l.content, nil
var b []byte
b, err = io.ReadAll(r)
if err != nil {
return "", err
}
return string(b), nil
}
func (r *genericResource) Err() resource.ResourceError {
@ -527,28 +534,6 @@ func (l *genericResource) Title() string {
return l.title
}
func (l *genericResource) initContent() error {
var err error
l.contentInit.Do(func() {
var r hugio.ReadSeekCloser
r, err = l.ReadSeekCloser()
if err != nil {
return
}
defer r.Close()
var b []byte
b, err = io.ReadAll(r)
if err != nil {
return
}
l.content = string(b)
})
return err
}
func (l *genericResource) getSpec() *Spec {
return l.spec
}
@ -588,12 +573,9 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
r := rc.clone()
if u.content != nil {
r.contentInit.Do(func() {
r.content = *u.content
r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(r.content), nil
}
})
r.sd.OpenReadSeekCloser = func() (hugio.ReadSeekCloser, error) {
return hugio.NewReadSeekerNoOpCloserFromString(*u.content), nil
}
}
r.sd.MediaType = u.mediaType
@ -620,7 +602,7 @@ func (rc *genericResource) cloneWithUpdates(u *transformationUpdate) (baseResour
}
func (l genericResource) clone() *genericResource {
l.resourceContent = &resourceContent{}
l.publishInit = &sync.Once{}
return &l
}
@ -633,13 +615,6 @@ type targetPather interface {
TargetPath() string
}
type resourceContent struct {
content string
contentInit sync.Once
publishInit sync.Once
}
type resourceHash struct {
value string
size int64

View file

@ -163,15 +163,15 @@ func (r *Spec) NewResource(rd ResourceSourceDescriptor) (resource.Resource, erro
}
gr := &genericResource{
Staler: &AtomicStaler{},
h: &resourceHash{},
paths: rp,
spec: r,
sd: rd,
params: make(map[string]any),
name: rd.Name,
title: rd.Name,
resourceContent: &resourceContent{},
Staler: &AtomicStaler{},
h: &resourceHash{},
publishInit: &sync.Once{},
paths: rp,
spec: r,
sd: rd,
params: make(map[string]any),
name: rd.Name,
title: rd.Name,
}
if rd.MediaType.MainType == "image" {