hugo/commands/server.go

1221 lines
33 KiB
Go
Raw Normal View History

// Copyright 2023 The Hugo Authors. All rights reserved.
//
2015-11-23 22:16:36 -05:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
2015-11-23 22:16:36 -05:00
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commands
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"sync"
"sync/atomic"
"github.com/bep/mclib"
"os/signal"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"syscall"
2014-09-22 09:45:05 -04:00
"time"
"github.com/bep/debounce"
"github.com/bep/simplecobra"
"github.com/fsnotify/fsnotify"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/common/urls"
"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/hugolib"
"github.com/gohugoio/hugo/hugolib/filesystems"
"github.com/gohugoio/hugo/livereload"
"github.com/gohugoio/hugo/tpl"
"github.com/gohugoio/hugo/transform"
"github.com/gohugoio/hugo/transform/livereloadinject"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"github.com/spf13/fsync"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
)
var (
logDuplicateTemplateExecuteRe = regexp.MustCompile(`: template: .*?:\d+:\d+: executing ".*?"`)
logDuplicateTemplateParseRe = regexp.MustCompile(`: template: .*?:\d+:\d*`)
)
var logReplacer = strings.NewReplacer(
"can't", "cant", // Chroma lexer doesn't do well with "can't"
"*hugolib.pageState", "page.Page", // Page is the public interface.
"Rebuild failed:", "",
)
2015-11-12 10:23:41 -05:00
const (
configChangeConfig = "config file"
configChangeGoMod = "go.mod file"
configChangeGoWork = "go work file"
)
func newHugoBuilder(r *rootCommand, s *serverCommand, onConfigLoaded ...func(reloaded bool) error) *hugoBuilder {
return &hugoBuilder{
r: r,
s: s,
visitedURLs: types.NewEvictingStringQueue(100),
fullRebuildSem: semaphore.NewWeighted(1),
debounce: debounce.New(4 * time.Second),
onConfigLoaded: func(reloaded bool) error {
for _, wc := range onConfigLoaded {
if err := wc(reloaded); err != nil {
return err
}
}
return nil
},
}
}
func newServerCommand() *serverCommand {
// Flags.
var uninstall bool
c := &serverCommand{
quit: make(chan bool),
commands: []simplecobra.Commander{
&simpleCommand{
name: "trust",
short: "Install the local CA in the system trust store.",
run: func(ctx context.Context, cd *simplecobra.Commandeer, r *rootCommand, args []string) error {
action := "-install"
if uninstall {
action = "-uninstall"
}
os.Args = []string{action}
return mclib.RunMain()
},
withc: func(cmd *cobra.Command, r *rootCommand) {
cmd.Flags().BoolVar(&uninstall, "uninstall", false, "Uninstall the local CA (but do not delete it).")
},
},
},
}
return c
}
func (c *serverCommand) Commands() []simplecobra.Commander {
return c.commands
}
type countingStatFs struct {
afero.Fs
statCounter uint64
}
func (fs *countingStatFs) Stat(name string) (os.FileInfo, error) {
f, err := fs.Fs.Stat(name)
if err == nil {
if !f.IsDir() {
atomic.AddUint64(&fs.statCounter, 1)
}
}
return f, err
}
// dynamicEvents contains events that is considered dynamic, as in "not static".
// Both of these categories will trigger a new build, but the asset events
// does not fit into the "navigate to changed" logic.
type dynamicEvents struct {
ContentEvents []fsnotify.Event
AssetEvents []fsnotify.Event
}
type fileChangeDetector struct {
sync.Mutex
current map[string]string
prev map[string]string
irrelevantRe *regexp.Regexp
}
func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
f.Lock()
defer f.Unlock()
f.current[name] = md5sum
}
:sparkles: Implement Page bundling and image handling This commit is not the smallest in Hugo's history. Some hightlights include: * Page bundles (for complete articles, keeping images and content together etc.). * Bundled images can be processed in as many versions/sizes as you need with the three methods `Resize`, `Fill` and `Fit`. * Processed images are cached inside `resources/_gen/images` (default) in your project. * Symbolic links (both files and dirs) are now allowed anywhere inside /content * A new table based build summary * The "Total in nn ms" now reports the total including the handling of the files inside /static. So if it now reports more than you're used to, it is just **more real** and probably faster than before (see below). A site building benchmark run compared to `v0.31.1` shows that this should be slightly faster and use less memory: ```bash ▶ ./benchSite.sh "TOML,num_langs=.*,num_root_sections=5,num_pages=(500|1000),tags_per_page=5,shortcodes,render" benchmark old ns/op new ns/op delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 101785785 78067944 -23.30% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 185481057 149159919 -19.58% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 103149918 85679409 -16.94% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 203515478 169208775 -16.86% benchmark old allocs new allocs delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 532464 391539 -26.47% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1056549 772702 -26.87% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 555974 406630 -26.86% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 1086545 789922 -27.30% benchmark old bytes new bytes delta BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 53243246 43598155 -18.12% BenchmarkSiteBuilding/TOML,num_langs=1,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 105811617 86087116 -18.64% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=500,tags_per_page=5,shortcodes,render-4 54558852 44545097 -18.35% BenchmarkSiteBuilding/TOML,num_langs=3,num_root_sections=5,num_pages=1000,tags_per_page=5,shortcodes,render-4 106903858 86978413 -18.64% ``` Fixes #3651 Closes #3158 Fixes #1014 Closes #2021 Fixes #1240 Updates #3757
2017-07-24 03:00:23 -04:00
func (f *fileChangeDetector) PrepareNew() {
if f == nil {
return
2014-01-29 17:50:31 -05:00
}
f.Lock()
defer f.Unlock()
if f.current == nil {
f.current = make(map[string]string)
f.prev = make(map[string]string)
return
}
f.prev = make(map[string]string)
for k, v := range f.current {
f.prev[k] = v
}
f.current = make(map[string]string)
}
func (f *fileChangeDetector) changed() []string {
if f == nil {
return nil
}
f.Lock()
defer f.Unlock()
var c []string
for k, v := range f.current {
vv, found := f.prev[k]
if !found || v != vv {
c = append(c, k)
}
}
return f.filterIrrelevant(c)
}
func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
var filtered []string
for _, v := range in {
if !f.irrelevantRe.MatchString(v) {
filtered = append(filtered, v)
}
}
return filtered
}
type fileServer struct {
baseURLs []string
roots []string
errorTemplate func(err any) (io.Reader, error)
c *serverCommand
}
func (f *fileServer) createEndpoint(i int) (*http.ServeMux, net.Listener, string, string, error) {
r := f.c.r
baseURL := f.baseURLs[i]
root := f.roots[i]
port := f.c.serverPorts[i].p
listener := f.c.serverPorts[i].ln
logger := f.c.r.logger
r.Printf("Environment: %q\n", f.c.hugoTry().Deps.Site.Hugo().Environment)
Add /config dir support This commit adds support for a configuration directory (default `config`). The different pieces in this puzzle are: * A new `--environment` (or `-e`) flag. This can also be set with the `HUGO_ENVIRONMENT` OS environment variable. The value for `environment` defaults to `production` when running `hugo` and `development` when running `hugo server`. You can set it to any value you want (e.g. `hugo server -e "Sensible Environment"`), but as it is used to load configuration from the file system, the letter case may be important. You can get this value in your templates with `{{ hugo.Environment }}`. * A new `--configDir` flag (defaults to `config` below your project). This can also be set with `HUGO_CONFIGDIR` OS environment variable. If the `configDir` exists, the configuration files will be read and merged on top of each other from left to right; the right-most value will win on duplicates. Given the example tree below: If `environment` is `production`, the left-most `config.toml` would be the one directly below the project (this can now be omitted if you want), and then `_default/config.toml` and finally `production/config.toml`. And since these will be merged, you can just provide the environment specific configuration setting in you production config, e.g. `enableGitInfo = true`. The order within the directories will be lexical (`config.toml` and then `params.toml`). ```bash config ├── _default │   ├── config.toml │   ├── languages.toml │   ├── menus │   │   ├── menus.en.toml │   │   └── menus.zh.toml │   └── params.toml ├── development │   └── params.toml └── production ├── config.toml └── params.toml ``` Some configuration maps support the language code in the filename (e.g. `menus.en.toml`): `menus` (`menu` also works) and `params`. Also note that the only folders with "a meaning" in the above listing is the top level directories below `config`. The `menus` sub folder is just added for better organization. We use `TOML` in the example above, but Hugo also supports `JSON` and `YAML` as configuration formats. These can be mixed. Fixes #5422
2018-11-15 03:28:02 -05:00
if i == 0 {
if f.c.renderToDisk {
r.Println("Serving pages from disk")
} else if f.c.renderStaticToDisk {
r.Println("Serving pages from memory and static files from disk")
} else {
r.Println("Serving pages from memory")
}
}
var httpFs *afero.HttpFs
f.c.withConf(func(conf *commonConfig) {
httpFs = afero.NewHttpFs(conf.fs.PublishDirServer)
})
fs := filesOnlyFs{httpFs.Dir(path.Join("/", root))}
if i == 0 && f.c.fastRenderMode {
r.Println("Running in Fast Render Mode. For full rebuilds on change: hugo server --disableFastRender")
}
// We're only interested in the path
u, err := url.Parse(baseURL)
if err != nil {
return nil, nil, "", "", fmt.Errorf("invalid baseURL: %w", err)
}
decorate := func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if f.c.showErrorInBrowser {
// First check the error state
err := f.c.getErrorWithContext()
if err != nil {
f.c.errState.setWasErr(true)
w.WriteHeader(500)
r, err := f.errorTemplate(err)
if err != nil {
logger.Errorln(err)
}
port = 1313
f.c.withConf(func(conf *commonConfig) {
if lrport := conf.configs.GetFirstLanguageConfig().BaseURLLiveReload().Port(); lrport != 0 {
port = lrport
}
})
lr := *u
lr.Host = fmt.Sprintf("%s:%d", lr.Hostname(), port)
fmt.Fprint(w, injectLiveReloadScript(r, lr))
return
}
}
if f.c.noHTTPCache {
2017-09-22 14:05:19 -04:00
w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0")
w.Header().Set("Pragma", "no-cache")
}
var serverConfig config.Server
f.c.withConf(func(conf *commonConfig) {
serverConfig = conf.configs.Base.Server
})
// Ignore any query params for the operations below.
requestURI, _ := url.PathUnescape(strings.TrimSuffix(r.RequestURI, "?"+r.URL.RawQuery))
for _, header := range serverConfig.MatchHeaders(requestURI) {
w.Header().Set(header.Key, header.Value)
}
if redirect := serverConfig.MatchRedirect(requestURI); !redirect.IsZero() {
// fullName := filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))
doRedirect := true
// This matches Netlify's behaviour and is needed for SPA behaviour.
// See https://docs.netlify.com/routing/redirects/rewrites-proxies/
if !redirect.Force {
path := filepath.Clean(strings.TrimPrefix(requestURI, u.Path))
2022-09-13 05:33:42 -04:00
if root != "" {
path = filepath.Join(root, path)
}
var fs afero.Fs
f.c.withConf(func(conf *commonConfig) {
fs = conf.fs.PublishDirServer
})
2022-09-13 05:33:42 -04:00
fi, err := fs.Stat(path)
if err == nil {
if fi.IsDir() {
// There will be overlapping directories, so we
// need to check for a file.
2022-09-13 05:33:42 -04:00
_, err = fs.Stat(filepath.Join(path, "index.html"))
doRedirect = err != nil
} else {
doRedirect = false
}
}
}
if doRedirect {
2022-09-13 05:33:42 -04:00
switch redirect.Status {
case 404:
w.WriteHeader(404)
file, err := fs.Open(strings.TrimPrefix(redirect.To, u.Path))
2022-09-13 05:33:42 -04:00
if err == nil {
defer file.Close()
io.Copy(w, file)
} else {
fmt.Fprintln(w, "<h1>Page Not Found</h1>")
}
return
case 200:
if r2 := f.rewriteRequest(r, strings.TrimPrefix(redirect.To, u.Path)); r2 != nil {
requestURI = redirect.To
r = r2
}
2022-09-13 05:33:42 -04:00
default:
w.Header().Set("Content-Type", "")
http.Redirect(w, r, redirect.To, redirect.Status)
return
2022-09-13 05:33:42 -04:00
}
}
}
if f.c.fastRenderMode && f.c.errState.buildErr() == nil {
if strings.HasSuffix(requestURI, "/") || strings.HasSuffix(requestURI, "html") || strings.HasSuffix(requestURI, "htm") {
if !f.c.visitedURLs.Contains(requestURI) {
// If not already on stack, re-render that single page.
if err := f.c.partialReRender(requestURI); err != nil {
f.c.handleBuildErr(err, fmt.Sprintf("Failed to render %q", requestURI))
if f.c.showErrorInBrowser {
http.Redirect(w, r, requestURI, http.StatusMovedPermanently)
return
}
}
}
f.c.visitedURLs.Add(requestURI)
}
}
h.ServeHTTP(w, r)
})
}
fileserver := decorate(http.FileServer(fs))
mu := http.NewServeMux()
if u.Path == "" || u.Path == "/" {
mu.Handle("/", fileserver)
} else {
mu.Handle(u.Path, http.StripPrefix(u.Path, fileserver))
}
if r.IsTestRun() {
var shutDownOnce sync.Once
mu.HandleFunc("/__stop", func(w http.ResponseWriter, r *http.Request) {
shutDownOnce.Do(func() {
close(f.c.quit)
})
})
}
endpoint := net.JoinHostPort(f.c.serverInterface, strconv.Itoa(port))
return mu, listener, u.String(), endpoint, nil
}
func (f *fileServer) rewriteRequest(r *http.Request, toPath string) *http.Request {
r2 := new(http.Request)
*r2 = *r
r2.URL = new(url.URL)
*r2.URL = *r.URL
r2.URL.Path = toPath
r2.Header.Set("X-Rewrite-Original-URI", r.URL.RequestURI())
return r2
}
type filesOnlyFs struct {
fs http.FileSystem
}
func (fs filesOnlyFs) Open(name string) (http.File, error) {
f, err := fs.fs.Open(name)
if err != nil {
return nil, err
}
return noDirFile{f}, nil
}
type noDirFile struct {
http.File
}
func (f noDirFile) Readdir(count int) ([]os.FileInfo, error) {
return nil, nil
}
type serverCommand struct {
r *rootCommand
commands []simplecobra.Commander
*hugoBuilder
quit chan bool // Closed when the server should shut down. Used in tests only.
serverPorts []serverPortListener
doLiveReload bool
// Flags.
renderToDisk bool
renderStaticToDisk bool
navigateToChanged bool
serverAppend bool
serverInterface string
tlsCertFile string
tlsKeyFile string
tlsAuto bool
serverPort int
liveReloadPort int
serverWatch bool
noHTTPCache bool
disableLiveReload bool
disableFastRender bool
disableBrowserError bool
}
func (c *serverCommand) Name() string {
return "server"
}
func (c *serverCommand) Run(ctx context.Context, cd *simplecobra.Commandeer, args []string) error {
// Watch runs its own server as part of the routine
if c.serverWatch {
watchDirs, err := c.getDirList()
if err != nil {
return err
}
watchGroups := helpers.ExtractAndGroupRootPaths(watchDirs)
for _, group := range watchGroups {
c.r.Printf("Watching for changes in %s\n", group)
}
watcher, err := c.newWatcher(c.r.poll, watchDirs...)
if err != nil {
return err
}
defer watcher.Close()
}
err := func() error {
defer c.r.timeTrack(time.Now(), "Built")
err := c.build()
return err
}()
if err != nil {
return err
}
return c.serve()
}
func (c *serverCommand) Init(cd *simplecobra.Commandeer) error {
cmd := cd.CobraCommand
cmd.Short = "A high performance webserver"
cmd.Long = `Hugo provides its own webserver which builds and serves the site.
While hugo server is high performance, it is a webserver with limited options.
'hugo server' will avoid writing the rendered and served content to disk,
preferring to store it in memory.
By default hugo will also watch your files for any changes you make and
automatically rebuild the site. It will then live reload any open browser pages
and push the latest content to them. As most Hugo sites are built in a fraction
of a second, you will be able to save and see your changes nearly instantly.`
cmd.Aliases = []string{"serve"}
cmd.Flags().IntVarP(&c.serverPort, "port", "p", 1313, "port on which the server will listen")
cmd.Flags().IntVar(&c.liveReloadPort, "liveReloadPort", -1, "port for live reloading (i.e. 443 in HTTPS proxy situations)")
cmd.Flags().StringVarP(&c.serverInterface, "bind", "", "127.0.0.1", "interface to which the server will bind")
cmd.Flags().StringVarP(&c.tlsCertFile, "tlsCertFile", "", "", "path to TLS certificate file")
cmd.Flags().StringVarP(&c.tlsKeyFile, "tlsKeyFile", "", "", "path to TLS key file")
cmd.Flags().BoolVar(&c.tlsAuto, "tlsAuto", false, "generate and use locally-trusted certificates.")
cmd.Flags().BoolVarP(&c.serverWatch, "watch", "w", true, "watch filesystem for changes and recreate as needed")
cmd.Flags().BoolVar(&c.noHTTPCache, "noHTTPCache", false, "prevent HTTP caching")
cmd.Flags().BoolVarP(&c.serverAppend, "appendPort", "", true, "append port to baseURL")
cmd.Flags().BoolVar(&c.disableLiveReload, "disableLiveReload", false, "watch without enabling live browser reload on rebuild")
cmd.Flags().BoolVar(&c.navigateToChanged, "navigateToChanged", false, "navigate to changed content file on live browser reload")
cmd.Flags().BoolVar(&c.renderToDisk, "renderToDisk", false, "serve all files from disk (default is from memory)")
cmd.Flags().BoolVar(&c.renderStaticToDisk, "renderStaticToDisk", false, "serve static files from disk and dynamic files from memory")
cmd.Flags().BoolVar(&c.disableFastRender, "disableFastRender", false, "enables full re-renders on changes")
cmd.Flags().BoolVar(&c.disableBrowserError, "disableBrowserError", false, "do not show build errors in the browser")
cmd.Flags().String("memstats", "", "log memory usage to this file")
cmd.Flags().String("meminterval", "100ms", "interval to poll memory usage (requires --memstats), valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\".")
cmd.Flags().SetAnnotation("tlsCertFile", cobra.BashCompSubdirsInDir, []string{})
cmd.Flags().SetAnnotation("tlsKeyFile", cobra.BashCompSubdirsInDir, []string{})
r := cd.Root.Command.(*rootCommand)
applyLocalFlagsBuild(cmd, r)
return nil
}
func (c *serverCommand) PreRun(cd, runner *simplecobra.Commandeer) error {
c.r = cd.Root.Command.(*rootCommand)
c.hugoBuilder = newHugoBuilder(
c.r,
c,
func(reloaded bool) error {
if !reloaded {
if err := c.createServerPorts(cd); err != nil {
return err
}
if (c.tlsCertFile == "" || c.tlsKeyFile == "") && c.tlsAuto {
c.withConfE(func(conf *commonConfig) error {
return c.createCertificates(conf)
})
}
}
if err := c.setBaseURLsInConfig(); err != nil {
return err
}
if !reloaded && c.fastRenderMode {
c.withConf(func(conf *commonConfig) {
conf.fs.PublishDir = hugofs.NewHashingFs(conf.fs.PublishDir, c.changeDetector)
conf.fs.PublishDirStatic = hugofs.NewHashingFs(conf.fs.PublishDirStatic, c.changeDetector)
})
}
return nil
},
)
destinationFlag := cd.CobraCommand.Flags().Lookup("destination")
c.renderToDisk = c.renderToDisk || (destinationFlag != nil && destinationFlag.Changed)
c.doLiveReload = !c.disableLiveReload
c.fastRenderMode = !c.disableFastRender
c.showErrorInBrowser = c.doLiveReload && !c.disableBrowserError
if c.fastRenderMode {
// For now, fast render mode only. It should, however, be fast enough
// for the full variant, too.
c.changeDetector = &fileChangeDetector{
// We use this detector to decide to do a Hot reload of a single path or not.
// We need to filter out source maps and possibly some other to be able
// to make that decision.
irrelevantRe: regexp.MustCompile(`\.map$`),
}
c.changeDetector.PrepareNew()
}
err := c.loadConfig(cd, true)
if err != nil {
return err
}
return nil
}
func (c *serverCommand) setBaseURLsInConfig() error {
if len(c.serverPorts) == 0 {
panic("no server ports set")
}
return c.withConfE(func(conf *commonConfig) error {
for i, language := range conf.configs.Languages {
isMultiHost := conf.configs.IsMultihost
var serverPort int
if isMultiHost {
serverPort = c.serverPorts[i].p
} else {
serverPort = c.serverPorts[0].p
}
langConfig := conf.configs.LanguageConfigMap[language.Lang]
baseURLStr, err := c.fixURL(langConfig.BaseURL, c.r.baseURL, serverPort)
if err != nil {
return err
}
baseURL, err := urls.NewBaseURLFromString(baseURLStr)
if err != nil {
return fmt.Errorf("failed to create baseURL from %q: %s", baseURLStr, err)
}
baseURLLiveReload := baseURL
if c.liveReloadPort != -1 {
baseURLLiveReload, _ = baseURLLiveReload.WithPort(c.liveReloadPort)
}
langConfig.C.SetBaseURL(baseURL, baseURLLiveReload)
}
return nil
})
}
func (c *serverCommand) getErrorWithContext() any {
errCount := c.errCount()
if errCount == 0 {
return nil
}
m := make(map[string]any)
m["Error"] = cleanErrorLog(c.r.logger.Errors())
m["Version"] = hugo.BuildVersionString()
ferrors := herrors.UnwrapFileErrorsWithErrorContext(c.errState.buildErr())
m["Files"] = ferrors
return m
}
func (c *serverCommand) createCertificates(conf *commonConfig) error {
hostname := "localhost"
if c.r.baseURL != "" {
u, err := url.Parse(c.r.baseURL)
if err != nil {
return err
}
hostname = u.Hostname()
}
// For now, store these in the Hugo cache dir.
// Hugo should probably introduce some concept of a less temporary application directory.
keyDir := filepath.Join(conf.configs.LoadingInfo.BaseConfig.CacheDir, "_mkcerts")
// Create the directory if it doesn't exist.
if _, err := os.Stat(keyDir); os.IsNotExist(err) {
if err := os.MkdirAll(keyDir, 0777); err != nil {
return err
}
}
c.tlsCertFile = filepath.Join(keyDir, fmt.Sprintf("%s.pem", hostname))
c.tlsKeyFile = filepath.Join(keyDir, fmt.Sprintf("%s-key.pem", hostname))
// Check if the certificate already exists and is valid.
certPEM, err := os.ReadFile(c.tlsCertFile)
if err == nil {
rootPem, err := os.ReadFile(filepath.Join(mclib.GetCAROOT(), "rootCA.pem"))
if err == nil {
if err := c.verifyCert(rootPem, certPEM, hostname); err == nil {
c.r.Println("Using existing", c.tlsCertFile, "and", c.tlsKeyFile)
return nil
}
}
}
c.r.Println("Creating TLS certificates in", keyDir)
// Yes, this is unfortunate, but it's currently the only way to use Mkcert as a library.
os.Args = []string{"-cert-file", c.tlsCertFile, "-key-file", c.tlsKeyFile, hostname}
return mclib.RunMain()
}
func (c *serverCommand) verifyCert(rootPEM, certPEM []byte, name string) error {
roots := x509.NewCertPool()
ok := roots.AppendCertsFromPEM(rootPEM)
if !ok {
return fmt.Errorf("failed to parse root certificate")
}
block, _ := pem.Decode(certPEM)
if block == nil {
return fmt.Errorf("failed to parse certificate PEM")
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return fmt.Errorf("failed to parse certificate: %v", err.Error())
}
opts := x509.VerifyOptions{
DNSName: name,
Roots: roots,
}
if _, err := cert.Verify(opts); err != nil {
return fmt.Errorf("failed to verify certificate: %v", err.Error())
}
return nil
}
func (c *serverCommand) createServerPorts(cd *simplecobra.Commandeer) error {
flags := cd.CobraCommand.Flags()
var cerr error
c.withConf(func(conf *commonConfig) {
isMultiHost := conf.configs.IsMultihost
c.serverPorts = make([]serverPortListener, 1)
if isMultiHost {
if !c.serverAppend {
cerr = errors.New("--appendPort=false not supported when in multihost mode")
return
}
c.serverPorts = make([]serverPortListener, len(conf.configs.Languages))
}
currentServerPort := c.serverPort
for i := 0; i < len(c.serverPorts); i++ {
l, err := net.Listen("tcp", net.JoinHostPort(c.serverInterface, strconv.Itoa(currentServerPort)))
if err == nil {
c.serverPorts[i] = serverPortListener{ln: l, p: currentServerPort}
} else {
if i == 0 && flags.Changed("port") {
// port set explicitly by user -- he/she probably meant it!
cerr = fmt.Errorf("server startup failed: %s", err)
return
}
c.r.Println("port", currentServerPort, "already in use, attempting to use an available port")
l, sp, err := helpers.TCPListen()
if err != nil {
cerr = fmt.Errorf("unable to find alternative port to use: %s", err)
return
}
c.serverPorts[i] = serverPortListener{ln: l, p: sp.Port}
}
currentServerPort = c.serverPorts[i].p + 1
}
})
return cerr
}
// fixURL massages the baseURL into a form needed for serving
// all pages correctly.
func (c *serverCommand) fixURL(baseURLFromConfig, baseURLFromFlag string, port int) (string, error) {
certsSet := (c.tlsCertFile != "" && c.tlsKeyFile != "") || c.tlsAuto
useLocalhost := false
baseURL := baseURLFromFlag
if baseURL == "" {
baseURL = baseURLFromConfig
useLocalhost = true
}
if !strings.HasSuffix(baseURL, "/") {
baseURL = baseURL + "/"
}
// do an initial parse of the input string
u, err := url.Parse(baseURL)
if err != nil {
return "", err
}
// if no Host is defined, then assume that no schema or double-slash were
// present in the url. Add a double-slash and make a best effort attempt.
if u.Host == "" && baseURL != "/" {
baseURL = "//" + baseURL
u, err = url.Parse(baseURL)
if err != nil {
return "", err
}
}
if useLocalhost {
if certsSet {
u.Scheme = "https"
} else if u.Scheme == "https" {
u.Scheme = "http"
}
u.Host = "localhost"
}
if c.serverAppend {
if strings.Contains(u.Host, ":") {
u.Host, _, err = net.SplitHostPort(u.Host)
if err != nil {
return "", fmt.Errorf("failed to split baseURL hostport: %w", err)
}
}
u.Host += fmt.Sprintf(":%d", port)
}
return u.String(), nil
}
func (c *serverCommand) partialReRender(urls ...string) error {
defer func() {
c.errState.setWasErr(false)
}()
c.errState.setBuildErr(nil)
visited := make(map[string]bool)
for _, url := range urls {
visited[url] = true
}
h, err := c.hugo()
if err != nil {
return err
}
// Note: We do not set NoBuildLock as the file lock is not acquired at this stage.
return h.Build(hugolib.BuildCfg{NoBuildLock: false, RecentlyVisited: visited, PartialReRender: true, ErrRecovery: c.errState.wasErr()})
}
func (c *serverCommand) serve() error {
var (
baseURLs []string
roots []string
h *hugolib.HugoSites
)
err := c.withConfE(func(conf *commonConfig) error {
isMultiHost := conf.configs.IsMultihost
var err error
h, err = c.r.HugFromConfig(conf)
if err != nil {
return err
}
// We need the server to share the same logger as the Hugo build (for error counts etc.)
c.r.logger = h.Log
if isMultiHost {
for _, l := range conf.configs.ConfigLangs() {
baseURLs = append(baseURLs, l.BaseURL().String())
roots = append(roots, l.Language().Lang)
}
} else {
l := conf.configs.GetFirstLanguageConfig()
baseURLs = []string{l.BaseURL().String()}
roots = []string{""}
}
return nil
})
if err != nil {
return err
}
2023-02-18 15:47:35 -05:00
// Cache it here. The HugoSites object may be unavailable later on due to intermittent configuration errors.
// To allow the en user to change the error template while the server is running, we use
// the freshest template we can provide.
var (
errTempl tpl.Template
templHandler tpl.TemplateHandler
)
getErrorTemplateAndHandler := func(h *hugolib.HugoSites) (tpl.Template, tpl.TemplateHandler) {
if h == nil {
return errTempl, templHandler
}
templHandler := h.Tmpl()
errTempl, found := templHandler.Lookup("_server/error.html")
if !found {
panic("template server/error.html not found")
}
return errTempl, templHandler
}
errTempl, templHandler = getErrorTemplateAndHandler(h)
srv := &fileServer{
baseURLs: baseURLs,
roots: roots,
c: c,
errorTemplate: func(ctx any) (io.Reader, error) {
// hugoTry does not block, getErrorTemplateAndHandler will fall back
// to cached values if nil.
templ, handler := getErrorTemplateAndHandler(c.hugoTry())
b := &bytes.Buffer{}
2023-02-25 03:24:59 -05:00
err := handler.ExecuteWithContext(context.Background(), templ, b, ctx)
return b, err
},
}
doLiveReload := !c.disableLiveReload
if doLiveReload {
livereload.Initialize()
}
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
var servers []*http.Server
wg1, ctx := errgroup.WithContext(context.Background())
2018-02-21 03:23:43 -05:00
for i := range baseURLs {
mu, listener, serverURL, endpoint, err := srv.createEndpoint(i)
2023-10-04 15:25:43 -04:00
var srv *http.Server
if c.tlsCertFile != "" && c.tlsKeyFile != "" {
srv = &http.Server{
Addr: endpoint,
Handler: mu,
TLSConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
},
}
} else {
srv = &http.Server{
Addr: endpoint,
Handler: mu,
}
}
servers = append(servers, srv)
if doLiveReload {
u, err := url.Parse(helpers.SanitizeURL(baseURLs[i]))
if err != nil {
return err
}
mu.HandleFunc(u.Path+"/livereload.js", livereload.ServeJS)
mu.HandleFunc(u.Path+"/livereload", livereload.Handler)
}
c.r.Printf("Web Server is available at %s (bind address %s) %s\n", serverURL, c.serverInterface, roots[i])
wg1.Go(func() error {
if c.tlsCertFile != "" && c.tlsKeyFile != "" {
err = srv.ServeTLS(listener, c.tlsCertFile, c.tlsKeyFile)
} else {
err = srv.Serve(listener)
}
if err != nil && err != http.ErrServerClosed {
return err
}
return nil
})
}
if c.r.IsTestRun() {
// Write a .ready file to disk to signal ready status.
// This is where the test is run from.
testInfo := map[string]any{
"baseURLs": srv.baseURLs,
}
dir := os.Getenv("WORK")
if dir != "" {
readyFile := filepath.Join(dir, ".ready")
// encode the test info as JSON into the .ready file.
b, err := json.Marshal(testInfo)
if err != nil {
return err
}
err = os.WriteFile(readyFile, b, 0777)
if err != nil {
return err
}
}
}
c.r.Println("Press Ctrl+C to stop")
err = func() error {
for {
select {
case <-c.quit:
return nil
case <-sigs:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
}()
if err != nil {
c.r.Println("Error:", err)
}
if h := c.hugoTry(); h != nil {
h.Close()
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
wg2, ctx := errgroup.WithContext(ctx)
for _, srv := range servers {
srv := srv
wg2.Go(func() error {
return srv.Shutdown(ctx)
})
}
err1, err2 := wg1.Wait(), wg2.Wait()
if err1 != nil {
return err1
}
return err2
}
type serverPortListener struct {
p int
ln net.Listener
}
type staticSyncer struct {
c *hugoBuilder
}
func (s *staticSyncer) isStatic(h *hugolib.HugoSites, filename string) bool {
return h.BaseFs.SourceFilesystems.IsStatic(filename)
}
func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
c := s.c
syncFn := func(sourceFs *filesystems.SourceFilesystem) (uint64, error) {
publishDir := helpers.FilePathSeparator
if sourceFs.PublishFolder != "" {
publishDir = filepath.Join(publishDir, sourceFs.PublishFolder)
}
syncer := fsync.NewSyncer()
c.withConf(func(conf *commonConfig) {
syncer.NoTimes = conf.configs.Base.NoTimes
syncer.NoChmod = conf.configs.Base.NoChmod
syncer.ChmodFilter = chmodFilter
syncer.SrcFs = sourceFs.Fs
syncer.DestFs = conf.fs.PublishDir
if c.s != nil && c.s.renderStaticToDisk {
syncer.DestFs = conf.fs.PublishDirStatic
}
})
logger := s.c.r.logger
for _, ev := range staticEvents {
// Due to our approach of layering both directories and the content's rendered output
// into one we can't accurately remove a file not in one of the source directories.
// If a file is in the local static dir and also in the theme static dir and we remove
// it from one of those locations we expect it to still exist in the destination
//
// If Hugo generates a file (from the content dir) over a static file
// the content generated file should take precedence.
//
// Because we are now watching and handling individual events it is possible that a static
// event that occupies the same path as a content generated file will take precedence
// until a regeneration of the content takes places.
//
// Hugo assumes that these cases are very rare and will permit this bad behavior
// The alternative is to track every single file and which pipeline rendered it
// and then to handle conflict resolution on every event.
fromPath := ev.Name
relPath, found := sourceFs.MakePathRelative(fromPath)
if !found {
// Not member of this virtual host.
continue
}
// Remove || rename is harder and will require an assumption.
// Hugo takes the following approach:
// If the static file exists in any of the static source directories after this event
// Hugo will re-sync it.
// If it does not exist in all of the static directories Hugo will remove it.
//
// This assumes that Hugo has not generated content on top of a static file and then removed
// the source of that static file. In this case Hugo will incorrectly remove that file
// from the published directory.
if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
if _, err := sourceFs.Fs.Stat(relPath); herrors.IsNotExist(err) {
// If file doesn't exist in any static dir, remove it
logger.Println("File no longer exists in static dir, removing", relPath)
c.withConf(func(conf *commonConfig) {
_ = conf.fs.PublishDirStatic.RemoveAll(relPath)
})
} else if err == nil {
// If file still exists, sync it
logger.Println("Syncing", relPath, "to", publishDir)
if err := syncer.Sync(relPath, relPath); err != nil {
c.r.logger.Errorln(err)
}
} else {
c.r.logger.Errorln(err)
}
continue
}
// For all other event operations Hugo will sync static.
logger.Println("Syncing", relPath, "to", publishDir)
if err := syncer.Sync(filepath.Join(publishDir, relPath), relPath); err != nil {
c.r.logger.Errorln(err)
}
}
return 0, nil
}
_, err := c.doWithPublishDirs(syncFn)
return err
}
2014-09-22 09:45:05 -04:00
func chmodFilter(dst, src os.FileInfo) bool {
// Hugo publishes data from multiple sources, potentially
// with overlapping directory structures. We cannot sync permissions
// for directories as that would mean that we might end up with write-protected
// directories inside /public.
// One example of this would be syncing from the Go Module cache,
// which have 0555 directories.
return src.IsDir()
}
2014-09-22 09:45:05 -04:00
func cleanErrorLog(content string) string {
content = strings.ReplaceAll(content, "\n", " ")
content = logReplacer.Replace(content)
content = logDuplicateTemplateExecuteRe.ReplaceAllString(content, "")
content = logDuplicateTemplateParseRe.ReplaceAllString(content, "")
seen := make(map[string]bool)
parts := strings.Split(content, ": ")
keep := make([]string, 0, len(parts))
for _, part := range parts {
if seen[part] {
continue
2014-09-22 09:45:05 -04:00
}
seen[part] = true
keep = append(keep, part)
}
return strings.Join(keep, ": ")
}
2014-09-22 09:45:05 -04:00
func injectLiveReloadScript(src io.Reader, baseURL url.URL) string {
var b bytes.Buffer
chain := transform.Chain{livereloadinject.New(baseURL)}
chain.Apply(&b, src)
2014-09-22 09:45:05 -04:00
return b.String()
}
2014-09-22 09:45:05 -04:00
func partitionDynamicEvents(sourceFs *filesystems.SourceFilesystems, events []fsnotify.Event) (de dynamicEvents) {
for _, e := range events {
if !sourceFs.IsContent(e.Name) {
de.AssetEvents = append(de.AssetEvents, e)
} else {
de.ContentEvents = append(de.ContentEvents, e)
}
}
return
}
2014-09-22 09:45:05 -04:00
func pickOneWriteOrCreatePath(events []fsnotify.Event) string {
name := ""
for _, ev := range events {
if ev.Op&fsnotify.Write == fsnotify.Write || ev.Op&fsnotify.Create == fsnotify.Create {
if files.IsIndexContentFile(ev.Name) {
return ev.Name
2014-09-22 09:45:05 -04:00
}
if files.IsContentFile(ev.Name) {
name = ev.Name
}
}
2014-09-22 09:45:05 -04:00
}
return name
}
func formatByteCount(b uint64) string {
const unit = 1000
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB",
float64(b)/float64(div), "kMGTPE"[exp])
2014-09-22 09:45:05 -04:00
}