hugo/commands/commandeer.go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"regexp"
	"sync"
	"time"

	hconfig "github.com/gohugoio/hugo/config"

	"golang.org/x/sync/semaphore"

	"github.com/gohugoio/hugo/common/herrors"
	"github.com/gohugoio/hugo/common/htime"
	"github.com/gohugoio/hugo/common/hugo"
	"github.com/gohugoio/hugo/common/paths"

	"github.com/spf13/cast"
	jww "github.com/spf13/jwalterweatherman"

	"github.com/gohugoio/hugo/common/loggers"
	"github.com/gohugoio/hugo/config"

	"github.com/spf13/cobra"

	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"

	"github.com/bep/clock"
	"github.com/bep/debounce"
	"github.com/bep/overlayfs"
	"github.com/gohugoio/hugo/common/types"
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugofs"
	"github.com/gohugoio/hugo/langs"
)

type commandeerHugoState struct {
	*deps.DepsCfg
	hugoSites *hugolib.HugoSites
	fsCreate  sync.Once
	created   chan struct{}
}

type commandeer struct {
	*commandeerHugoState

	logger       loggers.Logger
	serverConfig *config.Server

	buildLock func() (unlock func(), err error)

	// Loading state
	mustHaveConfigFile bool
	failOnInitErr      bool
	running            bool

	// Currently only set when in "fast render mode". But it seems to
	// be fast enough that we could maybe just add it for all server modes.
	changeDetector *fileChangeDetector

	// We need to reuse these on server rebuilds.
	// These 2 will be different if --renderStaticToDisk is set.
	publishDirFs       afero.Fs
	publishDirServerFs afero.Fs

	h    *hugoBuilderCommon
	ftch flagsToConfigHandler

	visitedURLs *types.EvictingStringQueue

	cfgInit func(c *commandeer) error

	// We watch these for changes.
	configFiles []string

	// Used in cases where we get flooded with events in server mode.
	debounce func(f func())

	serverPorts []serverPortListener

	languages          langs.Languages
	doLiveReload       bool
	renderStaticToDisk bool
	fastRenderMode     bool
	showErrorInBrowser bool
	wasError           bool

	configured bool
	paused     bool

	fullRebuildSem *semaphore.Weighted

	// Any error from the last build.
	buildErr error
}

type serverPortListener struct {
	p  int
	ln net.Listener
}

func newCommandeerHugoState() *commandeerHugoState {
	return &commandeerHugoState{
		created: make(chan struct{}),
	}
}

func (c *commandeerHugoState) hugo() *hugolib.HugoSites {
	<-c.created
	return c.hugoSites
}

func (c *commandeerHugoState) hugoTry() *hugolib.HugoSites {
	select {
	case <-c.created:
		return c.hugoSites
	case <-time.After(time.Millisecond * 100):
		return nil
	}
}

func (c *commandeer) errCount() int {
	return int(c.logger.LogCounters().ErrorCounter.Count())
}

func (c *commandeer) getErrorWithContext() any {
	errCount := c.errCount()

	if errCount == 0 {
		return nil
	}

	m := make(map[string]any)

	//xwm["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors())))
	m["Error"] = errors.New(cleanErrorLog(removeErrorPrefixFromLog(c.logger.Errors())))
	m["Version"] = hugo.BuildVersionString()
	ferrors := herrors.UnwrapFileErrorsWithErrorContext(c.buildErr)
	m["Files"] = ferrors

	return m
}

func (c *commandeer) Set(key string, value any) {
	if c.configured {
		panic("commandeer cannot be changed")
	}
	c.Cfg.Set(key, value)
}

func (c *commandeer) initFs(fs *hugofs.Fs) error {
	c.publishDirFs = fs.PublishDir
	c.publishDirServerFs = fs.PublishDirServer
	c.DepsCfg.Fs = fs

	return nil
}

func (c *commandeer) initClock(loc *time.Location) error {
	bt := c.Cfg.GetString("clock")
	if bt == "" {
		return nil
	}

	t, err := cast.StringToDateInDefaultLocation(bt, loc)
	if err != nil {
		return fmt.Errorf(`failed to parse "clock" flag: %s`, err)
	}

	htime.Clock = clock.Start(t)
	return nil
}

func newCommandeer(mustHaveConfigFile, failOnInitErr, running bool, h *hugoBuilderCommon, f flagsToConfigHandler, cfgInit func(c *commandeer) error, subCmdVs ...*cobra.Command) (*commandeer, error) {
	var rebuildDebouncer func(f func())
	if running {
		// The time value used is tested with mass content replacements in a fairly big Hugo site.
		// It is better to wait for some seconds in those cases rather than get flooded
		// with rebuilds.
		rebuildDebouncer = debounce.New(4 * time.Second)
	}

	out := ioutil.Discard
	if !h.quiet {
		out = os.Stdout
	}

	c := &commandeer{
		h:                   h,
		ftch:                f,
		commandeerHugoState: newCommandeerHugoState(),
		cfgInit:             cfgInit,
		visitedURLs:         types.NewEvictingStringQueue(10),
		debounce:            rebuildDebouncer,
		fullRebuildSem:      semaphore.NewWeighted(1),

		// Init state
		mustHaveConfigFile: mustHaveConfigFile,
		failOnInitErr:      failOnInitErr,
		running:            running,

		// This will be replaced later, but we need something to log to before the configuration is read.
		logger: loggers.NewLogger(jww.LevelWarn, jww.LevelError, out, ioutil.Discard, running),
	}

	return c, c.loadConfig()
}

type fileChangeDetector struct {
	sync.Mutex
	current map[string]string
	prev    map[string]string

	irrelevantRe *regexp.Regexp
}

func (f *fileChangeDetector) OnFileClose(name, md5sum string) {
	f.Lock()
	defer f.Unlock()
	f.current[name] = md5sum
}

func (f *fileChangeDetector) changed() []string {
	if f == nil {
		return nil
	}
	f.Lock()
	defer f.Unlock()
	var c []string
	for k, v := range f.current {
		vv, found := f.prev[k]
		if !found || v != vv {
			c = append(c, k)
		}
	}

	return f.filterIrrelevant(c)
}

func (f *fileChangeDetector) filterIrrelevant(in []string) []string {
	var filtered []string
	for _, v := range in {
		if !f.irrelevantRe.MatchString(v) {
			filtered = append(filtered, v)
		}
	}
	return filtered
}

func (f *fileChangeDetector) PrepareNew() {
	if f == nil {
		return
	}

	f.Lock()
	defer f.Unlock()

	if f.current == nil {
		f.current = make(map[string]string)
		f.prev = make(map[string]string)
		return
	}

	f.prev = make(map[string]string)
	for k, v := range f.current {
		f.prev[k] = v
	}
	f.current = make(map[string]string)
}

func (c *commandeer) loadConfig() error {
	if c.DepsCfg == nil {
		c.DepsCfg = &deps.DepsCfg{}
	}

	if c.logger != nil {
		// Truncate the error log if this is a reload.
		c.logger.Reset()
	}

	cfg := c.DepsCfg
	c.configured = false
	cfg.Running = c.running

	var dir string
	if c.h.source != "" {
		dir, _ = filepath.Abs(c.h.source)
	} else {
		dir, _ = os.Getwd()
	}

	var sourceFs afero.Fs = hugofs.Os
	if c.DepsCfg.Fs != nil {
		sourceFs = c.DepsCfg.Fs.Source
	}

	environment := c.h.getEnvironment(c.running)

	doWithConfig := func(cfg config.Provider) error {
		if c.ftch != nil {
			c.ftch.flagsToConfig(cfg)
		}

		cfg.Set("workingDir", dir)
		cfg.Set("environment", environment)
		return nil
	}

	cfgSetAndInit := func(cfg config.Provider) error {
		c.Cfg = cfg
		if c.cfgInit == nil {
			return nil
		}
		err := c.cfgInit(c)
		return err
	}

	configPath := c.h.source
	if configPath == "" {
		configPath = dir
	}
	config, configFiles, err := hugolib.LoadConfig(
		hugolib.ConfigSourceDescriptor{
			Fs:           sourceFs,
			Logger:       c.logger,
			Path:         configPath,
			WorkingDir:   dir,
			Filename:     c.h.cfgFile,
			AbsConfigDir: c.h.getConfigDir(dir),
			Environment:  environment,
		},
		cfgSetAndInit,
		doWithConfig)

	if err != nil {
		// We should improve the error handling here,
		// but with hugo mod init and similar there is a chicken and egg situation
		// with modules already configured in config.toml, so ignore those errors.
		if c.mustHaveConfigFile || (c.failOnInitErr && !moduleNotFoundRe.MatchString(err.Error())) {
			return err
		} else {
			// Just make it a warning.
			c.logger.Warnln(err)
		}
	} else if c.mustHaveConfigFile && len(configFiles) == 0 {
		return hugolib.ErrNoConfigFile
	}

	c.configFiles = configFiles

	var ok bool
	c.languages, ok = c.Cfg.Get("languagesSorted").(langs.Languages)
	if !ok {
		panic("languages not configured")
	}

	err = c.initClock(langs.GetLocation(c.languages[0]))
	if err != nil {
		return err
	}

	// Set some commonly used flags
	c.doLiveReload = c.running && !c.Cfg.GetBool("disableLiveReload")
	c.fastRenderMode = c.doLiveReload && !c.Cfg.GetBool("disableFastRender")
	c.showErrorInBrowser = c.doLiveReload && !c.Cfg.GetBool("disableBrowserError")

	// This is potentially double work, but we need to do this one more time now
	// that all the languages have been configured.
	if c.cfgInit != nil {
		if err := c.cfgInit(c); err != nil {
			return err
		}
	}

	logger, err := c.createLogger(config)
	if err != nil {
		return err
	}

	cfg.Logger = logger
	c.logger = logger
	c.serverConfig, err = hconfig.DecodeServer(cfg.Cfg)
	if err != nil {
		return err
	}

	createMemFs := config.GetBool("renderToMemory")
	c.renderStaticToDisk = config.GetBool("renderStaticToDisk")

	if createMemFs {
		// Rendering to memoryFS, publish to Root regardless of publishDir.
		config.Set("publishDir", "/")
		config.Set("publishDirStatic", "/")
	} else if c.renderStaticToDisk {
		// Hybrid, render dynamic content to Root.
		config.Set("publishDirStatic", config.Get("publishDir"))
		config.Set("publishDir", "/")

	}

	c.fsCreate.Do(func() {
		// Assume both source and destination are using same filesystem.
		fs := hugofs.NewFromSourceAndDestination(sourceFs, sourceFs, config)

		if c.publishDirFs != nil {
			// Need to reuse the destination on server rebuilds.
			fs.PublishDir = c.publishDirFs
			fs.PublishDirServer = c.publishDirServerFs
		} else {
			if c.renderStaticToDisk {
				publishDirStatic := config.GetString("publishDirStatic")
				workingDir := config.GetString("workingDir")
				absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)

				fs = hugofs.NewFromSourceAndDestination(sourceFs, afero.NewMemMapFs(), config)
				// Writes the dynamic output to memory,
				// while serve others directly from /public on disk.
				dynamicFs := fs.PublishDir
				staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)

				// Serve from both the static and dynamic fs,
				// the first will take priority.
				// THis is a read-only filesystem,
				// we do all the writes to
				// fs.Destination and fs.DestinationStatic.
				fs.PublishDirServer = overlayfs.New(
					overlayfs.Options{
						Fss: []afero.Fs{
							dynamicFs,
							staticFs,
						},
					},
				)
				fs.PublishDirStatic = staticFs
			} else if createMemFs {
				// Hugo writes the output to memory instead of the disk.
				fs = hugofs.NewFromSourceAndDestination(sourceFs, afero.NewMemMapFs(), config)
			}
		}

		if c.fastRenderMode {
			// For now, fast render mode only. It should, however, be fast enough
			// for the full variant, too.
			changeDetector := &fileChangeDetector{
				// We use this detector to decide to do a Hot reload of a single path or not.
				// We need to filter out source maps and possibly some other to be able
				// to make that decision.
				irrelevantRe: regexp.MustCompile(`\.map$`),
			}

			changeDetector.PrepareNew()
			fs.PublishDir = hugofs.NewHashingFs(fs.PublishDir, changeDetector)
			fs.PublishDirStatic = hugofs.NewHashingFs(fs.PublishDirStatic, changeDetector)
			c.changeDetector = changeDetector
		}

		if c.Cfg.GetBool("logPathWarnings") {
			// Note that we only care about the "dynamic creates" here,
			// so skip the static fs.
			fs.PublishDir = hugofs.NewCreateCountingFs(fs.PublishDir)
		}

		// To debug hard-to-find path issues.
		// fs.Destination = hugofs.NewStacktracerFs(fs.Destination, `fr/fr`)

		err = c.initFs(fs)
		if err != nil {
			close(c.created)
			return
		}

		var h *hugolib.HugoSites

		var createErr error
		h, createErr = hugolib.NewHugoSites(*c.DepsCfg)
		if h == nil || c.failOnInitErr {
			err = createErr
		}

		c.hugoSites = h
		// TODO(bep) improve.
		if c.buildLock == nil && h != nil {
			c.buildLock = h.LockBuild
		}
		close(c.created)
	})

	if err != nil {
		return err
	}

	cacheDir, err := helpers.GetCacheDir(sourceFs, config)
	if err != nil {
		return err
	}
	config.Set("cacheDir", cacheDir)

	return nil
}