hugo/modules/collect.go
Bjørn Erik Pedersen f38a2fbd2e Make hugo.toml the new config.toml
Both will of course work, but hugo.toml will win if both are set.

We should have done this a long time ago, of course, but the reason I'm picking this up now is that my VS Code setup by default picks up some
JSON config schema from some random other software which also names its config files config.toml.

Fixes #8979
2023-01-16 15:34:16 +01:00

734 lines
17 KiB
Go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package modules
import (
"bufio"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/bep/debounce"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/rogpeppe/go-internal/module"
"errors"
"github.com/gohugoio/hugo/config"
"github.com/spf13/afero"
)
var ErrNotExist = errors.New("module does not exist")
const vendorModulesFilename = "modules.txt"
// IsNotExist returns whether an error means that a module could not be found.
func IsNotExist(err error) bool {
return errors.Is(err, os.ErrNotExist)
}
// CreateProjectModule creates modules from the given config.
// This is used in tests only.
func CreateProjectModule(cfg config.Provider) (Module, error) {
workingDir := cfg.GetString("workingDir")
var modConfig Config
mod := createProjectModule(nil, workingDir, modConfig)
if err := ApplyProjectConfigDefaults(cfg, mod); err != nil {
return nil, err
}
return mod, nil
}
func (h *Client) Collect() (ModulesConfig, error) {
mc, coll := h.collect(true)
if coll.err != nil {
return mc, coll.err
}
if err := (&mc).setActiveMods(h.logger); err != nil {
return mc, err
}
if h.ccfg.HookBeforeFinalize != nil {
if err := h.ccfg.HookBeforeFinalize(&mc); err != nil {
return mc, err
}
}
if err := (&mc).finalize(h.logger); err != nil {
return mc, err
}
return mc, nil
}
func (h *Client) collect(tidy bool) (ModulesConfig, *collector) {
c := &collector{
Client: h,
}
c.collect()
if c.err != nil {
return ModulesConfig{}, c
}
// https://github.com/gohugoio/hugo/issues/6115
/*if !c.skipTidy && tidy {
if err := h.tidy(c.modules, true); err != nil {
c.err = err
return ModulesConfig{}, c
}
}*/
var workspaceFilename string
if h.ccfg.ModuleConfig.Workspace != WorkspaceDisabled {
workspaceFilename = h.ccfg.ModuleConfig.Workspace
}
return ModulesConfig{
AllModules: c.modules,
GoModulesFilename: c.GoModulesFilename,
GoWorkspaceFilename: workspaceFilename,
}, c
}
type ModulesConfig struct {
// All modules, including any disabled.
AllModules Modules
// All active modules.
ActiveModules Modules
// Set if this is a Go modules enabled project.
GoModulesFilename string
// Set if a Go workspace file is configured.
GoWorkspaceFilename string
}
func (m *ModulesConfig) setActiveMods(logger loggers.Logger) error {
var activeMods Modules
for _, mod := range m.AllModules {
if !mod.Config().HugoVersion.IsValid() {
logger.Warnf(`Module %q is not compatible with this Hugo version; run "hugo mod graph" for more information.`, mod.Path())
}
if !mod.Disabled() {
activeMods = append(activeMods, mod)
}
}
m.ActiveModules = activeMods
return nil
}
func (m *ModulesConfig) finalize(logger loggers.Logger) error {
for _, mod := range m.AllModules {
m := mod.(*moduleAdapter)
m.mounts = filterUnwantedMounts(m.mounts)
}
return nil
}
func filterUnwantedMounts(mounts []Mount) []Mount {
// Remove duplicates
seen := make(map[string]bool)
tmp := mounts[:0]
for _, m := range mounts {
if !seen[m.key()] {
tmp = append(tmp, m)
}
seen[m.key()] = true
}
return tmp
}
type collected struct {
// Pick the first and prevent circular loops.
seen map[string]bool
// Maps module path to a _vendor dir. These values are fetched from
// _vendor/modules.txt, and the first (top-most) will win.
vendored map[string]vendoredModule
// Set if a Go modules enabled project.
gomods goModules
// Ordered list of collected modules, including Go Modules and theme
// components stored below /themes.
modules Modules
}
// Collects and creates a module tree.
type collector struct {
*Client
// Store away any non-fatal error and return at the end.
err error
// Set to disable any Tidy operation in the end.
skipTidy bool
*collected
}
func (c *collector) initModules() error {
c.collected = &collected{
seen: make(map[string]bool),
vendored: make(map[string]vendoredModule),
gomods: goModules{},
}
// If both these are true, we don't even need Go installed to build.
if c.ccfg.IgnoreVendor == nil && c.isVendored(c.ccfg.WorkingDir) {
return nil
}
// We may fail later if we don't find the mods.
return c.loadModules()
}
func (c *collector) isSeen(path string) bool {
key := pathKey(path)
if c.seen[key] {
return true
}
c.seen[key] = true
return false
}
func (c *collector) getVendoredDir(path string) (vendoredModule, bool) {
v, found := c.vendored[path]
return v, found
}
func (c *collector) add(owner *moduleAdapter, moduleImport Import, disabled bool) (*moduleAdapter, error) {
var (
mod *goModule
moduleDir string
version string
vendored bool
)
modulePath := moduleImport.Path
var realOwner Module = owner
if !c.ccfg.shouldIgnoreVendor(modulePath) {
if err := c.collectModulesTXT(owner); err != nil {
return nil, err
}
// Try _vendor first.
var vm vendoredModule
vm, vendored = c.getVendoredDir(modulePath)
if vendored {
moduleDir = vm.Dir
realOwner = vm.Owner
version = vm.Version
if owner.projectMod {
// We want to keep the go.mod intact with the versions and all.
c.skipTidy = true
}
}
}
if moduleDir == "" {
var versionQuery string
mod = c.gomods.GetByPath(modulePath)
if mod != nil {
moduleDir = mod.Dir
versionQuery = mod.Version
}
if moduleDir == "" {
if c.GoModulesFilename != "" && isProbablyModule(modulePath) {
// Try to "go get" it and reload the module configuration.
if versionQuery == "" {
// See https://golang.org/ref/mod#version-queries
// This will select the latest release-version (not beta etc.).
versionQuery = "upgrade"
}
if err := c.Get(fmt.Sprintf("%s@%s", modulePath, versionQuery)); err != nil {
return nil, err
}
if err := c.loadModules(); err != nil {
return nil, err
}
mod = c.gomods.GetByPath(modulePath)
if mod != nil {
moduleDir = mod.Dir
}
}
// Fall back to project/themes/<mymodule>
if moduleDir == "" {
var err error
moduleDir, err = c.createThemeDirname(modulePath, owner.projectMod || moduleImport.pathProjectReplaced)
if err != nil {
c.err = err
return nil, nil
}
if found, _ := afero.Exists(c.fs, moduleDir); !found {
c.err = c.wrapModuleNotFound(fmt.Errorf(`module %q not found; either add it as a Hugo Module or store it in %q.`, modulePath, c.ccfg.ThemesDir))
return nil, nil
}
}
}
}
if found, _ := afero.Exists(c.fs, moduleDir); !found {
c.err = c.wrapModuleNotFound(fmt.Errorf("%q not found", moduleDir))
return nil, nil
}
if !strings.HasSuffix(moduleDir, fileSeparator) {
moduleDir += fileSeparator
}
ma := &moduleAdapter{
dir: moduleDir,
vendor: vendored,
disabled: disabled,
gomod: mod,
version: version,
// This may be the owner of the _vendor dir
owner: realOwner,
}
if mod == nil {
ma.path = modulePath
}
if !moduleImport.IgnoreConfig {
if err := c.applyThemeConfig(ma); err != nil {
return nil, err
}
}
if err := c.applyMounts(moduleImport, ma); err != nil {
return nil, err
}
c.modules = append(c.modules, ma)
return ma, nil
}
func (c *collector) addAndRecurse(owner *moduleAdapter, disabled bool) error {
moduleConfig := owner.Config()
if owner.projectMod {
if err := c.applyMounts(Import{}, owner); err != nil {
return err
}
}
for _, moduleImport := range moduleConfig.Imports {
disabled := disabled || moduleImport.Disable
if !c.isSeen(moduleImport.Path) {
tc, err := c.add(owner, moduleImport, disabled)
if err != nil {
return err
}
if tc == nil || moduleImport.IgnoreImports {
continue
}
if err := c.addAndRecurse(tc, disabled); err != nil {
return err
}
}
}
return nil
}
func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {
if moduleImport.NoMounts {
mod.mounts = nil
return nil
}
mounts := moduleImport.Mounts
modConfig := mod.Config()
if len(mounts) == 0 {
// Mounts not defined by the import.
mounts = modConfig.Mounts
}
if !mod.projectMod && len(mounts) == 0 {
// Create default mount points for every component folder that
// exists in the module.
for _, componentFolder := range files.ComponentFolders {
sourceDir := filepath.Join(mod.Dir(), componentFolder)
_, err := c.fs.Stat(sourceDir)
if err == nil {
mounts = append(mounts, Mount{
Source: componentFolder,
Target: componentFolder,
})
}
}
}
var err error
mounts, err = c.normalizeMounts(mod, mounts)
if err != nil {
return err
}
mounts, err = c.mountCommonJSConfig(mod, mounts)
if err != nil {
return err
}
mod.mounts = mounts
return nil
}
func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
var (
configFilename string
themeCfg map[string]any
hasConfigFile bool
err error
)
LOOP:
for _, configBaseName := range config.DefaultConfigNames {
for _, configFormats := range config.ValidConfigFileExtensions {
configFilename = filepath.Join(tc.Dir(), configBaseName+"."+configFormats)
hasConfigFile, _ = afero.Exists(c.fs, configFilename)
if hasConfigFile {
break LOOP
}
}
}
// The old theme information file.
themeTOML := filepath.Join(tc.Dir(), "theme.toml")
hasThemeTOML, _ := afero.Exists(c.fs, themeTOML)
if hasThemeTOML {
data, err := afero.ReadFile(c.fs, themeTOML)
if err != nil {
return err
}
themeCfg, err = metadecoders.Default.UnmarshalToMap(data, metadecoders.TOML)
if err != nil {
c.logger.Warnf("Failed to read module config for %q in %q: %s", tc.Path(), themeTOML, err)
} else {
maps.PrepareParams(themeCfg)
}
}
if hasConfigFile {
if configFilename != "" {
var err error
tc.cfg, err = config.FromFile(c.fs, configFilename)
if err != nil {
return err
}
}
tc.configFilenames = append(tc.configFilenames, configFilename)
}
// Also check for a config dir, which we overlay on top of the file configuration.
configDir := filepath.Join(tc.Dir(), "config")
dcfg, dirnames, err := config.LoadConfigFromDir(c.fs, configDir, c.ccfg.Environment)
if err != nil {
return err
}
if len(dirnames) > 0 {
tc.configFilenames = append(tc.configFilenames, dirnames...)
if hasConfigFile {
// Set will overwrite existing keys.
tc.cfg.Set("", dcfg.Get(""))
} else {
tc.cfg = dcfg
}
}
config, err := decodeConfig(tc.cfg, c.moduleConfig.replacementsMap)
if err != nil {
return err
}
const oldVersionKey = "min_version"
if hasThemeTOML {
// Merge old with new
if minVersion, found := themeCfg[oldVersionKey]; found {
if config.HugoVersion.Min == "" {
config.HugoVersion.Min = hugo.VersionString(cast.ToString(minVersion))
}
}
if config.Params == nil {
config.Params = make(map[string]any)
}
for k, v := range themeCfg {
if k == oldVersionKey {
continue
}
config.Params[k] = v
}
}
tc.config = config
return nil
}
func (c *collector) collect() {
defer c.logger.PrintTimerIfDelayed(time.Now(), "hugo: collected modules")
d := debounce.New(2 * time.Second)
d(func() {
c.logger.Println("hugo: downloading modules …")
})
defer d(func() {})
if err := c.initModules(); err != nil {
c.err = err
return
}
projectMod := createProjectModule(c.gomods.GetMain(), c.ccfg.WorkingDir, c.moduleConfig)
if err := c.addAndRecurse(projectMod, false); err != nil {
c.err = err
return
}
// Add the project mod on top.
c.modules = append(Modules{projectMod}, c.modules...)
}
func (c *collector) isVendored(dir string) bool {
_, err := c.fs.Stat(filepath.Join(dir, vendord, vendorModulesFilename))
return err == nil
}
func (c *collector) collectModulesTXT(owner Module) error {
vendorDir := filepath.Join(owner.Dir(), vendord)
filename := filepath.Join(vendorDir, vendorModulesFilename)
f, err := c.fs.Open(filename)
if err != nil {
if herrors.IsNotExist(err) {
return nil
}
return err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
// # github.com/alecthomas/chroma v0.6.3
line := scanner.Text()
line = strings.Trim(line, "# ")
line = strings.TrimSpace(line)
parts := strings.Fields(line)
if len(parts) != 2 {
return fmt.Errorf("invalid modules list: %q", filename)
}
path := parts[0]
shouldAdd := c.Client.moduleConfig.VendorClosest
if !shouldAdd {
if _, found := c.vendored[path]; !found {
shouldAdd = true
}
}
if shouldAdd {
c.vendored[path] = vendoredModule{
Owner: owner,
Dir: filepath.Join(vendorDir, path),
Version: parts[1],
}
}
}
return nil
}
func (c *collector) loadModules() error {
modules, err := c.listGoMods()
if err != nil {
return err
}
c.gomods = modules
return nil
}
// Matches postcss.config.js etc.
var commonJSConfigs = regexp.MustCompile(`(babel|postcss|tailwind)\.config\.js`)
func (c *collector) mountCommonJSConfig(owner *moduleAdapter, mounts []Mount) ([]Mount, error) {
for _, m := range mounts {
if strings.HasPrefix(m.Target, files.JsConfigFolderMountPrefix) {
// This follows the convention of the other component types (assets, content, etc.),
// if one or more is specified by the user, we skip the defaults.
// These mounts were added to Hugo in 0.75.
return mounts, nil
}
}
// Mount the common JS config files.
fis, err := afero.ReadDir(c.fs, owner.Dir())
if err != nil {
return mounts, err
}
for _, fi := range fis {
n := fi.Name()
should := n == files.FilenamePackageHugoJSON || n == files.FilenamePackageJSON
should = should || commonJSConfigs.MatchString(n)
if should {
mounts = append(mounts, Mount{
Source: n,
Target: filepath.Join(files.ComponentFolderAssets, files.FolderJSConfig, n),
})
}
}
return mounts, nil
}
func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mount, error) {
var out []Mount
dir := owner.Dir()
for _, mnt := range mounts {
errMsg := fmt.Sprintf("invalid module config for %q", owner.Path())
if mnt.Source == "" || mnt.Target == "" {
return nil, errors.New(errMsg + ": both source and target must be set")
}
mnt.Source = filepath.Clean(mnt.Source)
mnt.Target = filepath.Clean(mnt.Target)
var sourceDir string
if owner.projectMod && filepath.IsAbs(mnt.Source) {
// Abs paths in the main project is allowed.
sourceDir = mnt.Source
} else {
sourceDir = filepath.Join(dir, mnt.Source)
}
// Verify that Source exists
_, err := c.fs.Stat(sourceDir)
if err != nil {
continue
}
// Verify that target points to one of the predefined component dirs
targetBase := mnt.Target
idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator))
if idxPathSep != -1 {
targetBase = mnt.Target[0:idxPathSep]
}
if !files.IsComponentFolder(targetBase) {
return nil, fmt.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders)
}
out = append(out, mnt)
}
return out, nil
}
func (c *collector) wrapModuleNotFound(err error) error {
err = fmt.Errorf(err.Error()+": %w", ErrNotExist)
if c.GoModulesFilename == "" {
return err
}
baseMsg := "we found a go.mod file in your project, but"
switch c.goBinaryStatus {
case goBinaryStatusNotFound:
return fmt.Errorf(baseMsg+" you need to install Go to use it. See https://golang.org/dl/ : %q", err)
case goBinaryStatusTooOld:
return fmt.Errorf(baseMsg+" you need to a newer version of Go to use it. See https://golang.org/dl/ : %w", err)
}
return err
}
type vendoredModule struct {
Owner Module
Dir string
Version string
}
func createProjectModule(gomod *goModule, workingDir string, conf Config) *moduleAdapter {
// Create a pseudo module for the main project.
var path string
if gomod == nil {
path = "project"
}
return &moduleAdapter{
path: path,
dir: workingDir,
gomod: gomod,
projectMod: true,
config: conf,
}
}
// In the first iteration of Hugo Modules, we do not support multiple
// major versions running at the same time, so we pick the first (upper most).
// We will investigate namespaces in future versions.
// TODO(bep) add a warning when the above happens.
func pathKey(p string) string {
prefix, _, _ := module.SplitPathVersion(p)
return strings.ToLower(prefix)
}