hugo/modules/collect.go

749 lines
18 KiB
Go

// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package modules
import (
"bufio"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/bep/debounce"
"github.com/gohugoio/hugo/common/herrors"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/paths"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/hugo"
"github.com/gohugoio/hugo/parser/metadecoders"
"github.com/gohugoio/hugo/hugofs/files"
"golang.org/x/mod/module"
"github.com/gohugoio/hugo/config"
"github.com/spf13/afero"
)
var ErrNotExist = errors.New("module does not exist")
const vendorModulesFilename = "modules.txt"
func (h *Client) Collect() (ModulesConfig, error) {
mc, coll := h.collect(true)
if coll.err != nil {
return mc, coll.err
}
if err := (&mc).setActiveMods(h.logger); err != nil {
return mc, err
}
if h.ccfg.HookBeforeFinalize != nil {
if err := h.ccfg.HookBeforeFinalize(&mc); err != nil {
return mc, err
}
}
if err := (&mc).finalize(h.logger); err != nil {
return mc, err
}
return mc, nil
}
func (h *Client) collect(tidy bool) (ModulesConfig, *collector) {
if h == nil {
panic("nil client")
}
c := &collector{
Client: h,
}
c.collect()
if c.err != nil {
return ModulesConfig{}, c
}
// https://github.com/gohugoio/hugo/issues/6115
/*if !c.skipTidy && tidy {
if err := h.tidy(c.modules, true); err != nil {
c.err = err
return ModulesConfig{}, c
}
}*/
var workspaceFilename string
if h.ccfg.ModuleConfig.Workspace != WorkspaceDisabled {
workspaceFilename = h.ccfg.ModuleConfig.Workspace
}
return ModulesConfig{
AllModules: c.modules,
GoModulesFilename: c.GoModulesFilename,
GoWorkspaceFilename: workspaceFilename,
}, c
}
type ModulesConfig struct {
// All active modules.
AllModules Modules
// Set if this is a Go modules enabled project.
GoModulesFilename string
// Set if a Go workspace file is configured.
GoWorkspaceFilename string
}
func (m ModulesConfig) HasConfigFile() bool {
for _, mod := range m.AllModules {
if len(mod.ConfigFilenames()) > 0 {
return true
}
}
return false
}
func (m *ModulesConfig) setActiveMods(logger loggers.Logger) error {
for _, mod := range m.AllModules {
if !mod.Config().HugoVersion.IsValid() {
logger.Warnf(`Module %q is not compatible with this Hugo version: %s; run "hugo mod graph" for more information.`, mod.Path(), mod.Config().HugoVersion)
}
}
return nil
}
func (m *ModulesConfig) finalize(logger loggers.Logger) error {
for _, mod := range m.AllModules {
m := mod.(*moduleAdapter)
m.mounts = filterUnwantedMounts(m.mounts)
}
return nil
}
func filterUnwantedMounts(mounts []Mount) []Mount {
// Remove duplicates
seen := make(map[string]bool)
tmp := mounts[:0]
for _, m := range mounts {
if !seen[m.key()] {
tmp = append(tmp, m)
}
seen[m.key()] = true
}
return tmp
}
type collected struct {
// Pick the first and prevent circular loops.
seen map[string]bool
// Maps module path to a _vendor dir. These values are fetched from
// _vendor/modules.txt, and the first (top-most) will win.
vendored map[string]vendoredModule
// Set if a Go modules enabled project.
gomods goModules
// Ordered list of collected modules, including Go Modules and theme
// components stored below /themes.
modules Modules
}
// Collects and creates a module tree.
type collector struct {
*Client
// Store away any non-fatal error and return at the end.
err error
// Set to disable any Tidy operation in the end.
skipTidy bool
*collected
}
func (c *collector) initModules() error {
c.collected = &collected{
seen: make(map[string]bool),
vendored: make(map[string]vendoredModule),
gomods: goModules{},
}
// If both these are true, we don't even need Go installed to build.
if c.ccfg.IgnoreVendor == nil && c.isVendored(c.ccfg.WorkingDir) {
return nil
}
// We may fail later if we don't find the mods.
return c.loadModules()
}
func (c *collector) isSeen(path string) bool {
key := pathKey(path)
if c.seen[key] {
return true
}
c.seen[key] = true
return false
}
func (c *collector) getVendoredDir(path string) (vendoredModule, bool) {
v, found := c.vendored[path]
return v, found
}
func (c *collector) add(owner *moduleAdapter, moduleImport Import) (*moduleAdapter, error) {
var (
mod *goModule
moduleDir string
version string
vendored bool
)
modulePath := moduleImport.Path
var realOwner Module = owner
if !c.ccfg.shouldIgnoreVendor(modulePath) {
if err := c.collectModulesTXT(owner); err != nil {
return nil, err
}
// Try _vendor first.
var vm vendoredModule
vm, vendored = c.getVendoredDir(modulePath)
if vendored {
moduleDir = vm.Dir
realOwner = vm.Owner
version = vm.Version
if owner.projectMod {
// We want to keep the go.mod intact with the versions and all.
c.skipTidy = true
}
}
}
if moduleDir == "" {
var versionQuery string
mod = c.gomods.GetByPath(modulePath)
if mod != nil {
moduleDir = mod.Dir
versionQuery = mod.Version
}
if moduleDir == "" {
if c.GoModulesFilename != "" && isProbablyModule(modulePath) {
// Try to "go get" it and reload the module configuration.
if versionQuery == "" {
// See https://golang.org/ref/mod#version-queries
// This will select the latest release-version (not beta etc.).
versionQuery = "upgrade"
}
// Note that we cannot use c.Get for this, as that may
// trigger a new module collection and potentially create a infinite loop.
if err := c.get(fmt.Sprintf("%s@%s", modulePath, versionQuery)); err != nil {
return nil, err
}
if err := c.loadModules(); err != nil {
return nil, err
}
mod = c.gomods.GetByPath(modulePath)
if mod != nil {
moduleDir = mod.Dir
}
}
// Fall back to project/themes/<mymodule>
if moduleDir == "" {
var err error
moduleDir, err = c.createThemeDirname(modulePath, owner.projectMod || moduleImport.pathProjectReplaced)
if err != nil {
c.err = err
return nil, nil
}
if found, _ := afero.Exists(c.fs, moduleDir); !found {
//lint:ignore ST1005 end user message.
c.err = c.wrapModuleNotFound(fmt.Errorf(`module %q not found in %q; either add it as a Hugo Module or store it in %q.`, modulePath, moduleDir, c.ccfg.ThemesDir))
return nil, nil
}
}
}
}
if found, _ := afero.Exists(c.fs, moduleDir); !found {
c.err = c.wrapModuleNotFound(fmt.Errorf("%q not found", moduleDir))
return nil, nil
}
if !strings.HasSuffix(moduleDir, fileSeparator) {
moduleDir += fileSeparator
}
ma := &moduleAdapter{
dir: moduleDir,
vendor: vendored,
gomod: mod,
version: version,
// This may be the owner of the _vendor dir
owner: realOwner,
}
if mod == nil {
ma.path = modulePath
}
if !moduleImport.IgnoreConfig {
if err := c.applyThemeConfig(ma); err != nil {
return nil, err
}
}
if err := c.applyMounts(moduleImport, ma); err != nil {
return nil, err
}
c.modules = append(c.modules, ma)
return ma, nil
}
func (c *collector) addAndRecurse(owner *moduleAdapter) error {
moduleConfig := owner.Config()
if owner.projectMod {
if err := c.applyMounts(Import{}, owner); err != nil {
return fmt.Errorf("failed to apply mounts for project: %w", err)
}
}
for _, moduleImport := range moduleConfig.Imports {
if moduleImport.Disable {
continue
}
if !c.isSeen(moduleImport.Path) {
tc, err := c.add(owner, moduleImport)
if err != nil {
return err
}
if tc == nil || moduleImport.IgnoreImports {
continue
}
if err := c.addAndRecurse(tc); err != nil {
return err
}
}
}
return nil
}
func (c *collector) applyMounts(moduleImport Import, mod *moduleAdapter) error {
if moduleImport.NoMounts {
mod.mounts = nil
return nil
}
mounts := moduleImport.Mounts
modConfig := mod.Config()
if len(mounts) == 0 {
// Mounts not defined by the import.
mounts = modConfig.Mounts
}
if !mod.projectMod && len(mounts) == 0 {
// Create default mount points for every component folder that
// exists in the module.
for _, componentFolder := range files.ComponentFolders {
sourceDir := filepath.Join(mod.Dir(), componentFolder)
_, err := c.fs.Stat(sourceDir)
if err == nil {
mounts = append(mounts, Mount{
Source: componentFolder,
Target: componentFolder,
})
}
}
}
var err error
mounts, err = c.normalizeMounts(mod, mounts)
if err != nil {
return err
}
mounts, err = c.mountCommonJSConfig(mod, mounts)
if err != nil {
return err
}
mod.mounts = mounts
return nil
}
func (c *collector) applyThemeConfig(tc *moduleAdapter) error {
var (
configFilename string
themeCfg map[string]any
hasConfigFile bool
err error
)
LOOP:
for _, configBaseName := range config.DefaultConfigNames {
for _, configFormats := range config.ValidConfigFileExtensions {
configFilename = filepath.Join(tc.Dir(), configBaseName+"."+configFormats)
hasConfigFile, _ = afero.Exists(c.fs, configFilename)
if hasConfigFile {
break LOOP
}
}
}
// The old theme information file.
themeTOML := filepath.Join(tc.Dir(), "theme.toml")
hasThemeTOML, _ := afero.Exists(c.fs, themeTOML)
if hasThemeTOML {
data, err := afero.ReadFile(c.fs, themeTOML)
if err != nil {
return err
}
themeCfg, err = metadecoders.Default.UnmarshalToMap(data, metadecoders.TOML)
if err != nil {
c.logger.Warnf("Failed to read module config for %q in %q: %s", tc.Path(), themeTOML, err)
} else {
maps.PrepareParams(themeCfg)
}
}
if hasConfigFile {
if configFilename != "" {
var err error
tc.cfg, err = config.FromFile(c.fs, configFilename)
if err != nil {
return err
}
}
tc.configFilenames = append(tc.configFilenames, configFilename)
}
// Also check for a config dir, which we overlay on top of the file configuration.
configDir := filepath.Join(tc.Dir(), "config")
dcfg, dirnames, err := config.LoadConfigFromDir(c.fs, configDir, c.ccfg.Environment)
if err != nil {
return err
}
if len(dirnames) > 0 {
tc.configFilenames = append(tc.configFilenames, dirnames...)
if hasConfigFile {
// Set will overwrite existing keys.
tc.cfg.Set("", dcfg.Get(""))
} else {
tc.cfg = dcfg
}
}
config, err := decodeConfig(tc.cfg, c.moduleConfig.replacementsMap)
if err != nil {
return err
}
const oldVersionKey = "min_version"
if hasThemeTOML {
// Merge old with new
if minVersion, found := themeCfg[oldVersionKey]; found {
if config.HugoVersion.Min == "" {
config.HugoVersion.Min = hugo.VersionString(cast.ToString(minVersion))
}
}
if config.Params == nil {
config.Params = make(map[string]any)
}
for k, v := range themeCfg {
if k == oldVersionKey {
continue
}
config.Params[k] = v
}
}
tc.config = config
return nil
}
func (c *collector) collect() {
defer c.logger.PrintTimerIfDelayed(time.Now(), "hugo: collected modules")
d := debounce.New(2 * time.Second)
d(func() {
c.logger.Println("hugo: downloading modules …")
})
defer d(func() {})
if err := c.initModules(); err != nil {
c.err = err
return
}
projectMod := createProjectModule(c.gomods.GetMain(), c.ccfg.WorkingDir, c.moduleConfig)
if err := c.addAndRecurse(projectMod); err != nil {
c.err = err
return
}
// Add the project mod on top.
c.modules = append(Modules{projectMod}, c.modules...)
}
func (c *collector) isVendored(dir string) bool {
_, err := c.fs.Stat(filepath.Join(dir, vendord, vendorModulesFilename))
return err == nil
}
func (c *collector) collectModulesTXT(owner Module) error {
vendorDir := filepath.Join(owner.Dir(), vendord)
filename := filepath.Join(vendorDir, vendorModulesFilename)
f, err := c.fs.Open(filename)
if err != nil {
if herrors.IsNotExist(err) {
return nil
}
return err
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
// # github.com/alecthomas/chroma v0.6.3
line := scanner.Text()
line = strings.Trim(line, "# ")
line = strings.TrimSpace(line)
parts := strings.Fields(line)
if len(parts) != 2 {
return fmt.Errorf("invalid modules list: %q", filename)
}
path := parts[0]
shouldAdd := c.Client.moduleConfig.VendorClosest
if !shouldAdd {
if _, found := c.vendored[path]; !found {
shouldAdd = true
}
}
if shouldAdd {
c.vendored[path] = vendoredModule{
Owner: owner,
Dir: filepath.Join(vendorDir, path),
Version: parts[1],
}
}
}
return nil
}
func (c *collector) loadModules() error {
modules, err := c.listGoMods()
if err != nil {
return err
}
c.gomods = modules
return nil
}
// Matches postcss.config.js etc.
var commonJSConfigs = regexp.MustCompile(`(babel|postcss|tailwind)\.config\.js`)
func (c *collector) mountCommonJSConfig(owner *moduleAdapter, mounts []Mount) ([]Mount, error) {
for _, m := range mounts {
if strings.HasPrefix(m.Target, files.JsConfigFolderMountPrefix) {
// This follows the convention of the other component types (assets, content, etc.),
// if one or more is specified by the user, we skip the defaults.
// These mounts were added to Hugo in 0.75.
return mounts, nil
}
}
// Mount the common JS config files.
d, err := c.fs.Open(owner.Dir())
if err != nil {
return mounts, fmt.Errorf("failed to open dir %q: %q", owner.Dir(), err)
}
defer d.Close()
fis, err := d.(fs.ReadDirFile).ReadDir(-1)
if err != nil {
return mounts, fmt.Errorf("failed to read dir %q: %q", owner.Dir(), err)
}
for _, fi := range fis {
n := fi.Name()
should := n == files.FilenamePackageHugoJSON || n == files.FilenamePackageJSON
should = should || commonJSConfigs.MatchString(n)
if should {
mounts = append(mounts, Mount{
Source: n,
Target: filepath.Join(files.ComponentFolderAssets, files.FolderJSConfig, n),
})
}
}
return mounts, nil
}
func (c *collector) normalizeMounts(owner *moduleAdapter, mounts []Mount) ([]Mount, error) {
var out []Mount
dir := owner.Dir()
for _, mnt := range mounts {
errMsg := fmt.Sprintf("invalid module config for %q", owner.Path())
if mnt.Source == "" || mnt.Target == "" {
return nil, errors.New(errMsg + ": both source and target must be set")
}
mnt.Source = filepath.Clean(mnt.Source)
mnt.Target = filepath.Clean(mnt.Target)
var sourceDir string
if owner.projectMod && filepath.IsAbs(mnt.Source) {
// Abs paths in the main project is allowed.
sourceDir = mnt.Source
} else {
sourceDir = filepath.Join(dir, mnt.Source)
}
// Verify that Source exists
_, err := c.fs.Stat(sourceDir)
if err != nil {
if paths.IsSameFilePath(sourceDir, c.ccfg.PublishDir) {
// This is a little exotic, but there are use cases for mounting the public folder.
// This will typically also be in .gitingore, so create it.
if err := c.fs.MkdirAll(sourceDir, 0o755); err != nil {
return nil, fmt.Errorf("%s: %q", errMsg, err)
}
} else if strings.HasSuffix(sourceDir, files.FilenameHugoStatsJSON) {
// A common pattern for Tailwind 3 is to mount that file to get it on the server watch list.
// A common pattern is also to add hugo_stats.json to .gitignore.
// Create an empty file.
f, err := c.fs.Create(sourceDir)
if err != nil {
return nil, fmt.Errorf("%s: %q", errMsg, err)
}
f.Close()
} else {
// TODO(bep) commenting out for now, as this will create to much noise.
// c.logger.Warnf("module %q: mount source %q does not exist", owner.Path(), sourceDir)
continue
}
}
// Verify that target points to one of the predefined component dirs
targetBase := mnt.Target
idxPathSep := strings.Index(mnt.Target, string(os.PathSeparator))
if idxPathSep != -1 {
targetBase = mnt.Target[0:idxPathSep]
}
if !files.IsComponentFolder(targetBase) {
return nil, fmt.Errorf("%s: mount target must be one of: %v", errMsg, files.ComponentFolders)
}
out = append(out, mnt)
}
return out, nil
}
func (c *collector) wrapModuleNotFound(err error) error {
err = fmt.Errorf(err.Error()+": %w", ErrNotExist)
if c.GoModulesFilename == "" {
return err
}
baseMsg := "we found a go.mod file in your project, but"
switch c.goBinaryStatus {
case goBinaryStatusNotFound:
return fmt.Errorf(baseMsg+" you need to install Go to use it. See https://golang.org/dl/ : %q", err)
case goBinaryStatusTooOld:
return fmt.Errorf(baseMsg+" you need to a newer version of Go to use it. See https://golang.org/dl/ : %w", err)
}
return err
}
type vendoredModule struct {
Owner Module
Dir string
Version string
}
func createProjectModule(gomod *goModule, workingDir string, conf Config) *moduleAdapter {
// Create a pseudo module for the main project.
var path string
if gomod == nil {
path = "project"
}
return &moduleAdapter{
path: path,
dir: workingDir,
gomod: gomod,
projectMod: true,
config: conf,
}
}
// In the first iteration of Hugo Modules, we do not support multiple
// major versions running at the same time, so we pick the first (upper most).
// We will investigate namespaces in future versions.
// TODO(bep) add a warning when the above happens.
func pathKey(p string) string {
prefix, _, _ := module.SplitPathVersion(p)
return strings.ToLower(prefix)
}