diff --git a/commands/hugo.go b/commands/hugo.go index 4fd20a0f4..c19756008 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -870,7 +870,7 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat livereload.ForceRefresh() } case err := <-watcher.Errors(): - if err != nil { + if err != nil && !os.IsNotExist(err) { c.logger.Errorln("Error while watching:", err) } } diff --git a/commands/new.go b/commands/new.go index 7affd3547..c5b5cd182 100644 --- a/commands/new.go +++ b/commands/new.go @@ -80,17 +80,7 @@ func (n *newCmd) newContent(cmd *cobra.Command, args []string) error { return newUserError("path needs to be provided") } - createPath := args[0] - - var kind string - - createPath, kind = newContentPathSection(c.hugo(), createPath) - - if n.contentType != "" { - kind = n.contentType - } - - return create.NewContent(c.hugo(), kind, createPath) + return create.NewContent(c.hugo(), n.contentType, args[0]) } func mkdir(x ...string) { diff --git a/commands/new_site.go b/commands/new_site.go index 71097b8ff..11e9ce40a 100644 --- a/commands/new_site.go +++ b/commands/new_site.go @@ -102,7 +102,7 @@ func (n *newSiteCmd) doNewSite(fs *hugofs.Fs, basepath string, force bool) error // Create a default archetype file. helpers.SafeWriteToDisk(filepath.Join(archeTypePath, "default.md"), - strings.NewReader(create.ArchetypeTemplateTemplate), fs.Source) + strings.NewReader(create.DefaultArchetypeTemplateTemplate), fs.Source) jww.FEEDBACK.Printf("Congratulations! Your new Hugo site is created in %s.\n\n", basepath) jww.FEEDBACK.Println(nextStepsText()) diff --git a/create/content.go b/create/content.go index ea065423e..714939f4c 100644 --- a/create/content.go +++ b/create/content.go @@ -16,11 +16,14 @@ package create import ( "bytes" + "fmt" "io" "os" "path/filepath" "strings" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/common/paths" "github.com/pkg/errors" @@ -33,125 +36,136 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugolib" "github.com/spf13/afero" - jww "github.com/spf13/jwalterweatherman" ) -// NewContent creates a new content file in the content directory based upon the -// given kind, which is used to lookup an archetype. -func NewContent( - sites *hugolib.HugoSites, kind, targetPath string) error { - targetPath = filepath.Clean(targetPath) +const ( + // DefaultArchetypeTemplateTemplate is the template used in 'hugo new site' + // and the template we use as a fall back. + DefaultArchetypeTemplateTemplate = `--- +title: "{{ replace .Name "-" " " | title }}" +date: {{ .Date }} +draft: true +--- + +` +) + +// NewContent creates a new content file in h (or a full bundle if the archetype is a directory) +// in targetPath. +func NewContent(h *hugolib.HugoSites, kind, targetPath string) error { + cf := hugolib.NewContentFactory(h) + + if kind == "" { + kind = cf.SectionFromFilename(targetPath) + } + + b := &contentBuilder{ + archeTypeFs: h.PathSpec.BaseFs.Archetypes.Fs, + sourceFs: h.PathSpec.Fs.Source, + ps: h.PathSpec, + h: h, + cf: cf, + + kind: kind, + targetPath: targetPath, + } + ext := paths.Ext(targetPath) - ps := sites.PathSpec - archetypeFs := ps.BaseFs.SourceFilesystems.Archetypes.Fs - sourceFs := ps.Fs.Source - jww.INFO.Printf("attempting to create %q of %q of ext %q", targetPath, kind, ext) + b.setArcheTypeFilenameToUse(ext) - archetypeFilename, isDir := findArchetype(ps, kind, ext) - contentPath, s := resolveContentPath(sites, sourceFs, targetPath) + if b.isDir { + return b.buildDir() + } - if isDir { + if ext == "" { + return errors.Errorf("failed to resolve %q to a archetype template", targetPath) + } - langFs, err := hugofs.NewLanguageFs(sites.LanguageSet(), archetypeFs) + return b.buildFile() + +} + +type contentBuilder struct { + archeTypeFs afero.Fs + sourceFs afero.Fs + + ps *helpers.PathSpec + h *hugolib.HugoSites + cf hugolib.ContentFactory + + // Builder state + archetypeFilename string + targetPath string + kind string + isDir bool + dirMap archetypeMap +} + +func (b *contentBuilder) buildDir() error { + // Split the dir into content files and the rest. + if err := b.mapArcheTypeDir(); err != nil { + return err + } + + var contentTargetFilenames []string + var baseDir string + + for _, fi := range b.dirMap.contentFiles { + targetFilename := filepath.Join(b.targetPath, strings.TrimPrefix(fi.Meta().Path, b.archetypeFilename)) + abs, err := b.cf.CreateContentPlaceHolder(targetFilename) if err != nil { return err } - - cm, err := mapArcheTypeDir(ps, langFs, archetypeFilename) - if err != nil { - return err + if baseDir == "" { + baseDir = strings.TrimSuffix(abs, targetFilename) } - if cm.siteUsed { - if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { - return err + contentTargetFilenames = append(contentTargetFilenames, abs) + } + + var contentInclusionFilter *glob.FilenameFilter + if !b.dirMap.siteUsed { + // We don't need to build everything. + contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + for _, cn := range contentTargetFilenames { + if strings.HasPrefix(cn, filename) { + return true + } } - } + return false + }) - name := filepath.Base(targetPath) - return newContentFromDir(archetypeFilename, sites, sourceFs, cm, name, contentPath) } - // Building the sites can be expensive, so only do it if really needed. - siteUsed := false - - if archetypeFilename != "" { - - var err error - siteUsed, err = usesSiteVar(archetypeFs, archetypeFilename) - if err != nil { - return err - } - } - - if siteUsed { - if err := sites.Build(hugolib.BuildCfg{SkipRender: true}); err != nil { - return err - } - } - - content, err := executeArcheTypeAsTemplate(s, "", kind, targetPath, archetypeFilename) - if err != nil { + if err := b.h.Build(hugolib.BuildCfg{SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil { return err } - if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil { - return err - } - - jww.FEEDBACK.Println(contentPath, "created") - - editor := s.Cfg.GetString("newContentEditor") - if editor != "" { - jww.FEEDBACK.Printf("Editing %s with %q ...\n", targetPath, editor) - - editorCmd := append(strings.Fields(editor), contentPath) - cmd, err := hexec.SafeCommand(editorCmd[0], editorCmd[1:]...) - if err != nil { + for i, filename := range contentTargetFilenames { + if err := b.applyArcheType(filename, b.dirMap.contentFiles[i].Meta().Path); err != nil { return err } - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - return cmd.Run() } - return nil -} - -func targetSite(sites *hugolib.HugoSites, fi hugofs.FileMetaInfo) *hugolib.Site { - for _, s := range sites.Sites { - if fi.Meta().Lang == s.Language().Lang { - return s - } - } - return sites.Sites[0] -} - -func newContentFromDir( - archetypeDir string, - sites *hugolib.HugoSites, - targetFs afero.Fs, - cm archetypeMap, name, targetPath string) error { - for _, f := range cm.otherFiles { + // Copy the rest as is. + for _, f := range b.dirMap.otherFiles { meta := f.Meta() filename := meta.Path - // Just copy the file to destination. + in, err := meta.Open() if err != nil { return errors.Wrap(err, "failed to open non-content file") } - targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir)) - + targetFilename := filepath.Join(baseDir, b.targetPath, strings.TrimPrefix(filename, b.archetypeFilename)) targetDir := filepath.Dir(targetFilename) - if err := targetFs.MkdirAll(targetDir, 0777); err != nil && !os.IsExist(err) { - return errors.Wrapf(err, "failed to create target directory for %s:", targetDir) + + if err := b.sourceFs.MkdirAll(targetDir, 0o777); err != nil && !os.IsExist(err) { + return errors.Wrapf(err, "failed to create target directory for %q", targetDir) } - out, err := targetFs.Create(targetFilename) + out, err := b.sourceFs.Create(targetFilename) if err != nil { return err } @@ -164,41 +178,81 @@ func newContentFromDir( in.Close() out.Close() } - - for _, f := range cm.contentFiles { - filename := f.Meta().Path - s := targetSite(sites, f) - targetFilename := filepath.Join(targetPath, strings.TrimPrefix(filename, archetypeDir)) - - content, err := executeArcheTypeAsTemplate(s, name, archetypeDir, targetFilename, filename) - if err != nil { - return errors.Wrap(err, "failed to execute archetype template") - } - - if err := helpers.SafeWriteToDisk(targetFilename, bytes.NewReader(content), targetFs); err != nil { - return errors.Wrap(err, "failed to save results") - } - } - - jww.FEEDBACK.Println(targetPath, "created") - return nil } -type archetypeMap struct { - // These needs to be parsed and executed as Go templates. - contentFiles []hugofs.FileMetaInfo - // These are just copied to destination. - otherFiles []hugofs.FileMetaInfo - // If the templates needs a fully built site. This can potentially be - // expensive, so only do when needed. - siteUsed bool +func (b *contentBuilder) buildFile() error { + contentPlaceholderAbsFilename, err := b.cf.CreateContentPlaceHolder(b.targetPath) + if err != nil { + return err + } + + usesSite, err := b.usesSiteVar(b.archetypeFilename) + if err != nil { + return err + } + + var contentInclusionFilter *glob.FilenameFilter + if !usesSite { + // We don't need to build everything. + contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool { + return strings.HasPrefix(contentPlaceholderAbsFilename, filename) + }) + } + + if err := b.h.Build(hugolib.BuildCfg{SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil { + return err + } + + if err := b.applyArcheType(contentPlaceholderAbsFilename, b.archetypeFilename); err != nil { + return err + } + + b.h.Log.Infof("Content %q created", contentPlaceholderAbsFilename) + + return b.openInEditorIfConfigured(contentPlaceholderAbsFilename) } -func mapArcheTypeDir( - ps *helpers.PathSpec, - fs afero.Fs, - archetypeDir string) (archetypeMap, error) { +func (b *contentBuilder) setArcheTypeFilenameToUse(ext string) { + var pathsToCheck []string + + if b.kind != "" { + pathsToCheck = append(pathsToCheck, b.kind+ext) + } + pathsToCheck = append(pathsToCheck, "default"+ext, "default") + + for _, p := range pathsToCheck { + fi, err := b.archeTypeFs.Stat(p) + if err == nil { + b.archetypeFilename = p + b.isDir = fi.IsDir() + return + } + } + +} + +func (b *contentBuilder) applyArcheType(contentFilename, archetypeFilename string) error { + p := b.h.GetContentPage(contentFilename) + if p == nil { + panic(fmt.Sprintf("[BUG] no Page found for %q", contentFilename)) + } + + f, err := b.sourceFs.Create(contentFilename) + if err != nil { + return err + } + defer f.Close() + + if archetypeFilename == "" { + return b.cf.AppplyArchetypeTemplate(f, p, b.kind, DefaultArchetypeTemplateTemplate) + } + + return b.cf.AppplyArchetypeFilename(f, p, b.kind, archetypeFilename) + +} + +func (b *contentBuilder) mapArcheTypeDir() error { var m archetypeMap walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error { @@ -215,7 +269,7 @@ func mapArcheTypeDir( if files.IsContentFile(path) { m.contentFiles = append(m.contentFiles, fil) if !m.siteUsed { - m.siteUsed, err = usesSiteVar(fs, path) + m.siteUsed, err = b.usesSiteVar(path) if err != nil { return err } @@ -230,120 +284,60 @@ func mapArcheTypeDir( walkCfg := hugofs.WalkwayConfig{ WalkFn: walkFn, - Fs: fs, - Root: archetypeDir, + Fs: b.archeTypeFs, + Root: b.archetypeFilename, } w := hugofs.NewWalkway(walkCfg) if err := w.Walk(); err != nil { - return m, errors.Wrapf(err, "failed to walk archetype dir %q", archetypeDir) + return errors.Wrapf(err, "failed to walk archetype dir %q", b.archetypeFilename) } - return m, nil + b.dirMap = m + + return nil } -func usesSiteVar(fs afero.Fs, filename string) (bool, error) { - f, err := fs.Open(filename) +func (b *contentBuilder) openInEditorIfConfigured(filename string) error { + editor := b.h.Cfg.GetString("newContentEditor") + if editor == "" { + return nil + } + + b.h.Log.Infof("Editing %q with %q ...\n", filename, editor) + + cmd, err := hexec.SafeCommand(editor, filename) + if err != nil { + return err + } + + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} + +func (b *contentBuilder) usesSiteVar(filename string) (bool, error) { + if filename == "" { + return false, nil + } + bb, err := afero.ReadFile(b.archeTypeFs, filename) if err != nil { return false, errors.Wrap(err, "failed to open archetype file") } - defer f.Close() - return helpers.ReaderContains(f, []byte(".Site")), nil + + return bytes.Contains(bb, []byte(".Site")) || bytes.Contains(bb, []byte("site.")), nil + } -// Resolve the target content path. -func resolveContentPath(sites *hugolib.HugoSites, fs afero.Fs, targetPath string) (string, *hugolib.Site) { - targetDir := filepath.Dir(targetPath) - first := sites.Sites[0] - - var ( - s *hugolib.Site - siteContentDir string - ) - - // Try the filename: my-post.en.md - for _, ss := range sites.Sites { - if strings.Contains(targetPath, "."+ss.Language().Lang+".") { - s = ss - break - } - } - - var dirLang string - - for _, dir := range sites.BaseFs.Content.Dirs { - meta := dir.Meta() - contentDir := meta.Filename - - if !strings.HasSuffix(contentDir, helpers.FilePathSeparator) { - contentDir += helpers.FilePathSeparator - } - - if strings.HasPrefix(targetPath, contentDir) { - siteContentDir = contentDir - dirLang = meta.Lang - break - } - } - - if s == nil && dirLang != "" { - for _, ss := range sites.Sites { - if ss.Lang() == dirLang { - s = ss - break - } - } - } - - if s == nil { - s = first - } - - if targetDir != "" && targetDir != "." { - exists, _ := helpers.Exists(targetDir, fs) - - if exists { - return targetPath, s - } - } - - if siteContentDir == "" { - } - - if siteContentDir != "" { - pp := filepath.Join(siteContentDir, strings.TrimPrefix(targetPath, siteContentDir)) - return s.PathSpec.AbsPathify(pp), s - } else { - var contentDir string - for _, dir := range sites.BaseFs.Content.Dirs { - contentDir = dir.Meta().Filename - if dir.Meta().Lang == s.Lang() { - break - } - } - return s.PathSpec.AbsPathify(filepath.Join(contentDir, targetPath)), s - } -} - -// FindArchetype takes a given kind/archetype of content and returns the path -// to the archetype in the archetype filesystem, blank if none found. -func findArchetype(ps *helpers.PathSpec, kind, ext string) (outpath string, isDir bool) { - fs := ps.BaseFs.Archetypes.Fs - - var pathsToCheck []string - - if kind != "" { - pathsToCheck = append(pathsToCheck, kind+ext) - } - pathsToCheck = append(pathsToCheck, "default"+ext, "default") - - for _, p := range pathsToCheck { - fi, err := fs.Stat(p) - if err == nil { - return p, fi.IsDir() - } - } - - return "", false +type archetypeMap struct { + // These needs to be parsed and executed as Go templates. + contentFiles []hugofs.FileMetaInfo + // These are just copied to destination. + otherFiles []hugofs.FileMetaInfo + // If the templates needs a fully built site. This can potentially be + // expensive, so only do when needed. + siteUsed bool } diff --git a/create/content_template_handler.go b/create/content_template_handler.go deleted file mode 100644 index 09cf4c0a5..000000000 --- a/create/content_template_handler.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2017 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package create - -import ( - "bytes" - "fmt" - "path/filepath" - "strings" - "time" - - "github.com/gohugoio/hugo/common/paths" - - "github.com/pkg/errors" - - "github.com/gohugoio/hugo/helpers" - "github.com/gohugoio/hugo/source" - - "github.com/gohugoio/hugo/hugolib" - "github.com/gohugoio/hugo/tpl" - "github.com/spf13/afero" -) - -// ArchetypeFileData represents the data available to an archetype template. -type ArchetypeFileData struct { - // The archetype content type, either given as --kind option or extracted - // from the target path's section, i.e. "blog/mypost.md" will resolve to - // "blog". - Type string - - // The current date and time as a RFC3339 formatted string, suitable for use in front matter. - Date string - - // The Site, fully equipped with all the pages etc. Note: This will only be set if it is actually - // used in the archetype template. Also, if this is a multilingual setup, - // this site is the site that best matches the target content file, based - // on the presence of language code in the filename. - Site *hugolib.SiteInfo - - // Name will in most cases be the same as TranslationBaseName, e.g. "my-post". - // But if that value is "index" (bundles), the Name is instead the owning folder. - // This is the value you in most cases would want to use to construct the title in your - // archetype template. - Name string - - // The target content file. Note that the .Content will be empty, as that - // has not been created yet. - source.File -} - -const ( - // ArchetypeTemplateTemplate is used as initial template when adding an archetype template. - ArchetypeTemplateTemplate = `--- -title: "{{ replace .Name "-" " " | title }}" -date: {{ .Date }} -draft: true ---- - -` -) - -var ( - archetypeShortcodeReplacementsPre = strings.NewReplacer( - "{{<", "{x{<", - "{{%", "{x{%", - ">}}", ">}x}", - "%}}", "%}x}") - - archetypeShortcodeReplacementsPost = strings.NewReplacer( - "{x{<", "{{<", - "{x{%", "{{%", - ">}x}", ">}}", - "%}x}", "%}}") -) - -func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archetypeFilename string) ([]byte, error) { - var ( - archetypeContent []byte - archetypeTemplate []byte - err error - ) - - f, err := s.SourceSpec.NewFileInfoFrom(targetPath, targetPath) - if err != nil { - return nil, err - } - - if name == "" { - name = f.TranslationBaseName() - - if name == "index" || name == "_index" { - // Page bundles; the directory name will hopefully have a better name. - dir := strings.TrimSuffix(f.Dir(), helpers.FilePathSeparator) - _, name = filepath.Split(dir) - } - } - - data := ArchetypeFileData{ - Type: kind, - Date: time.Now().Format(time.RFC3339), - Name: name, - File: f, - Site: s.Info, - } - - if archetypeFilename == "" { - // TODO(bep) archetype revive the issue about wrong tpl funcs arg order - archetypeTemplate = []byte(ArchetypeTemplateTemplate) - } else { - archetypeTemplate, err = afero.ReadFile(s.BaseFs.Archetypes.Fs, archetypeFilename) - if err != nil { - return nil, fmt.Errorf("failed to read archetype file %s", err) - } - - } - - // The archetype template may contain shortcodes, and these does not play well - // with the Go templates. Need to set some temporary delimiters. - archetypeTemplate = []byte(archetypeShortcodeReplacementsPre.Replace(string(archetypeTemplate))) - - // Reuse the Hugo template setup to get the template funcs properly set up. - templateHandler := s.Deps.Tmpl().(tpl.TemplateManager) - templateName := paths.Filename(archetypeFilename) - if err := templateHandler.AddTemplate("_text/"+templateName, string(archetypeTemplate)); err != nil { - return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename) - } - - templ, _ := templateHandler.Lookup(templateName) - - var buff bytes.Buffer - if err := templateHandler.Execute(templ, &buff, data); err != nil { - return nil, errors.Wrapf(err, "Failed to process archetype file %q:", archetypeFilename) - } - - archetypeContent = []byte(archetypeShortcodeReplacementsPost.Replace(buff.String())) - - return archetypeContent, nil -} diff --git a/create/content_test.go b/create/content_test.go index 38ff7de8d..d40634083 100644 --- a/create/content_test.go +++ b/create/content_test.go @@ -34,27 +34,31 @@ import ( "github.com/spf13/afero" ) +// TODO(bep) clean this up. Export the test site builder in Hugolib or something. func TestNewContent(t *testing.T) { cases := []struct { + name string kind string path string expected []string }{ - {"post", "post/sample-1.md", []string{`title = "Post Arch title"`, `test = "test1"`, "date = \"2015-01-12T19:20:04-07:00\""}}, - {"post", "post/org-1.org", []string{`#+title: ORG-1`}}, - {"emptydate", "post/sample-ed.md", []string{`title = "Empty Date Arch title"`, `test = "test1"`}}, - {"stump", "stump/sample-2.md", []string{`title: "Sample 2"`}}, // no archetype file - {"", "sample-3.md", []string{`title: "Sample 3"`}}, // no archetype - {"product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter - {"lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}}, - {"lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}}, - {"lang", "content/post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}}, - {"lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}}, - {"lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}}, - {"lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, - {"lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, - {"lang", "content/post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}}, - {"shortcodes", "shortcodes/go.md", []string{ + {"Post", "post", "post/sample-1.md", []string{`title = "Post Arch title"`, `test = "test1"`, "date = \"2015-01-12T19:20:04-07:00\""}}, + {"Post org-mode", "post", "post/org-1.org", []string{`#+title: ORG-1`}}, + {"Empty date", "emptydate", "post/sample-ed.md", []string{`title = "Empty Date Arch title"`, `test = "test1"`}}, + {"Archetype file not found", "stump", "stump/sample-2.md", []string{`title: "Sample 2"`}}, // no archetype file + {"No archetype", "", "sample-3.md", []string{`title: "Sample 3"`}}, // no archetype + {"Empty archetype", "product", "product/sample-4.md", []string{`title = "SAMPLE-4"`}}, // empty archetype front matter + {"Filenames", "filenames", "content/mypage/index.md", []string{"title = \"INDEX\"\n+++\n\n\nContentBaseName: mypage"}}, + {"Lang 1", "lang", "post/lang-1.md", []string{`Site Lang: en|Name: Lang 1|i18n: Hugo Rocks!`}}, + {"Lang 2", "lang", "post/lang-2.en.md", []string{`Site Lang: en|Name: Lang 2|i18n: Hugo Rocks!`}}, + {"Lang nn file", "lang", "content/post/lang-3.nn.md", []string{`Site Lang: nn|Name: Lang 3|i18n: Hugo Rokkar!`}}, + {"Lang nn dir", "lang", "content_nn/post/lang-4.md", []string{`Site Lang: nn|Name: Lang 4|i18n: Hugo Rokkar!`}}, + {"Lang en in nn dir", "lang", "content_nn/post/lang-5.en.md", []string{`Site Lang: en|Name: Lang 5|i18n: Hugo Rocks!`}}, + {"Lang en default", "lang", "post/my-bundle/index.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, + {"Lang en file", "lang", "post/my-bundle/index.en.md", []string{`Site Lang: en|Name: My Bundle|i18n: Hugo Rocks!`}}, + {"Lang nn bundle", "lang", "content/post/my-bundle/index.nn.md", []string{`Site Lang: nn|Name: My Bundle|i18n: Hugo Rokkar!`}}, + {"Site", "site", "content/mypage/index.md", []string{"RegularPages .Site: 10", "RegularPages site: 10"}}, + {"Shortcodes", "shortcodes", "shortcodes/go.md", []string{ `title = "GO"`, "{{< myshortcode >}}", "{{% myshortcode %}}", @@ -62,11 +66,14 @@ func TestNewContent(t *testing.T) { }}, // shortcodes } + c := qt.New(t) + for i, cas := range cases { cas := cas - t.Run(fmt.Sprintf("%s-%d", cas.kind, i), func(t *testing.T) { - t.Parallel() - c := qt.New(t) + + c.Run(cas.name, func(c *qt.C) { + c.Parallel() + mm := afero.NewMemMapFs() c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) @@ -79,11 +86,11 @@ func TestNewContent(t *testing.T) { if !strings.HasPrefix(fname, "content") { fname = filepath.Join("content", fname) } - content := readFileFromFs(t, fs.Source, fname) + content := readFileFromFs(c, fs.Source, fname) for _, v := range cas.expected { found := strings.Contains(content, v) if !found { - t.Fatalf("[%d] %q missing from output:\n%q", i, v, content) + c.Fatalf("[%d] %q missing from output:\n%q", i, v, content) } } }) @@ -96,10 +103,10 @@ func TestNewContentFromDir(t *testing.T) { c := qt.New(t) archetypeDir := filepath.Join("archetypes", "my-bundle") - c.Assert(mm.MkdirAll(archetypeDir, 0755), qt.IsNil) + c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil) archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle") - c.Assert(mm.MkdirAll(archetypeThemeDir, 0755), qt.IsNil) + c.Assert(mm.MkdirAll(archetypeThemeDir, 0o755), qt.IsNil) contentFile := ` File: %s @@ -108,15 +115,15 @@ Name: {{ replace .Name "-" " " | title }} i18n: {{ T "hugo" }} ` - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0o755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0755), qt.IsNil) - c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) c.Assert(initFs(mm), qt.IsNil) cfg, fs := newTestCfg(c, mm) @@ -135,15 +142,90 @@ i18n: {{ T "hugo" }} cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Post`, `i18n: Hugo Rocks!`) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Site Lang: nn`, `Name: My Post`, `i18n: Hugo Rokkar!`) - cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: My Post`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Site Lang: en`, `Name: Bio`) c.Assert(create.NewContent(h, "my-theme-bundle", "post/my-theme-post"), qt.IsNil) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Site Lang: en`, `Name: My Theme Post`, `i18n: Hugo Rocks!`) cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`) } +func TestNewContentFromDirSiteFunction(t *testing.T) { + mm := afero.NewMemMapFs() + c := qt.New(t) + + archetypeDir := filepath.Join("archetypes", "my-bundle") + c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil) + + contentFile := ` +File: %s +site RegularPages: {{ len site.RegularPages }} + +` + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + + c.Assert(initFs(mm), qt.IsNil) + cfg, fs := newTestCfg(c, mm) + + h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + c.Assert(err, qt.IsNil) + c.Assert(len(h.Sites), qt.Equals, 2) + + c.Assert(create.NewContent(h, "my-bundle", "post/my-post"), qt.IsNil) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `site RegularPages: 10`) +} + +func TestNewContentFromDirNoSite(t *testing.T) { + mm := afero.NewMemMapFs() + c := qt.New(t) + + archetypeDir := filepath.Join("archetypes", "my-bundle") + c.Assert(mm.MkdirAll(archetypeDir, 0o755), qt.IsNil) + + archetypeThemeDir := filepath.Join("themes", "mytheme", "archetypes", "my-theme-bundle") + c.Assert(mm.MkdirAll(archetypeThemeDir, 0o755), qt.IsNil) + + contentFile := ` +File: %s +Name: {{ replace .Name "-" " " | title }} +i18n: {{ T "hugo" }} +` + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "index.nn.md"), []byte(fmt.Sprintf(contentFile, "index.nn.md")), 0o755), qt.IsNil) + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "pages", "bio.md"), []byte(fmt.Sprintf(contentFile, "bio.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeDir, "resources", "hugo2.xml"), []byte(`hugo2: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "index.md"), []byte(fmt.Sprintf(contentFile, "index.md")), 0o755), qt.IsNil) + c.Assert(afero.WriteFile(mm, filepath.Join(archetypeThemeDir, "resources", "hugo1.json"), []byte(`hugo1: {{ printf "no template handling in here" }}`), 0o755), qt.IsNil) + + c.Assert(initFs(mm), qt.IsNil) + cfg, fs := newTestCfg(c, mm) + + h, err := hugolib.NewHugoSites(deps.DepsCfg{Cfg: cfg, Fs: fs}) + c.Assert(err, qt.IsNil) + c.Assert(len(h.Sites), qt.Equals, 2) + + c.Assert(create.NewContent(h, "my-bundle", "post/my-post"), qt.IsNil) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/resources/hugo2.xml")), `hugo2: {{ printf "no template handling in here" }}`) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.md")), `File: index.md`, `Name: My Post`, `i18n: Hugo Rocks!`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/index.nn.md")), `File: index.nn.md`, `Name: My Post`, `i18n: Hugo Rokkar!`) + + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-post/pages/bio.md")), `File: bio.md`, `Name: Bio`) + + c.Assert(create.NewContent(h, "my-theme-bundle", "post/my-theme-post"), qt.IsNil) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/index.md")), `File: index.md`, `Name: My Theme Post`, `i18n: Hugo Rocks!`) + cContains(c, readFileFromFs(t, fs.Source, filepath.Join("content", "post/my-theme-post/resources/hugo1.json")), `hugo1: {{ printf "no template handling in here" }}`) +} + func initFs(fs afero.Fs) error { - perm := os.FileMode(0755) + perm := os.FileMode(0o755) var err error // create directories @@ -159,7 +241,16 @@ func initFs(fs afero.Fs) error { } } - // create files + // create some dummy content + for i := 1; i <= 10; i++ { + filename := filepath.Join("content", fmt.Sprintf("page%d.md", i)) + afero.WriteFile(fs, filename, []byte(`--- +title: Test +--- +`), 0666) + } + + // create archetype files for _, v := range []struct { path string content string @@ -177,6 +268,29 @@ func initFs(fs afero.Fs) error { content: `+++ title = "{{ .BaseFileName | upper }}" +++`, + }, + { + path: filepath.Join("archetypes", "filenames.md"), + content: `... +title = "{{ .BaseFileName | upper }}" ++++ + + +ContentBaseName: {{ .File.ContentBaseName }} + +`, + }, + { + path: filepath.Join("archetypes", "site.md"), + content: `... +title = "{{ .BaseFileName | upper }}" ++++ + +Len RegularPages .Site: {{ len .Site.RegularPages }} +Len RegularPages site: {{ len site.RegularPages }} + + +`, }, { path: filepath.Join("archetypes", "emptydate.md"), @@ -184,7 +298,7 @@ title = "{{ .BaseFileName | upper }}" }, { path: filepath.Join("archetypes", "lang.md"), - content: `Site Lang: {{ .Site.Language.Lang }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`, + content: `Site Lang: {{ site.Language.Lang }}|Name: {{ replace .Name "-" " " | title }}|i18n: {{ T "hugo" }}`, }, // #3623x { @@ -227,7 +341,7 @@ func cContains(c *qt.C, v interface{}, matches ...string) { } // TODO(bep) extract common testing package with this and some others -func readFileFromFs(t *testing.T, fs afero.Fs, filename string) string { +func readFileFromFs(t testing.TB, fs afero.Fs, filename string) string { t.Helper() filename = filepath.FromSlash(filename) b, err := afero.ReadFile(fs, filename) @@ -257,23 +371,34 @@ languageName = "English" [languages.nn] weight = 2 languageName = "Nynorsk" -contentDir = "content_nn" +[module] +[[module.mounts]] + source = 'archetypes' + target = 'archetypes' +[[module.mounts]] + source = 'content' + target = 'content' + lang = 'en' +[[module.mounts]] + source = 'content_nn' + target = 'content' + lang = 'nn' ` if mm == nil { mm = afero.NewMemMapFs() } - mm.MkdirAll(filepath.FromSlash("content_nn"), 0777) + mm.MkdirAll(filepath.FromSlash("content_nn"), 0o777) - mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0777) + mm.MkdirAll(filepath.FromSlash("themes/mytheme"), 0o777) c.Assert(afero.WriteFile(mm, filepath.Join("i18n", "en.toml"), []byte(`[hugo] -other = "Hugo Rocks!"`), 0755), qt.IsNil) +other = "Hugo Rocks!"`), 0o755), qt.IsNil) c.Assert(afero.WriteFile(mm, filepath.Join("i18n", "nn.toml"), []byte(`[hugo] -other = "Hugo Rokkar!"`), 0755), qt.IsNil) +other = "Hugo Rokkar!"`), 0o755), qt.IsNil) - c.Assert(afero.WriteFile(mm, "config.toml", []byte(cfg), 0755), qt.IsNil) + c.Assert(afero.WriteFile(mm, "config.toml", []byte(cfg), 0o755), qt.IsNil) v, _, err := hugolib.LoadConfig(hugolib.ConfigSourceDescriptor{Fs: mm, Filename: "config.toml"}) c.Assert(err, qt.IsNil) diff --git a/deps/deps.go b/deps/deps.go index c0546db76..6b9da21fe 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -253,7 +253,7 @@ func New(cfg DepsCfg) (*Deps, error) { return nil, err } - sp := source.NewSourceSpec(ps, fs.Source) + sp := source.NewSourceSpec(ps, nil, fs.Source) timeoutms := cfg.Language.GetInt("timeout") if timeoutms <= 0 { diff --git a/hugofs/glob/glob.go b/hugofs/glob/glob.go index 57115ddfa..6dd0df5ed 100644 --- a/hugofs/glob/glob.go +++ b/hugofs/glob/glob.go @@ -16,6 +16,7 @@ package glob import ( "path" "path/filepath" + "runtime" "strings" "sync" @@ -23,46 +24,100 @@ import ( "github.com/gobwas/glob/syntax" ) +var ( + isWindows = runtime.GOOS == "windows" + defaultGlobCache = &globCache{ + isCaseSensitive: false, + isWindows: isWindows, + cache: make(map[string]globErr), + } + + filenamesGlobCache = &globCache{ + isCaseSensitive: true, // TODO(bep) bench + isWindows: isWindows, + cache: make(map[string]globErr), + } +) + type globErr struct { glob glob.Glob err error } -var ( - globCache = make(map[string]globErr) - globMu sync.RWMutex -) +type globCache struct { + // Config + isCaseSensitive bool + isWindows bool -type caseInsensitiveGlob struct { - g glob.Glob + // Cache + sync.RWMutex + cache map[string]globErr } -func (g caseInsensitiveGlob) Match(s string) bool { - return g.g.Match(strings.ToLower(s)) - -} -func GetGlob(pattern string) (glob.Glob, error) { +func (gc *globCache) GetGlob(pattern string) (glob.Glob, error) { var eg globErr - globMu.RLock() + gc.RLock() var found bool - eg, found = globCache[pattern] - globMu.RUnlock() + eg, found = gc.cache[pattern] + gc.RUnlock() if found { return eg.glob, eg.err } + var g glob.Glob var err error - g, err := glob.Compile(strings.ToLower(pattern), '/') - eg = globErr{caseInsensitiveGlob{g: g}, err} - globMu.Lock() - globCache[pattern] = eg - globMu.Unlock() + pattern = filepath.ToSlash(pattern) + + if gc.isCaseSensitive { + g, err = glob.Compile(pattern, '/') + } else { + g, err = glob.Compile(strings.ToLower(pattern), '/') + + } + + eg = globErr{ + globDecorator{ + g: g, + isCaseSensitive: gc.isCaseSensitive, + isWindows: gc.isWindows}, + err, + } + + gc.Lock() + gc.cache[pattern] = eg + gc.Unlock() return eg.glob, eg.err } +type globDecorator struct { + // Whether both pattern and the strings to match will be matched + // by their original case. + isCaseSensitive bool + + // On Windows we may get filenames with Windows slashes to match, + // which wee need to normalize. + isWindows bool + + g glob.Glob +} + +func (g globDecorator) Match(s string) bool { + if g.isWindows { + s = filepath.ToSlash(s) + } + if !g.isCaseSensitive { + s = strings.ToLower(s) + } + return g.g.Match(s) +} + +func GetGlob(pattern string) (glob.Glob, error) { + return defaultGlobCache.GetGlob(pattern) +} + func NormalizePath(p string) string { return strings.Trim(path.Clean(filepath.ToSlash(strings.ToLower(p))), "/.") } @@ -106,3 +161,78 @@ func HasGlobChar(s string) bool { } return false } + +type FilenameFilter struct { + shouldInclude func(filename string) bool + inclusions []glob.Glob + exclusions []glob.Glob + isWindows bool +} + +// NewFilenameFilter creates a new Glob where the Match method will +// return true if the file should be exluded. +// Note that the inclusions will be checked first. +func NewFilenameFilter(inclusions, exclusions []string) (*FilenameFilter, error) { + filter := &FilenameFilter{isWindows: isWindows} + + for _, include := range inclusions { + g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(include)) + if err != nil { + return nil, err + } + filter.inclusions = append(filter.inclusions, g) + } + for _, exclude := range exclusions { + g, err := filenamesGlobCache.GetGlob(filepath.FromSlash(exclude)) + if err != nil { + return nil, err + } + filter.exclusions = append(filter.exclusions, g) + } + + return filter, nil +} + +// NewFilenameFilterForInclusionFunc create a new filter using the provided inclusion func. +func NewFilenameFilterForInclusionFunc(shouldInclude func(filename string) bool) *FilenameFilter { + return &FilenameFilter{shouldInclude: shouldInclude, isWindows: isWindows} +} + +// Match returns whether filename should be included. +func (f *FilenameFilter) Match(filename string) bool { + if f == nil { + return true + } + + if f.shouldInclude != nil { + if f.shouldInclude(filename) { + return true + } + if f.isWindows { + // The Glob matchers below handles this by themselves, + // for the shouldInclude we need to take some extra steps + // to make this robust. + winFilename := filepath.FromSlash(filename) + if filename != winFilename { + if f.shouldInclude(winFilename) { + return true + } + } + } + + } + + for _, inclusion := range f.inclusions { + if inclusion.Match(filename) { + return true + } + } + + for _, exclusion := range f.exclusions { + if exclusion.Match(filename) { + return false + } + } + + return f.inclusions == nil && f.shouldInclude == nil +} diff --git a/hugofs/glob/glob_test.go b/hugofs/glob/glob_test.go index cd64ba112..7ef3fbbed 100644 --- a/hugofs/glob/glob_test.go +++ b/hugofs/glob/glob_test.go @@ -15,6 +15,7 @@ package glob import ( "path/filepath" + "strings" "testing" qt "github.com/frankban/quicktest" @@ -72,6 +73,40 @@ func TestGetGlob(t *testing.T) { c.Assert(g.Match("data/my.json"), qt.Equals, true) } +func TestFilenameFilter(t *testing.T) { + c := qt.New(t) + + excludeAlmostAllJSON, err := NewFilenameFilter([]string{"a/b/c/foo.json"}, []string{"**.json"}) + c.Assert(err, qt.IsNil) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("data/my.json")), qt.Equals, false) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.json")), qt.Equals, true) + c.Assert(excludeAlmostAllJSON.Match(filepath.FromSlash("a/b/c/foo.bar")), qt.Equals, false) + + nopFilter, err := NewFilenameFilter(nil, nil) + c.Assert(err, qt.IsNil) + c.Assert(nopFilter.Match("ab.txt"), qt.Equals, true) + + includeOnlyFilter, err := NewFilenameFilter([]string{"**.json", "**.jpg"}, nil) + c.Assert(err, qt.IsNil) + c.Assert(includeOnlyFilter.Match("ab.json"), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.jpg"), qt.Equals, true) + c.Assert(includeOnlyFilter.Match("ab.gif"), qt.Equals, false) + + exlcudeOnlyFilter, err := NewFilenameFilter(nil, []string{"**.json", "**.jpg"}) + c.Assert(err, qt.IsNil) + c.Assert(exlcudeOnlyFilter.Match("ab.json"), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.jpg"), qt.Equals, false) + c.Assert(exlcudeOnlyFilter.Match("ab.gif"), qt.Equals, true) + + var nilFilter *FilenameFilter + c.Assert(nilFilter.Match("ab.gif"), qt.Equals, true) + + funcFilter := NewFilenameFilterForInclusionFunc(func(s string) bool { return strings.HasSuffix(s, ".json") }) + c.Assert(funcFilter.Match("ab.json"), qt.Equals, true) + c.Assert(funcFilter.Match("ab.bson"), qt.Equals, false) + +} + func BenchmarkGetGlob(b *testing.B) { for i := 0; i < b.N; i++ { _, err := GetGlob("**/foo") diff --git a/hugolib/content_factory.go b/hugolib/content_factory.go new file mode 100644 index 000000000..b94608e86 --- /dev/null +++ b/hugolib/content_factory.go @@ -0,0 +1,181 @@ +// Copyright 2021 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hugolib + +import ( + "io" + "path/filepath" + "strings" + "time" + + "github.com/gohugoio/hugo/helpers" + + "github.com/gohugoio/hugo/source" + + "github.com/gohugoio/hugo/resources/page" + + "github.com/pkg/errors" + "github.com/spf13/afero" +) + +// ContentFactory creates content files from archetype templates. +type ContentFactory struct { + h *HugoSites + + // We parse the archetype templates as Go templates, so we need + // to replace any shortcode with a temporary placeholder. + shortocdeReplacerPre *strings.Replacer + shortocdeReplacerPost *strings.Replacer +} + +// AppplyArchetypeFilename archetypeFilename to w as a template using the given Page p as the foundation for the data context. +func (f ContentFactory) AppplyArchetypeFilename(w io.Writer, p page.Page, archetypeKind, archetypeFilename string) error { + + fi, err := f.h.SourceFilesystems.Archetypes.Fs.Stat(archetypeFilename) + if err != nil { + return err + } + + if fi.IsDir() { + return errors.Errorf("archetype directory (%q) not supported", archetypeFilename) + } + + templateSource, err := afero.ReadFile(f.h.SourceFilesystems.Archetypes.Fs, archetypeFilename) + if err != nil { + return errors.Wrapf(err, "failed to read archetype file %q: %s", archetypeFilename, err) + + } + + return f.AppplyArchetypeTemplate(w, p, archetypeKind, string(templateSource)) + +} + +// AppplyArchetypeFilename templateSource to w as a template using the given Page p as the foundation for the data context. +func (f ContentFactory) AppplyArchetypeTemplate(w io.Writer, p page.Page, archetypeKind, templateSource string) error { + ps := p.(*pageState) + if archetypeKind == "" { + archetypeKind = p.Type() + } + + d := &archetypeFileData{ + Type: archetypeKind, + Date: time.Now().Format(time.RFC3339), + Page: p, + File: p.File(), + } + + templateSource = f.shortocdeReplacerPre.Replace(templateSource) + + templ, err := ps.s.TextTmpl().Parse("archetype.md", string(templateSource)) + if err != nil { + return errors.Wrapf(err, "failed to parse archetype template: %s", err) + } + + result, err := executeToString(ps.s.Tmpl(), templ, d) + if err != nil { + return errors.Wrapf(err, "failed to execute archetype template: %s", err) + } + + _, err = io.WriteString(w, f.shortocdeReplacerPost.Replace(result)) + + return err + +} + +func (f ContentFactory) SectionFromFilename(filename string) string { + filename = filepath.Clean(filename) + rel, _ := f.h.AbsProjectContentDir(filename) + if rel == "" { + return "" + } + + parts := strings.Split(helpers.ToSlashTrimLeading(rel), "/") + if len(parts) < 2 { + return "" + } + return parts[0] +} + +// CreateContentPlaceHolder creates a content placeholder file inside the +// best matching content directory. +func (f ContentFactory) CreateContentPlaceHolder(filename string) (string, error) { + filename = filepath.Clean(filename) + _, abs := f.h.AbsProjectContentDir(filename) + + contentDir := filepath.Dir(abs) + if err := f.h.Fs.Source.MkdirAll(contentDir, 0777); err != nil { + return "", err + } + + // This will be overwritten later, just write a placholder to get + // the paths correct. + placeholder := `--- +title: "Content Placeholder" +_build: + render: never + list: never + publishResources: false +--- + +` + + if err := afero.WriteFile(f.h.Fs.Source, abs, []byte(placeholder), 0777); err != nil { + return "", err + } + + return abs, nil +} + +// NewContentFactory creates a new ContentFactory for h. +func NewContentFactory(h *HugoSites) ContentFactory { + return ContentFactory{ + h: h, + shortocdeReplacerPre: strings.NewReplacer( + "{{<", "{x{<", + "{{%", "{x{%", + ">}}", ">}x}", + "%}}", "%}x}"), + shortocdeReplacerPost: strings.NewReplacer( + "{x{<", "{{<", + "{x{%", "{{%", + ">}x}", ">}}", + "%}x}", "%}}"), + } +} + +// archetypeFileData represents the data available to an archetype template. +type archetypeFileData struct { + // The archetype content type, either given as --kind option or extracted + // from the target path's section, i.e. "blog/mypost.md" will resolve to + // "blog". + Type string + + // The current date and time as a RFC3339 formatted string, suitable for use in front matter. + Date string + + // The temporary page. Note that only the file path information is relevant at this stage. + Page page.Page + + // File is the same as Page.File, embedded here for historic reasons. + // TODO(bep) make this a method. + source.File +} + +func (f *archetypeFileData) Site() page.Site { + return f.Page.Site() +} + +func (f *archetypeFileData) Name() string { + return f.Page.File().ContentBaseName() +} diff --git a/hugolib/content_factory_test.go b/hugolib/content_factory_test.go new file mode 100644 index 000000000..50cc783f6 --- /dev/null +++ b/hugolib/content_factory_test.go @@ -0,0 +1,60 @@ +package hugolib + +import ( + "bytes" + "path/filepath" + "testing" + + qt "github.com/frankban/quicktest" +) + +func TestContentFactory(t *testing.T) { + t.Parallel() + + c := qt.New(t) + + c.Run("Simple", func(c *qt.C) { + workingDir := "/my/work" + b := newTestSitesBuilder(c) + b.WithWorkingDir(workingDir).WithConfigFile("toml", ` + +workingDir="/my/work" + +[module] +[[module.mounts]] +source = 'mcontent/en' +target = 'content' +lang = 'en' +[[module.mounts]] +source = 'archetypes' +target = 'archetypes' + +`) + + b.WithSourceFile(filepath.Join("mcontent/en/bundle", "index.md"), "") + + b.WithSourceFile(filepath.Join("archetypes", "post.md"), `--- +title: "{{ replace .Name "-" " " | title }}" +date: {{ .Date }} +draft: true +--- + +Hello World. +`) + b.CreateSites() + cf := NewContentFactory(b.H) + abs, err := cf.CreateContentPlaceHolder(filepath.FromSlash("mcontent/en/blog/mypage.md")) + b.Assert(err, qt.IsNil) + b.Assert(abs, qt.Equals, filepath.FromSlash("/my/work/mcontent/en/blog/mypage.md")) + b.Build(BuildCfg{SkipRender: true}) + + p := b.H.GetContentPage(abs) + b.Assert(p, qt.Not(qt.IsNil)) + + var buf bytes.Buffer + b.Assert(cf.AppplyArchetypeFilename(&buf, p, "", "post.md"), qt.IsNil) + + b.Assert(buf.String(), qt.Contains, `title: "Mypage"`) + }) + +} diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go index d238d2e03..dcfee34ff 100644 --- a/hugolib/filesystems/basefs.go +++ b/hugolib/filesystems/basefs.go @@ -102,6 +102,42 @@ func (b *BaseFs) RelContentDir(filename string) string { return filename } +// AbsProjectContentDir tries to create a TODO1 +func (b *BaseFs) AbsProjectContentDir(filename string) (string, string) { + isAbs := filepath.IsAbs(filename) + for _, dir := range b.SourceFilesystems.Content.Dirs { + meta := dir.Meta() + if meta.Module != "project" { + continue + } + if isAbs { + if strings.HasPrefix(filename, meta.Filename) { + return strings.TrimPrefix(filename, meta.Filename), filename + } + } else { + contentDir := strings.TrimPrefix(strings.TrimPrefix(meta.Filename, meta.BaseDir), filePathSeparator) + if strings.HasPrefix(filename, contentDir) { + relFilename := strings.TrimPrefix(filename, contentDir) + absFilename := filepath.Join(meta.Filename, relFilename) + return relFilename, absFilename + } + } + + } + + if !isAbs { + // A filename on the form "posts/mypage.md", put it inside + // the first content folder, usually /content. + // The Dirs are ordered with the most important last, so pick that. + contentDirs := b.SourceFilesystems.Content.Dirs + firstContentDir := contentDirs[len(contentDirs)-1].Meta().Filename + return filename, filepath.Join(firstContentDir, filename) + + } + + return "", "" +} + // ResolveJSConfigFile resolves the JS-related config file to a absolute // filename. One example of such would be postcss.config.js. func (fs *BaseFs) ResolveJSConfigFile(name string) string { diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 27c490cc0..141019a85 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -22,6 +22,8 @@ import ( "sync" "sync/atomic" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/fsnotify/fsnotify" "github.com/gohugoio/hugo/identity" @@ -677,6 +679,9 @@ type BuildCfg struct { // Recently visited URLs. This is used for partial re-rendering. RecentlyVisited map[string]bool + // Can be set to build only with a sub set of the content source. + ContentInclusionFilter *glob.FilenameFilter + testCounters *testCounters } @@ -819,7 +824,7 @@ func (h *HugoSites) Pages() page.Pages { } func (h *HugoSites) loadData(fis []hugofs.FileMetaInfo) (err error) { - spec := source.NewSourceSpec(h.PathSpec, nil) + spec := source.NewSourceSpec(h.PathSpec, nil, nil) h.data = make(map[string]interface{}) for _, fi := range fis { diff --git a/hugolib/pages_capture_test.go b/hugolib/pages_capture_test.go index 0fdc73e76..4b2979a0a 100644 --- a/hugolib/pages_capture_test.go +++ b/hugolib/pages_capture_test.go @@ -51,7 +51,7 @@ func TestPagesCapture(t *testing.T) { ps, err := helpers.NewPathSpec(hugofs.NewFrom(fs, cfg), cfg, loggers.NewErrorLogger()) c.Assert(err, qt.IsNil) - sourceSpec := source.NewSourceSpec(ps, fs) + sourceSpec := source.NewSourceSpec(ps, nil, fs) t.Run("Collect", func(t *testing.T) { c := qt.New(t) diff --git a/hugolib/site.go b/hugolib/site.go index 18c9bfc80..96cf0b93c 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -1193,7 +1193,7 @@ func (s *Site) processPartial(config *BuildCfg, init func(config *BuildCfg) erro filenamesChanged = helpers.UniqueStringsReuse(filenamesChanged) - if err := s.readAndProcessContent(filenamesChanged...); err != nil { + if err := s.readAndProcessContent(*config, filenamesChanged...); err != nil { return err } @@ -1207,7 +1207,7 @@ func (s *Site) process(config BuildCfg) (err error) { err = errors.Wrap(err, "initialize") return } - if err = s.readAndProcessContent(); err != nil { + if err = s.readAndProcessContent(config); err != nil { err = errors.Wrap(err, "readAndProcessContent") return } @@ -1376,8 +1376,8 @@ func (s *Site) eventToIdentity(e fsnotify.Event) (identity.PathIdentity, bool) { return identity.PathIdentity{}, false } -func (s *Site) readAndProcessContent(filenames ...string) error { - sourceSpec := source.NewSourceSpec(s.PathSpec, s.BaseFs.Content.Fs) +func (s *Site) readAndProcessContent(buildConfig BuildCfg, filenames ...string) error { + sourceSpec := source.NewSourceSpec(s.PathSpec, buildConfig.ContentInclusionFilter, s.BaseFs.Content.Fs) proc := newPagesProcessor(s.h, sourceSpec) diff --git a/langs/i18n/translationProvider.go b/langs/i18n/translationProvider.go index e8bfbf6f1..9e82b18d9 100644 --- a/langs/i18n/translationProvider.go +++ b/langs/i18n/translationProvider.go @@ -46,7 +46,7 @@ func NewTranslationProvider() *TranslationProvider { // Update updates the i18n func in the provided Deps. func (tp *TranslationProvider) Update(d *deps.Deps) error { - spec := source.NewSourceSpec(d.PathSpec, nil) + spec := source.NewSourceSpec(d.PathSpec, nil, nil) bundle := i18n.NewBundle(language.English) bundle.RegisterUnmarshalFunc("toml", toml.Unmarshal) diff --git a/source/content_directory_test.go b/source/content_directory_test.go index d3723c6b1..8f1d0df4f 100644 --- a/source/content_directory_test.go +++ b/source/content_directory_test.go @@ -57,7 +57,7 @@ func TestIgnoreDotFilesAndDirectories(t *testing.T) { ps, err := helpers.NewPathSpec(fs, v, nil) c.Assert(err, qt.IsNil) - s := NewSourceSpec(ps, fs.Source) + s := NewSourceSpec(ps, nil, fs.Source) if ignored := s.IgnoreFile(filepath.FromSlash(test.path)); test.ignore != ignored { t.Errorf("[%d] File not ignored", i) diff --git a/source/filesystem_test.go b/source/filesystem_test.go index 0b8c1d395..6343c6a41 100644 --- a/source/filesystem_test.go +++ b/source/filesystem_test.go @@ -106,5 +106,5 @@ func newTestSourceSpec() *SourceSpec { if err != nil { panic(err) } - return NewSourceSpec(ps, fs.Source) + return NewSourceSpec(ps, nil, fs.Source) } diff --git a/source/sourceSpec.go b/source/sourceSpec.go index e8407a14d..3640c83d5 100644 --- a/source/sourceSpec.go +++ b/source/sourceSpec.go @@ -19,6 +19,8 @@ import ( "regexp" "runtime" + "github.com/gohugoio/hugo/hugofs/glob" + "github.com/gohugoio/hugo/langs" "github.com/spf13/afero" @@ -33,8 +35,7 @@ type SourceSpec struct { SourceFs afero.Fs - // This is set if the ignoreFiles config is set. - ignoreFilesRe []*regexp.Regexp + shouldInclude func(filename string) bool Languages map[string]interface{} DefaultContentLanguage string @@ -42,7 +43,7 @@ type SourceSpec struct { } // NewSourceSpec initializes SourceSpec using languages the given filesystem and PathSpec. -func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { +func NewSourceSpec(ps *helpers.PathSpec, inclusionFilter *glob.FilenameFilter, fs afero.Fs) *SourceSpec { cfg := ps.Cfg defaultLang := cfg.GetString("defaultContentLanguage") languages := cfg.GetStringMap("languages") @@ -72,8 +73,19 @@ func NewSourceSpec(ps *helpers.PathSpec, fs afero.Fs) *SourceSpec { } } + shouldInclude := func(filename string) bool { + if !inclusionFilter.Match(filename) { + return false + } + for _, r := range regexps { + if r.MatchString(filename) { + return false + } + } + return true + } - return &SourceSpec{ignoreFilesRe: regexps, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} + return &SourceSpec{shouldInclude: shouldInclude, PathSpec: ps, SourceFs: fs, Languages: languages, DefaultContentLanguage: defaultLang, DisabledLanguages: disabledLangsSet} } // IgnoreFile returns whether a given file should be ignored. @@ -97,24 +109,16 @@ func (s *SourceSpec) IgnoreFile(filename string) bool { } } - if len(s.ignoreFilesRe) == 0 { - return false - } - - for _, re := range s.ignoreFilesRe { - if re.MatchString(filename) { - return true - } + if !s.shouldInclude(filename) { + return true } if runtime.GOOS == "windows" { // Also check the forward slash variant if different. unixFilename := filepath.ToSlash(filename) if unixFilename != filename { - for _, re := range s.ignoreFilesRe { - if re.MatchString(unixFilename) { - return true - } + if !s.shouldInclude(unixFilename) { + return true } } }