From ebcc1e6699dc60adf9b4b0b4edd80eeb185355f1 Mon Sep 17 00:00:00 2001 From: bep Date: Wed, 11 Feb 2015 20:24:56 +0100 Subject: [PATCH] Add data files support in themes If duplicate keys, the main data dir wins. Fixes #892 --- commands/hugo.go | 18 +++++------- helpers/general.go | 8 ++++-- helpers/path.go | 23 +++++++++++++++ hugolib/site.go | 68 ++++++++++++++++++++++++++------------------ hugolib/site_test.go | 29 +++++++++++++++---- 5 files changed, 100 insertions(+), 46 deletions(-) diff --git a/commands/hugo.go b/commands/hugo.go index 65195f8d6..747a4b40e 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -252,13 +252,13 @@ func copyStatic() error { syncer.SrcFs = hugofs.SourceFs syncer.DestFs = hugofs.DestinationFS - if themeSet() { - themeDir := helpers.AbsPathify("themes/"+viper.GetString("theme")) + "/static/" - if _, err := os.Stat(themeDir); os.IsNotExist(err) { - jww.ERROR.Println("Unable to find static directory for theme:", viper.GetString("theme"), "in", themeDir) - return nil - } + themeDir, err := helpers.GetThemeStaticDirPath() + if err != nil { + jww.ERROR.Println(err) + return nil + } + if themeDir != "" { // Copy Static to Destination jww.INFO.Println("syncing from", themeDir, "to", publishDir) utils.CheckErr(syncer.Sync(publishDir, themeDir), fmt.Sprintf("Error copying static files of theme to %s", publishDir)) @@ -292,17 +292,13 @@ func getDirList() []string { filepath.Walk(helpers.AbsPathify(viper.GetString("ContentDir")), walker) filepath.Walk(helpers.AbsPathify(viper.GetString("LayoutDir")), walker) filepath.Walk(helpers.AbsPathify(viper.GetString("StaticDir")), walker) - if themeSet() { + if helpers.ThemeSet() { filepath.Walk(helpers.AbsPathify("themes/"+viper.GetString("theme")), walker) } return a } -func themeSet() bool { - return viper.GetString("theme") != "" -} - func buildSite(watching ...bool) (err error) { startTime := time.Now() site := &hugolib.Site{} diff --git a/helpers/general.go b/helpers/general.go index 32666defa..83854389e 100644 --- a/helpers/general.go +++ b/helpers/general.go @@ -19,13 +19,13 @@ import ( "encoding/hex" "errors" "fmt" + bp "github.com/spf13/hugo/bufferpool" + "github.com/spf13/viper" "io" "net" "path/filepath" "reflect" "strings" - - bp "github.com/spf13/hugo/bufferpool" ) // Filepath separator defined by os.Separator. @@ -100,6 +100,10 @@ func BytesToReader(in []byte) io.Reader { return bytes.NewReader(in) } +func ThemeSet() bool { + return viper.GetString("theme") != "" +} + // SliceToLower goes through the source slice and lowers all values. func SliceToLower(s []string) []string { if s == nil { diff --git a/helpers/path.go b/helpers/path.go index c83686062..9c3c2ba15 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -178,6 +178,29 @@ func GetStaticDirPath() string { return AbsPathify(viper.GetString("StaticDir")) } +// GetThemeStaticDirPath returns the theme's static dir path if theme is set. +// If theme is set and the static dir doesn't exist, an error is returned. +func GetThemeStaticDirPath() (string, error) { + return getThemeDirPath("static") +} + +// GetThemeStaticDirPath returns the theme's data dir path if theme is set. +// If theme is set and the data dir doesn't exist, an error is returned. +func GetThemeDataDirPath() (string, error) { + return getThemeDirPath("data") +} + +func getThemeDirPath(path string) (string, error) { + var themeDir string + if ThemeSet() { + themeDir = AbsPathify("themes/"+viper.GetString("theme")) + FilePathSeparator + path + if _, err := os.Stat(themeDir); os.IsNotExist(err) { + return "", fmt.Errorf("Unable to find %s directory for theme %s in %s", path, viper.GetString("theme"), themeDir) + } + } + return themeDir, nil +} + func GetThemesDirPath() string { return AbsPathify(filepath.Join("themes", viper.GetString("theme"), "static")) } diff --git a/hugolib/site.go b/hugolib/site.go index c95b48611..bf9d6812e 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -267,42 +267,46 @@ func (s *Site) addTemplate(name, data string) error { return s.Tmpl.AddTemplate(name, data) } -func (s *Site) loadData(fs source.Input) (err error) { +func (s *Site) loadData(sources []source.Input) (err error) { s.Data = make(map[string]interface{}) var current map[string]interface{} - - for _, r := range fs.Files() { - // Crawl in data tree to insert data - current = s.Data - for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) { - if key != "" { - if _, ok := current[key]; !ok { - current[key] = make(map[string]interface{}) + for _, currentSource := range sources { + for _, r := range currentSource.Files() { + // Crawl in data tree to insert data + current = s.Data + for _, key := range strings.Split(r.Dir(), helpers.FilePathSeparator) { + if key != "" { + if _, ok := current[key]; !ok { + current[key] = make(map[string]interface{}) + } + current = current[key].(map[string]interface{}) } - current = current[key].(map[string]interface{}) } - } - data, err := readData(r) - if err != nil { - return fmt.Errorf("Failed to read data from %s: %s", filepath.Join(r.Path(), r.LogicalName()), err) - } + data, err := readData(r) + if err != nil { + return fmt.Errorf("Failed to read data from %s: %s", filepath.Join(r.Path(), r.LogicalName()), err) + } - // Copy content from current to data when needed - if _, ok := current[r.BaseFileName()]; ok { - data := data.(map[string]interface{}) + // Copy content from current to data when needed + if _, ok := current[r.BaseFileName()]; ok { + data := data.(map[string]interface{}) - for key, value := range current[r.BaseFileName()].(map[string]interface{}) { - if _, override := data[key]; override { - // filepath.Walk walks the files in lexical order, '/' comes before '.' - jww.ERROR.Printf("Data for key '%s' in path '%s' is overridden in subfolder", key, r.Path()) + for key, value := range current[r.BaseFileName()].(map[string]interface{}) { + if _, override := data[key]; override { + // filepath.Walk walks the files in lexical order, '/' comes before '.' + // this warning could happen if + // 1. A theme uses the same key; the main data folder wins + // 2. A sub folder uses the same key: the sub folder wins + jww.WARN.Printf("Data for key '%s' in path '%s' is overridden in subfolder", key, r.Path()) + } + data[key] = value } - data[key] = value } - } - // Insert data - current[r.BaseFileName()] = data + // Insert data + current[r.BaseFileName()] = data + } } return @@ -329,7 +333,17 @@ func (s *Site) Process() (err error) { s.Tmpl.PrintErrors() s.timerStep("initialize & template prep") - if err = s.loadData(&source.Filesystem{Base: s.absDataDir()}); err != nil { + dataSources := make([]source.Input, 0, 2) + + dataSources = append(dataSources, &source.Filesystem{Base: s.absDataDir()}) + + // have to be last - duplicate keys in earlier entries will win + themeStaticDir, err := helpers.GetThemeDataDirPath() + if err == nil { + dataSources = append(dataSources, &source.Filesystem{Base: themeStaticDir}) + } + + if err = s.loadData(dataSources); err != nil { return } s.timerStep("load data") diff --git a/hugolib/site_test.go b/hugolib/site_test.go index 04af0e61e..47f6b789b 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -760,7 +760,7 @@ func TestDataDirJson(t *testing.T) { t.Fatalf("Error %s", err) } - doTestDataDir(t, expected, sources) + doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: sources}}) } func TestDataDirToml(t *testing.T) { @@ -774,7 +774,7 @@ func TestDataDirToml(t *testing.T) { t.Fatalf("Error %s", err) } - doTestDataDir(t, expected, sources) + doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: sources}}) } func TestDataDirYamlWithOverridenValue(t *testing.T) { @@ -789,7 +789,24 @@ func TestDataDirYamlWithOverridenValue(t *testing.T) { expected := map[string]interface{}{"a": map[string]interface{}{"a": 1}, "test": map[string]interface{}{"v1": map[string]interface{}{"v1-2": 2}, "v2": map[string]interface{}{"v2": []interface{}{2, 3}}}} - doTestDataDir(t, expected, sources) + doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: sources}}) +} + +// issue 892 +func TestDataDirMultipleSources(t *testing.T) { + s1 := []source.ByteSource{ + {filepath.FromSlash("test/first.toml"), []byte("[foo]\nbar = 1")}, + } + + s2 := []source.ByteSource{ + {filepath.FromSlash("test/first.toml"), []byte("[foo]\nbar = 2")}, + {filepath.FromSlash("test/second.toml"), []byte("[foo]\ntender = 2")}, + } + + expected := map[string]interface{}{"a": map[string]interface{}{"a": 1}} + + doTestDataDir(t, expected, []source.Input{&source.InMemorySource{ByteSource: s1}, &source.InMemorySource{ByteSource: s2}}) + } func TestDataDirUnknownFormat(t *testing.T) { @@ -797,15 +814,15 @@ func TestDataDirUnknownFormat(t *testing.T) { {filepath.FromSlash("test.roml"), []byte("boo")}, } s := &Site{} - err := s.loadData(&source.InMemorySource{ByteSource: sources}) + err := s.loadData([]source.Input{&source.InMemorySource{ByteSource: sources}}) if err == nil { t.Fatalf("Should return an error") } } -func doTestDataDir(t *testing.T, expected interface{}, sources []source.ByteSource) { +func doTestDataDir(t *testing.T, expected interface{}, sources []source.Input) { s := &Site{} - err := s.loadData(&source.InMemorySource{ByteSource: sources}) + err := s.loadData(sources) if err != nil { t.Fatalf("Error loading data: %s", err) }