Introduce source.Filesystem

This provides an abstraction over how files are processed by Hugo.  This
allows for alternatives like CMS systems or Dropbox, etc.

hugolib/planner.go

@@ -6,19 +6,19 @@ import (
 )
 
 func (s *Site) ShowPlan(out io.Writer) (err error) {
-	if len(s.Files) <= 0 {
+	if s.Source == nil || len(s.Source.Files()) <= 0 {
 		fmt.Fprintf(out, "No source files provided.\n")
 	}
 
-	for _, file := range s.Files {
+	for _, p := range s.Pages {
-		fmt.Fprintf(out, "%s\n", file)
+		fmt.Fprintf(out, "%s\n", p.FileName)
 		fmt.Fprintf(out, " canonical => ")
 		if s.Target == nil {
 			fmt.Fprintf(out, "%s\n", "!no target specified!")
 			continue
 		}
 
-		trns, err := s.Target.Translate(file)
+		trns, err := s.Target.Translate(p.OutFile)
 		if err != nil {
 			return err
 		}

hugolib/site.go

@@ -17,6 +17,7 @@ import (
 	"bitbucket.org/pkg/inflect"
 	"bytes"
 	"fmt"
+	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/target"
 	helpers "github.com/spf13/hugo/template"
 	"github.com/spf13/hugo/template/bundle"
@@ -69,7 +70,7 @@ type Site struct {
 	Pages      Pages
 	Tmpl       bundle.Template
 	Indexes    IndexList
-	Files      []string
+	Source     source.Input
 	Sections   Index
 	Info       SiteInfo
 	Shortcodes map[string]ShortcodeFunc
@@ -186,26 +187,11 @@ func (s *Site) initialize() {
 
 	staticDir := s.Config.GetAbsPath(s.Config.StaticDir + "/")
 
-	walker := func(path string, fi os.FileInfo, err error) error {
+	s.Source = &source.Filesystem{
-		if err != nil {
+		AvoidPaths: []string{staticDir},
-			return nil
+		Base:       s.absContentDir(),
-		}
-
-		if fi.IsDir() {
-			if path == staticDir {
-				return filepath.SkipDir
-			}
-			return nil
-		} else {
-			if ignoreDotFile(path) {
-				return nil
-			}
-			s.Files = append(s.Files, path)
-			return nil
-		}
 	}
 
-	filepath.Walk(s.absContentDir(), walker)
 	s.Info = SiteInfo{
 		BaseUrl: template.URL(s.Config.BaseUrl),
 		Title:   s.Config.Title,
@@ -228,10 +214,6 @@ func exists(path string) (bool, error) {
 	return false, err
 }
 
-func ignoreDotFile(path string) bool {
-	return filepath.Base(path)[0] == '.'
-}
-
 func (s *Site) absLayoutDir() string {
 	return s.Config.GetAbsPath(s.Config.LayoutDir)
 }
@@ -275,12 +257,8 @@ func (s *Site) AbsUrlify() {
 }
 
 func (s *Site) CreatePages() (err error) {
-	for _, fileName := range s.Files {
+	for _, file := range s.Source.Files() {
-		f, err := os.Open(fileName)	
+		page, err := ReadFrom(file.Contents, file.Name)
-		if err != nil {
-			return err
-		}
-		page, err := ReadFrom(f, fileName)
 		if err != nil {
 			return err
 		}

hugolib/site_show_plan_test.go

@@ -2,57 +2,80 @@ package hugolib
 
 import (
 	"bytes"
-	"testing"
+	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/target"
+	"testing"
 )
 
-func checkShowPlanExpected(t *testing.T, expected, got string) {
+var fakeSource = []struct {
+	name    string
+	content []byte
+}{
+	{"foo/bar/file.md", []byte(SIMPLE_PAGE)},
+}
+
+type inMemorySource struct {
+}
+
+func (i inMemorySource) Files() (files []*source.File) {
+	files = make([]*source.File, len(fakeSource))
+	for i, fake := range fakeSource {
+		files[i] = &source.File{
+			Name:     fake.name,
+			Contents: bytes.NewReader(fake.content),
+		}
+	}
+	return
+}
+
+func checkShowPlanExpected(t *testing.T, s *Site, expected string) {
+	out := new(bytes.Buffer)
+	if err := s.ShowPlan(out); err != nil {
+		t.Fatalf("ShowPlan unexpectedly returned an error: %s", err)
+	}
+	got := out.String()
 	if got != expected {
 		t.Errorf("ShowPlan expected:\n%q\ngot\n%q", expected, got)
 	}
 }
 
 func TestDegenerateNoFiles(t *testing.T) {
-	s := new(Site)
+	checkShowPlanExpected(t, new(Site), "No source files provided.\n")
-	out := new(bytes.Buffer)
-	if err := s.ShowPlan(out); err != nil {
-		t.Errorf("ShowPlan unexpectedly returned an error: %s", err)
-	}
-	expected := "No source files provided.\n"
-	got := out.String()
-	checkShowPlanExpected(t, expected, got)
 }
 
 func TestDegenerateNoTarget(t *testing.T) {
-	s := new(Site)
+	s := &Site{Source: new(inMemorySource)}
-	s.Files = append(s.Files, "foo/bar/file.md")
+	must(s.CreatePages())
-	out := new(bytes.Buffer)
-	if err := s.ShowPlan(out); err != nil {
-		t.Errorf("ShowPlan unexpectedly returned an error: %s", err)
-	}
-
 	expected := "foo/bar/file.md\n canonical => !no target specified!\n"
-	checkShowPlanExpected(t, expected, out.String())
+	checkShowPlanExpected(t, s, expected)
 }
 
 func TestFileTarget(t *testing.T) {
-	s := &Site{Target: new(target.Filesystem)}
+	s := &Site{
-	s.Files = append(s.Files, "foo/bar/file.md")
+		Source: new(inMemorySource),
-	out := new(bytes.Buffer)
+		Target: new(target.Filesystem),
-	s.ShowPlan(out)
+	}
-
+	must(s.CreatePages())
-	expected := "foo/bar/file.md\n canonical => foo/bar/file/index.html\n"
+	checkShowPlanExpected(t, s, "foo/bar/file.md\n canonical => foo/bar/file/index.html\n")
-	checkShowPlanExpected(t, expected, out.String())
 }
 
 func TestFileTargetUgly(t *testing.T) {
-	s := &Site{Target: &target.Filesystem{UglyUrls: true}}
+	s := &Site{
-	s.Files = append(s.Files, "foo/bar/file.md")
+		Target: &target.Filesystem{UglyUrls: true},
-	out := new(bytes.Buffer)
+		Source: new(inMemorySource),
-	s.ShowPlan(out)
+	}
-
+	s.CreatePages()
 	expected := "foo/bar/file.md\n canonical => foo/bar/file.html\n"
-	checkShowPlanExpected(t, expected, out.String())
+	checkShowPlanExpected(t, s, expected)
 }
 
+func TestFileTargetPublishDir(t *testing.T) {
+	s := &Site{
+		Target: &target.Filesystem{PublishDir: "../public"},
+		Source: new(inMemorySource),
+	}
 
+	must(s.CreatePages())
+	expected := "foo/bar/file.md\n canonical => ../public/foo/bar/file/index.html\n"
+	checkShowPlanExpected(t, s, expected)
+}

hugolib/site_url_test.go

@@ -49,8 +49,8 @@ func TestPageCount(t *testing.T) {
 	s := &Site{Target: target}
 	s.prepTemplates()
 	must(s.addTemplate("indexes/blue.html", INDEX_TEMPLATE))
-	s.Files = append(s.Files, "blue/doc1.md")
+	//s.Files = append(s.Files, "blue/doc1.md")
-	s.Files = append(s.Files, "blue/doc2.md")
+	//s.Files = append(s.Files, "blue/doc2.md")
 	s.Pages = append(s.Pages, mustReturn(ReadFrom(strings.NewReader(SLUG_DOC_1), filepath.FromSlash("content/blue/doc1.md"))))
 	s.Pages = append(s.Pages, mustReturn(ReadFrom(strings.NewReader(SLUG_DOC_2), filepath.FromSlash("content/blue/doc2.md"))))
 

source/content_directory_test.go

@@ -1,4 +1,4 @@
-package hugolib
+package source
 
 import (
 	"testing"

source/filesystem.go

@@ -0,0 +1,72 @@
+package source
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+type Input interface {
+	Files() []*File
+}
+
+type File struct {
+	Name     string
+	Contents io.Reader
+}
+
+type Filesystem struct {
+	files      []*File
+	Base       string
+	AvoidPaths []string
+}
+
+func (f *Filesystem) Files() []*File {
+	f.captureFiles()
+	return f.files
+}
+
+func (f *Filesystem) add(name string, reader io.Reader) {
+	f.files = append(f.files, &File{Name: name, Contents: reader})
+}
+
+func (f *Filesystem) captureFiles() {
+
+	walker := func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+
+		if fi.IsDir() {
+			if f.avoid(path) {
+				return filepath.SkipDir
+			}
+			return nil
+		} else {
+			if ignoreDotFile(path) {
+				return nil
+			}
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			f.add(path, file)
+			return nil
+		}
+	}
+
+	filepath.Walk(f.Base, walker)
+}
+
+func (f *Filesystem) avoid(path string) bool {
+	for _, avoid := range f.AvoidPaths {
+		if avoid == path {
+			return true
+		}
+	}
+	return false
+}
+
+func ignoreDotFile(path string) bool {
+	return filepath.Base(path)[0] == '.'
+}

source/filesystem_test.go

@@ -0,0 +1,32 @@
+package source
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestEmptySourceFilesystem(t *testing.T) {
+	src := new(Filesystem)
+	if len(src.Files()) != 0 {
+		t.Errorf("new filesystem should contain 0 files.")
+	}
+}
+
+func TestAddFile(t *testing.T) {
+	src := new(Filesystem)
+	src.add("foobar", bytes.NewReader([]byte("aaa")))
+	if len(src.Files()) != 1 {
+		t.Errorf("Files() should return 1 file")
+	}
+
+	f := src.Files()[0]
+	if f.Name != "foobar" {
+		t.Errorf("File name should be 'foobar', got: %s", f.Name)
+	}
+
+	b := new(bytes.Buffer)
+	b.ReadFrom(f.Contents)
+	if b.String() != "aaa" {
+		t.Errorf("File contents should be 'aaa', got: %s", b.String())
+	}
+}

target/file.go

@@ -35,15 +35,16 @@ func (fs *Filesystem) Publish(path string, r io.Reader) (err error) {
 	}
 
 	path, _ = filepath.Split(translated)
-	dest := filepath.Join(fs.PublishDir, path)
+	ospath := filepath.FromSlash(path)
-	ospath := filepath.FromSlash(dest)
 
-	err = os.MkdirAll(ospath, 0764) // rwx, rw, r
+	if ospath != "" {
-	if err != nil {
+		err = os.MkdirAll(ospath, 0764) // rwx, rw, r
-		return
+		if err != nil {
+			return
+		}
 	}
 
-	file, err := os.Create(filepath.Join(fs.PublishDir, translated))
+	file, err := os.Create(translated)
 	if err != nil {
 		return
 	}
@@ -61,6 +62,9 @@ func (fs *Filesystem) Translate(src string) (dest string, err error) {
 	dir, file := path.Split(src)
 	ext := fs.extension(path.Ext(file))
 	name := filename(file)
+	if fs.PublishDir != "" {
+		dir = path.Join(fs.PublishDir, dir)
+	}
 
 	if fs.UglyUrls {
 		return path.Join(dir, fmt.Sprintf("%s%s", name, ext)), nil