Read/reread individual source content files

next is incremental conversion

commands/hugo.go

@@ -594,7 +594,7 @@ func NewWatcher(port int) error {
 
 				for _, ev := range evs {
 					ext := filepath.Ext(ev.Name)
-					istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___")|| strings.HasSuffix(ext, "jb_bak___")
+					istemp := strings.HasSuffix(ext, "~") || (ext == ".swp") || (ext == ".swx") || (ext == ".tmp") || strings.HasPrefix(ext, ".goutputstream") || strings.HasSuffix(ext, "jb_old___") || strings.HasSuffix(ext, "jb_bak___")
 					if istemp {
 						continue
 					}

docs/content/meta/roadmap.md

@@ -19,7 +19,6 @@ In no particular order, here is what we are working on:
  * Import from other website systems
     * from Drupal (See https://bitbucket.org/rickb777/drupal2hugo by Rick Beton (@rickb777))
     * from WordPress (See [#100][], especially https://github.com/SchumacherFM/wordpress-to-hugo-exporter by Cyrill Schumacher (@SchumacherFM), but volunteers are needed to make it work with latest versions of WordPress.)
-    * from Jekyll (See [#101][])
  * An interactive web based editor (See http://discuss.gohugo.io/t/web-based-editor/155)
  * Additional [themes](https://github.com/spf13/hugoThemes) (always on-going, contributions welcome!)
  * Dynamic image resizing via shortcodes

hugolib/handler_page.go

@@ -32,6 +32,7 @@ type basicPageHandler Handle
 
 func (b basicPageHandler) Read(f *source.File, s *Site) HandledResult {
 	page, err := NewPage(f.Path())
+
 	if err != nil {
 		return HandledResult{file: f, err: err}
 	}

hugolib/page.go

@@ -61,7 +61,6 @@ type Page struct {
 	PublishDate         time.Time
 	Tmpl                tpl.Template
 	Markup              string
-
 	extension           string
 	contentType         string
 	renderable          bool
@@ -77,13 +76,13 @@ type Page struct {
 	plainSecondaryInit  sync.Once
 	renderingConfig     *helpers.Blackfriday
 	renderingConfigInit sync.Once
+	pageMenus           PageMenus
+	pageMenusInit       sync.Once
+	isCJKLanguage       bool
 	PageMeta
 	Source
 	Position `json:"-"`
 	Node
-	pageMenus     PageMenus
-	pageMenusInit sync.Once
-	isCJKLanguage bool
 }
 
 type Source struct {
@@ -106,6 +105,42 @@ type Position struct {
 }
 
 type Pages []*Page
+//
+//func (ps Pages) Replace(page *Page) {
+//	if i := ps.FindPagePos(page); i >= 0 {
+//		ps[i] = page
+//	}
+//}
+
+//func (ps Pages) FindPageByFilePath(inPath string) *Page {
+//	for _, x := range ps {
+//		if x.Source.LogicalName() == inPath {
+//			return x
+//		}
+//	}
+//	return nil
+//}
+
+// FindPagePos Given a page, it will find the position in Pages
+// will return -1 if not found
+func (ps Pages) FindPagePos(page *Page) int {
+	for i, x := range ps {
+		if x.Source.LogicalName() == page.Source.LogicalName() {
+			return i
+		}
+	}
+	return -1
+}
+
+// FindPage Given a page, it will return the page in Pages
+// will return nil if not found
+//func (ps Pages) FindPage(page *Page) *Page {
+//	if i := ps.FindPagePos(page); i >= 0 {
+//		return ps[i]
+//	}
+//
+//	return nil
+//}
 
 func (p *Page) Plain() string {
 	p.initPlain()

hugolib/site.go

@@ -451,7 +451,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
 		}
 	}
 
-
 	if len(tmplChanged) > 0 {
 		s.prepTemplates()
 		s.Tmpl.PrintErrors()
@@ -462,10 +461,41 @@ func (s *Site) ReBuild(changed map[string]bool) error {
 		s.ReadDataFromSourceFS()
 	}
 
-	if len (sourceChanged) > 0 {
+	if len(sourceChanged) > 0 {
-		if err = s.CreatePages(); err != nil {
+
-			return err
+		results := make(chan HandledResult)
+		filechan := make(chan *source.File)
+		errs := make(chan error)
+		wg := &sync.WaitGroup{}
+
+		wg.Add(2)
+		for i := 0; i < 2; i++ {
+			go sourceReader(s, filechan, results, wg)
 		}
+
+		go incrementalReadCollator(s, results, errs)
+
+		for _, x := range sourceChanged {
+			file, err := s.ReReadFile(x)
+			if err != nil {
+				errs <- err
+			}
+
+			filechan <- file
+		}
+
+		close(filechan)
+		wg.Wait()
+		close(results)
+
+		s.timerStep("read pages from source")
+
+		//renderErrs := <-s.ConvertSource()
+		s.timerStep("convert source")
+		// TODO(spf13) port this
+
+		fmt.Errorf("%s", errs)
+
 		s.setupPrevNext()
 		if err = s.BuildSiteMeta(); err != nil {
 			return err
@@ -497,7 +527,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
 	return nil
 }
 
-
 func (s *Site) Analyze() error {
 	if err := s.Process(); err != nil {
 		return err
@@ -764,6 +793,47 @@ type pageResult struct {
 	err  error
 }
 
+// ReReadFile resets file to be read from disk again
+func (s *Site) ReReadFile(absFilePath string) (*source.File, error) {
+	fmt.Println("rereading", absFilePath)
+	var file *source.File
+
+	reader, err := source.NewLazyFileReader(absFilePath)
+	if err != nil {
+		return nil, err
+	}
+	fmt.Println(s.absDataDir())
+
+	file, err = source.NewFileFromAbs(s.absContentDir(), absFilePath, reader)
+
+	fmt.Println("file created", file.Path())
+
+	if err != nil {
+		return nil, err
+	}
+
+	// maybe none of this rest needs to be here.
+	// replaced := false
+
+	// fmt.Println(len(s.Files))
+
+	// for i, x := range s.Files {
+	// 	fmt.Println(x)
+	// 	fmt.Println("*** COMPARING:")
+	// 	fmt.Println("   ", x.LogicalName())
+	// 	fmt.Println("   ", absFilePath)
+	// 	if x.LogicalName() == absFilePath {
+	// 		s.Files[i] = file
+	// 		replaced = true
+	// 	}
+	// }
+
+	// if !replaced {
+	// 	s.Files = append(s.Files, file)
+	// }
+	return file, nil
+}
+
 func (s *Site) ReadPagesFromSource() chan error {
 	if s.Source == nil {
 		panic(fmt.Sprintf("s.Source not set %s", s.absContentDir()))
@@ -856,13 +926,18 @@ func (s *Site) CreatePages() error {
 func sourceReader(s *Site, files <-chan *source.File, results chan<- HandledResult, wg *sync.WaitGroup) {
 	defer wg.Done()
 	for file := range files {
+		fmt.Println("reading", file.Path())
+		readSourceFile(s, file, results)
+	}
+}
+
+func readSourceFile(s *Site, file *source.File, results chan<- HandledResult) {
 	h := NewMetaHandler(file.Extension())
 	if h != nil {
 		h.Read(file, s, results)
 	} else {
 		jww.ERROR.Println("Unsupported File Type", file.Path())
 	}
-	}
 }
 
 func pageConverter(s *Site, pages <-chan *Page, results HandleResults, wg *sync.WaitGroup) {
@@ -905,6 +980,65 @@ func converterCollator(s *Site, results <-chan HandledResult, errs chan<- error)
 	errs <- fmt.Errorf("Errors rendering pages: %s", strings.Join(errMsgs, "\n"))
 }
 
+func (s *Site) AddPage(page *Page) {
+	if page.ShouldBuild() {
+		s.Pages = append(s.Pages, page)
+	}
+
+	if page.IsDraft() {
+		s.draftCount++
+	}
+
+	if page.IsFuture() {
+		s.futureCount++
+	}
+}
+
+func (s *Site) RemovePage(page *Page) {
+	if i := s.Pages.FindPagePos(page); i >= 0 {
+		if page.IsDraft() {
+			s.draftCount--
+		}
+
+		if page.IsFuture() {
+			s.futureCount--
+		}
+
+		s.Pages = append(s.Pages[:i], s.Pages[i+1:]...)
+	}
+}
+
+func (s *Site) ReplacePage(page *Page) {
+	// will find existing page that matches filepath and remove it
+	s.RemovePage(page)
+	s.AddPage(page)
+}
+
+func incrementalReadCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
+	errMsgs := []string{}
+	for r := range results {
+		if r.err != nil {
+			errMsgs = append(errMsgs, r.Error())
+			continue
+		}
+
+		// !page == file
+		if r.page == nil {
+			// TODO(spf13): Make this incremental as well
+			s.Files = append(s.Files, r.file)
+		} else {
+			s.ReplacePage(r.page)
+		}
+	}
+
+	s.Pages.Sort()
+	if len(errMsgs) == 0 {
+		errs <- nil
+		return
+	}
+	errs <- fmt.Errorf("Errors reading pages: %s", strings.Join(errMsgs, "\n"))
+}
+
 func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
 	errMsgs := []string{}
 	for r := range results {
@@ -917,17 +1051,7 @@ func readCollator(s *Site, results <-chan HandledResult, errs chan<- error) {
 		if r.page == nil {
 			s.Files = append(s.Files, r.file)
 		} else {
-			if r.page.ShouldBuild() {
+			s.AddPage(r.page)
-				s.Pages = append(s.Pages, r.page)
-			}
-
-			if r.page.IsDraft() {
-				s.draftCount++
-			}
-
-			if r.page.IsFuture() {
-				s.futureCount++
-			}
 		}
 	}
 

source/file.go

@@ -14,14 +14,16 @@
 package source
 
 import (
-	"github.com/spf13/hugo/helpers"
 	"io"
 	"path/filepath"
 	"strings"
+
+	"github.com/spf13/hugo/helpers"
 )
 
+// All paths are relative from the source directory base
 type File struct {
-	relpath     string // Original Full Path eg. /Users/Home/Hugo/foo.txt
+	relpath     string // Original Full Path eg. content/foo.txt
 	logicalName string // foo.txt
 	Contents    io.Reader
 	section     string // The first directory
@@ -30,6 +32,7 @@ type File struct {
 	uniqueID    string // MD5 of the filename
 }
 
+// UniqueID: MD5 of the filename
 func (f *File) UniqueID() string {
 	return f.uniqueID
 }
@@ -42,15 +45,17 @@ func (f *File) Bytes() []byte {
 	return helpers.ReaderToBytes(f.Contents)
 }
 
-// Filename without extension
+// BaseFileName Filename without extension
 func (f *File) BaseFileName() string {
 	return helpers.Filename(f.LogicalName())
 }
 
+// Section The first directory
 func (f *File) Section() string {
 	return f.section
 }
 
+// LogicalName The filename and extension of the file
 func (f *File) LogicalName() string {
 	return f.logicalName
 }
@@ -71,6 +76,7 @@ func (f *File) Ext() string {
 	return f.Extension()
 }
 
+// Path the relative path including file name and extension from the base of the source directory
 func (f *File) Path() string {
 	return f.relpath
 }

source/filesystem.go

@@ -14,13 +14,14 @@
 package source
 
 import (
-	"github.com/spf13/viper"
 	"io"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/spf13/viper"
+
 	"github.com/spf13/hugo/helpers"
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -59,14 +60,11 @@ func (f *Filesystem) Files() []*File {
 	return f.files
 }
 
+// add populates a file in the Filesystem.files
 func (f *Filesystem) add(name string, reader io.Reader) (err error) {
 	var file *File
 
-	//if f.Base == "" {
-	//file = NewFileWithContents(name, reader)
-	//} else {
 	file, err = NewFileFromAbs(f.Base, name, reader)
-	//}
 
 	if err == nil {
 		f.files = append(f.files, file)
@@ -79,50 +77,59 @@ func (f *Filesystem) getRelativePath(name string) (final string, err error) {
 }
 
 func (f *Filesystem) captureFiles() {
-
 	walker := func(filePath string, fi os.FileInfo, err error) error {
 		if err != nil {
 			return nil
 		}
 
-		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+		b, err := f.shouldRead(filePath, fi)
-			link, err := filepath.EvalSymlinks(filePath)
 		if err != nil {
-				jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
+			return err
-				return nil
-			}
-			linkfi, err := os.Stat(link)
-			if err != nil {
-				jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
-				return nil
-			}
-			if !linkfi.Mode().IsRegular() {
-				jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
-			}
-			return nil
-		}
-
-		if fi.IsDir() {
-			if f.avoid(filePath) || isNonProcessablePath(filePath) {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
-		if isNonProcessablePath(filePath) {
-			return nil
 		}
+		if b {
 			rd, err := NewLazyFileReader(filePath)
 			if err != nil {
 				return err
 			}
 			f.add(filePath, rd)
-		return nil
+		}
+		return err
 	}
 
 	filepath.Walk(f.Base, walker)
 }
 
+func (f *Filesystem) shouldRead(filePath string, fi os.FileInfo) (bool, error) {
+	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+		link, err := filepath.EvalSymlinks(filePath)
+		if err != nil {
+			jww.ERROR.Printf("Cannot read symbolic link '%s', error was: %s", filePath, err)
+			return false, nil
+		}
+		linkfi, err := os.Stat(link)
+		if err != nil {
+			jww.ERROR.Printf("Cannot stat '%s', error was: %s", link, err)
+			return false, nil
+		}
+		if !linkfi.Mode().IsRegular() {
+			jww.ERROR.Printf("Symbolic links for directories not supported, skipping '%s'", filePath)
+		}
+		return false, nil
+	}
+
+	if fi.IsDir() {
+		if f.avoid(filePath) || isNonProcessablePath(filePath) {
+			return false, filepath.SkipDir
+		}
+		return false, nil
+	}
+
+	if isNonProcessablePath(filePath) {
+		return false, nil
+	}
+	return true, nil
+}
+
 func (f *Filesystem) avoid(filePath string) bool {
 	for _, avoid := range f.AvoidPaths {
 		if avoid == filePath {