Handle remove & rename source operations incrementally

commands/hugo.go

@@ -546,9 +546,9 @@ func buildSite(watching ...bool) (err error) {
 	return nil
 }
 
-func rebuildSite(changes map[string]bool) error {
+func rebuildSite(events []fsnotify.Event) error {
 	startTime := time.Now()
-	err := mainSite.ReBuild(changes)
+	err := mainSite.ReBuild(events)
 	if err != nil {
 		return err
 	}
@@ -585,12 +585,10 @@ func NewWatcher(port int) error {
 		for {
 			select {
 			case evs := <-watcher.Events:
-				jww.INFO.Println("File System Event:", evs)
+				jww.INFO.Println("Recieved System Events:", evs)
 
-				staticChanged := false
+				staticEvents := []fsnotify.Event{} //ev make(map[string]bool)
-				dynamicChanged := false
+				dynamicEvents := []fsnotify.Event{} //make(map[string]bool)
-				staticFilesChanged := make(map[string]bool)
-				dynamicFilesChanged := make(map[string]bool)
 
 				for _, ev := range evs {
 					ext := filepath.Ext(ev.Name)
@@ -598,10 +596,6 @@ func NewWatcher(port int) error {
 					if istemp {
 						continue
 					}
-					// renames are always followed with Create/Modify
-					if ev.Op&fsnotify.Rename == fsnotify.Rename {
-						continue
-					}
 
 					// Write and rename operations are often followed by CHMOD.
 					// There may be valid use cases for rebuilding the site on CHMOD,
@@ -615,27 +609,24 @@ func NewWatcher(port int) error {
 						continue
 					}
 
-					isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath()))
-					staticChanged = staticChanged || isstatic
-					dynamicChanged = dynamicChanged || !isstatic
-
-					if isstatic {
-						if staticPath, err := helpers.MakeStaticPathRelative(ev.Name); err == nil {
-							staticFilesChanged[staticPath] = true
-						}
-					} else {
-						dynamicFilesChanged[ev.Name] = true
-					}
-
 					// add new directory to watch list
 					if s, err := os.Stat(ev.Name); err == nil && s.Mode().IsDir() {
 						if ev.Op&fsnotify.Create == fsnotify.Create {
 							watcher.Add(ev.Name)
 						}
 					}
+
+					isstatic := strings.HasPrefix(ev.Name, helpers.GetStaticDirPath()) || (len(helpers.GetThemesDirPath()) > 0 && strings.HasPrefix(ev.Name, helpers.GetThemesDirPath()))
+
+					if isstatic {
+						staticEvents = append(staticEvents, ev)
+//						}
+					} else {
+						dynamicEvents = append(dynamicEvents, ev)
+					}
 				}
 
-				if staticChanged {
+				if len(staticEvents) > 0 {
 					jww.FEEDBACK.Printf("Static file changed, syncing\n")
 					if viper.GetBool("ForceSyncStatic") {
 						jww.FEEDBACK.Printf("Syncing all static files\n")
@@ -645,7 +636,6 @@ func NewWatcher(port int) error {
 							utils.StopOnErr(err, fmt.Sprintf("Error copying static files to %s", helpers.AbsPathify(viper.GetString("PublishDir"))))
 						}
 					} else {
-
 						syncer := fsync.NewSyncer()
 						syncer.NoTimes = viper.GetBool("notimes")
 						syncer.SrcFs = hugofs.SourceFs
@@ -660,24 +650,40 @@ func NewWatcher(port int) error {
 						staticDir := helpers.GetStaticDirPath()
 						themeStaticDir := helpers.GetThemesDirPath()
 
-						jww.FEEDBACK.Printf("StaticDir '%s'\nThemeStaticDir '%s'\n", staticDir, themeStaticDir)
+						jww.FEEDBACK.Printf("Syncing from: \n \tStaticDir: '%s'\n\tThemeStaticDir: '%s'\n", staticDir, themeStaticDir)
 
-						for path := range staticFilesChanged {
+						for _, ev := range staticEvents {
+							fmt.Println(ev)
+							fromPath := ev.Name
 							var publishPath string
 
-							if strings.HasPrefix(path, staticDir) {
+							// If we are here we already know the event took place in a static dir
-								publishPath = filepath.Join(publishDir, strings.TrimPrefix(path, staticDir))
+							relPath, err := helpers.MakeStaticPathRelative(fromPath)
-							} else if strings.HasPrefix(path, themeStaticDir) {
-								publishPath = filepath.Join(publishDir, strings.TrimPrefix(path, themeStaticDir))
-							}
-							jww.FEEDBACK.Printf("Syncing file '%s'", path)
-
-							if _, err := os.Stat(path); err == nil {
-								jww.INFO.Println("syncing from ", path, " to ", publishPath)
-								err := syncer.Sync(publishPath, path)
 							if err != nil {
-									jww.FEEDBACK.Printf("Error on syncing file '%s'\n", path)
+								fmt.Println(err)
+								continue
 							}
+
+							if strings.HasPrefix(fromPath, staticDir) {
+								publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, staticDir))
+							} else if strings.HasPrefix(relPath, themeStaticDir) {
+								publishPath = filepath.Join(publishDir, strings.TrimPrefix(fromPath, themeStaticDir))
+							}
+							jww.FEEDBACK.Println("Syncing file", relPath)
+
+							// Due to our approach of layering many directories onto one we can't accurately
+							// remove file not in one of the source directories.
+							// If a file is in the local static dir and also in the theme static dir and we remove
+							// it from one of those locations we expect it to still exist in the destination
+
+							 // if remove or rename ignore
+							if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
+								continue
+							}
+
+							jww.INFO.Println("syncing from ", fromPath, " to ", publishPath)
+							if er := syncer.Sync(publishPath, fromPath); er != nil {
+								jww.ERROR.Printf("Error on syncing file '%s'\n %s\n", relPath, er)
 							}
 						}
 					}
@@ -686,8 +692,9 @@ func NewWatcher(port int) error {
 						// Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized
 
 						// force refresh when more than one file
-						if len(staticFilesChanged) == 1 {
+						if len(staticEvents) == 1 {
-							for path := range staticFilesChanged {
+							for _, ev := range staticEvents {
+								path, _ := helpers.MakeStaticPathRelative(ev.Name)
 								livereload.RefreshPath(path)
 							}
 
@@ -697,14 +704,12 @@ func NewWatcher(port int) error {
 					}
 				}
 
-				if dynamicChanged {
+				if len(dynamicEvents) >0 {
 					fmt.Print("\nChange detected, rebuilding site\n")
 					const layout = "2006-01-02 15:04 -0700"
 					fmt.Println(time.Now().Format(layout))
-					//TODO here
 
-					//	utils.CheckErr(buildSite(true))
+					rebuildSite(dynamicEvents)
-					rebuildSite(dynamicFilesChanged)
 
 					if !BuildWatch && !viper.GetBool("DisableLiveReload") {
 						// Will block forever trying to write to a channel that nobody is reading if livereload isn't initalized

hugolib/page.go

@@ -112,20 +112,20 @@ type Pages []*Page
 //	}
 //}
 
-//func (ps Pages) FindPageByFilePath(inPath string) *Page {
+func (ps Pages) FindPagePosByFilePath(inPath string) int {
-//	for _, x := range ps {
+	for i, x := range ps {
-//		if x.Source.LogicalName() == inPath {
+		if x.Source.Path() == inPath {
-//			return x
+			return i
-//		}
+		}
-//	}
+	}
-//	return nil
+	return -1
-//}
+}
 
 // FindPagePos Given a page, it will find the position in Pages
 // will return -1 if not found
 func (ps Pages) FindPagePos(page *Page) int {
 	for i, x := range ps {
-		if x.Source.LogicalName() == page.Source.LogicalName() {
+		if x.Source.Path() == page.Source.Path() {
 			return i
 		}
 	}

hugolib/site.go

@@ -43,6 +43,7 @@ import (
 	jww "github.com/spf13/jwalterweatherman"
 	"github.com/spf13/nitro"
 	"github.com/spf13/viper"
+	"gopkg.in/fsnotify.v1"
 )
 
 var _ = transform.AbsURL
@@ -426,28 +427,29 @@ func (s *Site) Build() (err error) {
 	return nil
 }
 
-func (s *Site) ReBuild(changed map[string]bool) error {
+func (s *Site) ReBuild(events []fsnotify.Event) error {
 	s.timerStep("initialize rebuild")
 	// First we need to determine what changed
 
-	sourceChanged := []string{}
+	sourceChanged := []fsnotify.Event{}
-	tmplChanged := []string{}
+	tmplChanged := []fsnotify.Event{}
-	dataChanged := []string{}
+	dataChanged := []fsnotify.Event{}
+
 	var err error
 
-	for f := range changed {
+	for _, ev := range events {
 		// Need to re-read source
-		if strings.HasPrefix(f, s.absContentDir()) {
+		if strings.HasPrefix(ev.Name, s.absContentDir()) {
-			fmt.Println("Source changed", f)
+			fmt.Println("Source changed", ev)
-			sourceChanged = append(sourceChanged, f)
+			sourceChanged = append(sourceChanged, ev)
 		}
-		if strings.HasPrefix(f, s.absLayoutDir()) || strings.HasPrefix(f, s.absThemeDir()) {
+		if strings.HasPrefix(ev.Name, s.absLayoutDir()) || strings.HasPrefix(ev.Name, s.absThemeDir()) {
-			fmt.Println("Template changed", f)
+			fmt.Println("Template changed", ev)
-			tmplChanged = append(tmplChanged, f)
+			tmplChanged = append(tmplChanged, ev)
 		}
-		if strings.HasPrefix(f, s.absDataDir()) {
+		if strings.HasPrefix(ev.Name, s.absDataDir()) {
-			fmt.Println("Data changed", f)
+			fmt.Println("Data changed", ev)
-			dataChanged = append(dataChanged, f)
+			dataChanged = append(dataChanged,ev)
 		}
 	}
 
@@ -497,8 +499,15 @@ func (s *Site) ReBuild(changed map[string]bool) error {
 		go incrementalReadCollator(s, readResults, pageChan, fileConvChan, coordinator, errs)
 		go converterCollator(s, convertResults, errs)
 
-		for _, x := range sourceChanged {
+		for _, ev := range sourceChanged {
-			file, err := s.ReReadFile(x)
+			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
+				//remove the file & a create will follow
+				path, _ := helpers.GetRelativePath(ev.Name, s.absContentDir())
+				s.RemovePageByPath(path)
+				continue
+			}
+
+			file, err := s.ReReadFile(ev.Name)
 			if err != nil {
 				errs <- err
 			}
@@ -540,7 +549,6 @@ func (s *Site) ReBuild(changed map[string]bool) error {
 	if err = s.Render(); err != nil {
 		// Better reporting when the template is missing (commit 2bbecc7b)
 		jww.ERROR.Printf("Error rendering site: %s", err)
-
 		jww.ERROR.Printf("Available templates:")
 		var keys []string
 		for _, template := range s.Tmpl.Templates() {
@@ -1005,6 +1013,23 @@ func (s *Site) AddPage(page *Page) {
 	}
 }
 
+
+func (s *Site) RemovePageByPath(path string) {
+	if i := s.Pages.FindPagePosByFilePath(path); i >= 0 {
+		page := s.Pages[i]
+
+		if page.IsDraft() {
+			s.draftCount--
+		}
+
+		if page.IsFuture() {
+			s.futureCount--
+		}
+
+		s.Pages = append(s.Pages[:i], s.Pages[i+1:]...)
+	}
+}
+
 func (s *Site) RemovePage(page *Page) {
 	if i := s.Pages.FindPagePos(page); i >= 0 {
 		if page.IsDraft() {