deploy: Implement include/exclude filters for deploy

Fixes #6922

deploy/deploy.go

@@ -31,6 +31,7 @@ import (
 	"sync"
 
 	"github.com/dustin/go-humanize"
+	"github.com/gobwas/glob"
 	"github.com/gohugoio/hugo/config"
 	"github.com/pkg/errors"
 	"github.com/spf13/afero"
@@ -125,7 +126,11 @@ func (d *Deployer) Deploy(ctx context.Context) error {
 	}
 
 	// Load local files from the source directory.
-	local, err := walkLocal(d.localFs, d.matchers)
+	var include, exclude glob.Glob
+	if d.target != nil {
+		include, exclude = d.target.includeGlob, d.target.excludeGlob
+	}
+	local, err := walkLocal(d.localFs, d.matchers, include, exclude)
 	if err != nil {
 		return err
 	}
@@ -437,7 +442,7 @@ func (lf *localFile) MD5() []byte {
 
 // walkLocal walks the source directory and returns a flat list of files,
 // using localFile.SlashPath as the map keys.
-func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error) {
+func walkLocal(fs afero.Fs, matchers []*matcher, include, exclude glob.Glob) (map[string]*localFile, error) {
 	retval := map[string]*localFile{}
 	err := afero.Walk(fs, "", func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -461,8 +466,18 @@ func walkLocal(fs afero.Fs, matchers []*matcher) (map[string]*localFile, error)
 			path = norm.NFC.String(path)
 		}
 
-		// Find the first matching matcher (if any).
+		// Check include/exclude matchers.
 		slashpath := filepath.ToSlash(path)
+		if include != nil && !include.Match(slashpath) {
+			jww.INFO.Printf("  dropping %q due to include\n", slashpath)
+			return nil
+		}
+		if exclude != nil && exclude.Match(slashpath) {
+			jww.INFO.Printf("  dropping %q due to exclude\n", slashpath)
+			return nil
+		}
+
+		// Find the first matching matcher (if any).
 		var m *matcher
 		for _, cur := range matchers {
 			if cur.Matches(slashpath) {

deploy/deployConfig.go

@@ -17,7 +17,9 @@ import (
 	"fmt"
 	"regexp"
 
+	"github.com/gobwas/glob"
 	"github.com/gohugoio/hugo/config"
+	hglob "github.com/gohugoio/hugo/hugofs/glob"
 	"github.com/mitchellh/mapstructure"
 )
 
@@ -41,6 +43,32 @@ type target struct {
 	// GoogleCloudCDNOrigin specifies the Google Cloud project and CDN origin to
 	// invalidate when deploying this target.  It is specified as <project>/<origin>.
 	GoogleCloudCDNOrigin string
+
+	// Optional patterns of files to include/exclude for this target.
+	// Parsed using github.com/gobwas/glob.
+	Include string
+	Exclude string
+
+	// Parsed versions of Include/Exclude.
+	includeGlob glob.Glob
+	excludeGlob glob.Glob
+}
+
+func (tgt *target) parseIncludeExclude() error {
+	var err error
+	if tgt.Include != "" {
+		tgt.includeGlob, err = hglob.GetGlob(tgt.Include)
+		if err != nil {
+			return fmt.Errorf("invalid deployment.target.include %q: %v", tgt.Include, err)
+		}
+	}
+	if tgt.Exclude != "" {
+		tgt.excludeGlob, err = hglob.GetGlob(tgt.Exclude)
+		if err != nil {
+			return fmt.Errorf("invalid deployment.target.exclude %q: %v", tgt.Exclude, err)
+		}
+	}
+	return nil
 }
 
 // matcher represents configuration to be applied to files whose paths match
@@ -87,6 +115,11 @@ func decodeConfig(cfg config.Provider) (deployConfig, error) {
 	if err := mapstructure.WeakDecode(cfg.GetStringMap(deploymentConfigKey), &dcfg); err != nil {
 		return dcfg, err
 	}
+	for _, tgt := range dcfg.Targets {
+		if err := tgt.parseIncludeExclude(); err != nil {
+			return dcfg, err
+		}
+	}
 	var err error
 	for _, m := range dcfg.Matchers {
 		m.re, err = regexp.Compile(m.Pattern)

deploy/deployConfig_test.go

@@ -38,18 +38,21 @@ order = ["o1", "o2"]
 name = "name0"
 url = "url0"
 cloudfrontdistributionid = "cdn0"
+include = "*.html"
 
 # All uppercase.
 [[deployment.targets]]
 NAME = "name1"
 URL = "url1"
 CLOUDFRONTDISTRIBUTIONID = "cdn1"
+INCLUDE = "*.jpg"
 
 # Camelcase.
 [[deployment.targets]]
 name = "name2"
 url = "url2"
 cloudFrontDistributionID = "cdn2"
+exclude = "*.png"
 
 # All lowercase.
 [[deployment.matchers]]
@@ -90,11 +93,21 @@ force = true
 
 	// Targets.
 	c.Assert(len(dcfg.Targets), qt.Equals, 3)
+	wantInclude := []string{"*.html", "*.jpg", ""}
+	wantExclude := []string{"", "", "*.png"}
 	for i := 0; i < 3; i++ {
 		tgt := dcfg.Targets[i]
 		c.Assert(tgt.Name, qt.Equals, fmt.Sprintf("name%d", i))
 		c.Assert(tgt.URL, qt.Equals, fmt.Sprintf("url%d", i))
 		c.Assert(tgt.CloudFrontDistributionID, qt.Equals, fmt.Sprintf("cdn%d", i))
+		c.Assert(tgt.Include, qt.Equals, wantInclude[i])
+		if wantInclude[i] != "" {
+			c.Assert(tgt.includeGlob, qt.Not(qt.IsNil))
+		}
+		c.Assert(tgt.Exclude, qt.Equals, wantExclude[i])
+		if wantExclude[i] != "" {
+			c.Assert(tgt.excludeGlob, qt.Not(qt.IsNil))
+		}
 	}
 
 	// Matchers.

deploy/deploy_test.go

@@ -640,6 +640,86 @@ func TestMaxDeletes(t *testing.T) {
 	}
 }
 
+// TestIncludeExclude verifies that the include/exclude options for targets work.
+func TestIncludeExclude(t *testing.T) {
+	ctx := context.Background()
+
+	tests := []struct {
+		Include string
+		Exclude string
+		Want    deploySummary
+	}{
+		{
+			Want: deploySummary{NumLocal: 5, NumUploads: 5},
+		},
+		{
+			Include: "**aaa",
+			Want:    deploySummary{NumLocal: 3, NumUploads: 3},
+		},
+		{
+			Include: "**bbb",
+			Want:    deploySummary{NumLocal: 2, NumUploads: 2},
+		},
+		{
+			Include: "aaa",
+			Want:    deploySummary{NumLocal: 1, NumUploads: 1},
+		},
+		{
+			Exclude: "**aaa",
+			Want:    deploySummary{NumLocal: 2, NumUploads: 2},
+		},
+		{
+			Exclude: "**bbb",
+			Want:    deploySummary{NumLocal: 3, NumUploads: 3},
+		},
+		{
+			Exclude: "aaa",
+			Want:    deploySummary{NumLocal: 4, NumUploads: 4},
+		},
+		{
+			Include: "**aaa",
+			Exclude: "**nested**",
+			Want:    deploySummary{NumLocal: 2, NumUploads: 2},
+		},
+	}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("include %q exclude %q", test.Include, test.Exclude), func(t *testing.T) {
+			fsTests, cleanup, err := initFsTests()
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer cleanup()
+			fsTest := fsTests[1] // just do file-based test
+
+			_, err = initLocalFs(ctx, fsTest.fs)
+			if err != nil {
+				t.Fatal(err)
+			}
+			tgt := &target{
+				Include: test.Include,
+				Exclude: test.Exclude,
+			}
+			if err := tgt.parseIncludeExclude(); err != nil {
+				t.Error(err)
+			}
+			deployer := &Deployer{
+				localFs:    fsTest.fs,
+				maxDeletes: -1,
+				bucket:     fsTest.bucket,
+				target:     tgt,
+			}
+
+			// Sync remote with local.
+			if err := deployer.Deploy(ctx); err != nil {
+				t.Errorf("deploy: failed: %v", err)
+			}
+			if !cmp.Equal(deployer.summary, test.Want) {
+				t.Errorf("deploy: got %v, want %v", deployer.summary, test.Want)
+			}
+		})
+	}
+}
+
 // TestCompression verifies that gzip compression works correctly.
 // In particular, MD5 hashes must be of the compressed content.
 func TestCompression(t *testing.T) {

docs/content/en/hosting-and-deployment/hugo-deploy.md

@@ -82,8 +82,13 @@ name = "mydeployment"
 # If you are using a CloudFront CDN, deploy will invalidate the cache as needed.
 cloudFrontDistributionID = <ID>
 
-
+# Optionally, you can include or exclude specific files.
-# ... add more [[deployment.targets]] sections ...
+# See https://godoc.org/github.com/gobwas/glob#Glob for the glob pattern syntax.
+# If non-empty, the pattern is matched against the local path.
+# If exclude is non-empty, and a file's path matches it, that file is dropped.
+# If include is non-empty, and a file's path does not match it, that file is dropped.
+# include = "**.html" # would only include files with ".html" suffix
+# exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix
 
 
 # [[deployment.matchers]] configure behavior for files that match the Pattern.