cache/filecache: Split implementation and config into separate files

cache/filecache/filecache.go

@@ -17,7 +17,6 @@ import (
 	"bytes"
 	"io"
 	"io/ioutil"
-	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -28,56 +27,10 @@ import (
 
 	"github.com/gohugoio/hugo/hugolib/paths"
 
-	"github.com/pkg/errors"
-
 	"github.com/BurntSushi/locker"
-	"github.com/bep/mapstructure"
 	"github.com/spf13/afero"
 )
 
-const (
-	cachesConfigKey = "caches"
-
-	resourcesGenDir = ":resourceDir/_gen"
-)
-
-var defaultCacheConfig = cacheConfig{
-	MaxAge: -1, // Never expire
-	Dir:    ":cacheDir",
-}
-
-const (
-	cacheKeyGetJSON = "getjson"
-	cacheKeyGetCSV  = "getcsv"
-	cacheKeyImages  = "images"
-	cacheKeyAssets  = "assets"
-)
-
-var defaultCacheConfigs = map[string]cacheConfig{
-	cacheKeyGetJSON: defaultCacheConfig,
-	cacheKeyGetCSV:  defaultCacheConfig,
-	cacheKeyImages: cacheConfig{
-		MaxAge: -1,
-		Dir:    resourcesGenDir,
-	},
-	cacheKeyAssets: cacheConfig{
-		MaxAge: -1,
-		Dir:    resourcesGenDir,
-	},
-}
-
-type cachesConfig map[string]cacheConfig
-
-type cacheConfig struct {
-	// Max age of cache entries in this cache. Any items older than this will
-	// be removed and not returned from the cache.
-	// -1 means forever, 0 means cache is disabled.
-	MaxAge int
-
-	// The directory where files are stored.
-	Dir string
-}
-
 // Cache caches a set of files in a directory. This is usually a file on
 // disk, but since this is backed by an Afero file system, it can be anything.
 type Cache struct {
@@ -316,26 +269,6 @@ func (f Caches) Get(name string) *Cache {
 	return f[strings.ToLower(name)]
 }
 
-// GetJSONCache gets the file cache for getJSON.
-func (f Caches) GetJSONCache() *Cache {
-	return f[cacheKeyGetJSON]
-}
-
-// GetCSVCache gets the file cache for getCSV.
-func (f Caches) GetCSVCache() *Cache {
-	return f[cacheKeyGetCSV]
-}
-
-// ImageCache gets the file cache for processed images.
-func (f Caches) ImageCache() *Cache {
-	return f[cacheKeyImages]
-}
-
-// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
-func (f Caches) AssetsCache() *Cache {
-	return f[cacheKeyAssets]
-}
-
 // NewCachesFromPaths creates a new set of file caches from the given
 // configuration.
 func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
@@ -359,84 +292,6 @@ func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
 	return m, nil
 }
 
-func decodeConfig(p *paths.Paths) (cachesConfig, error) {
-	c := make(cachesConfig)
-	valid := make(map[string]bool)
-	// Add defaults
-	for k, v := range defaultCacheConfigs {
-		c[k] = v
-		valid[k] = true
-	}
-
-	cfg := p.Cfg
-
-	m := cfg.GetStringMap(cachesConfigKey)
-
-	_, isOsFs := p.Fs.Source.(*afero.OsFs)
-
-	for k, v := range m {
-		cc := defaultCacheConfig
-
-		if err := mapstructure.WeakDecode(v, &cc); err != nil {
-			return nil, err
-		}
-
-		if cc.Dir == "" {
-			return c, errors.New("must provide cache Dir")
-		}
-
-		name := strings.ToLower(k)
-		if !valid[name] {
-			return nil, errors.Errorf("%q is not a valid cache name", name)
-		}
-
-		c[name] = cc
-	}
-
-	// This is a very old flag in Hugo, but we need to respect it.
-	disabled := cfg.GetBool("ignoreCache")
-
-	for k, v := range c {
-		v.Dir = filepath.Clean(v.Dir)
-		dir := filepath.ToSlash(v.Dir)
-		parts := strings.Split(dir, "/")
-		first := parts[0]
-
-		if strings.HasPrefix(first, ":") {
-			resolved, err := resolveDirPlaceholder(p, first)
-			if err != nil {
-				return c, err
-			}
-			resolved = filepath.ToSlash(resolved)
-
-			v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...))
-
-		} else if isOsFs && !path.IsAbs(dir) {
-			return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir)
-		}
-
-		if disabled {
-			v.MaxAge = 0
-		}
-
-		c[k] = v
-	}
-
-	return c, nil
-}
-
-// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
-func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
-	switch strings.ToLower(placeholder) {
-	case ":resourcedir":
-		return p.AbsResourcesDir, nil
-	case ":cachedir":
-		return helpers.GetCacheDir(p.Fs.Source, p.Cfg)
-	}
-
-	return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
-}
-
 func cleanID(name string) string {
 	return filepath.Clean(name)
 }

cache/filecache/filecache_config.go

@@ -0,0 +1,168 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/gohugoio/hugo/helpers"
+	"github.com/gohugoio/hugo/hugolib/paths"
+
+	"github.com/bep/mapstructure"
+	"github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+const (
+	cachesConfigKey = "caches"
+
+	resourcesGenDir = ":resourceDir/_gen"
+)
+
+var defaultCacheConfig = cacheConfig{
+	MaxAge: -1, // Never expire
+	Dir:    ":cacheDir",
+}
+
+const (
+	cacheKeyGetJSON = "getjson"
+	cacheKeyGetCSV  = "getcsv"
+	cacheKeyImages  = "images"
+	cacheKeyAssets  = "assets"
+)
+
+var defaultCacheConfigs = map[string]cacheConfig{
+	cacheKeyGetJSON: defaultCacheConfig,
+	cacheKeyGetCSV:  defaultCacheConfig,
+	cacheKeyImages: cacheConfig{
+		MaxAge: -1,
+		Dir:    resourcesGenDir,
+	},
+	cacheKeyAssets: cacheConfig{
+		MaxAge: -1,
+		Dir:    resourcesGenDir,
+	},
+}
+
+type cachesConfig map[string]cacheConfig
+
+type cacheConfig struct {
+	// Max age of cache entries in this cache. Any items older than this will
+	// be removed and not returned from the cache.
+	// -1 means forever, 0 means cache is disabled.
+	MaxAge int
+
+	// The directory where files are stored.
+	Dir string
+}
+
+// GetJSONCache gets the file cache for getJSON.
+func (f Caches) GetJSONCache() *Cache {
+	return f[cacheKeyGetJSON]
+}
+
+// GetCSVCache gets the file cache for getCSV.
+func (f Caches) GetCSVCache() *Cache {
+	return f[cacheKeyGetCSV]
+}
+
+// ImageCache gets the file cache for processed images.
+func (f Caches) ImageCache() *Cache {
+	return f[cacheKeyImages]
+}
+
+// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
+func (f Caches) AssetsCache() *Cache {
+	return f[cacheKeyAssets]
+}
+
+func decodeConfig(p *paths.Paths) (cachesConfig, error) {
+	c := make(cachesConfig)
+	valid := make(map[string]bool)
+	// Add defaults
+	for k, v := range defaultCacheConfigs {
+		c[k] = v
+		valid[k] = true
+	}
+
+	cfg := p.Cfg
+
+	m := cfg.GetStringMap(cachesConfigKey)
+
+	_, isOsFs := p.Fs.Source.(*afero.OsFs)
+
+	for k, v := range m {
+		cc := defaultCacheConfig
+
+		if err := mapstructure.WeakDecode(v, &cc); err != nil {
+			return nil, err
+		}
+
+		if cc.Dir == "" {
+			return c, errors.New("must provide cache Dir")
+		}
+
+		name := strings.ToLower(k)
+		if !valid[name] {
+			return nil, errors.Errorf("%q is not a valid cache name", name)
+		}
+
+		c[name] = cc
+	}
+
+	// This is a very old flag in Hugo, but we need to respect it.
+	disabled := cfg.GetBool("ignoreCache")
+
+	for k, v := range c {
+		v.Dir = filepath.Clean(v.Dir)
+		dir := filepath.ToSlash(v.Dir)
+		parts := strings.Split(dir, "/")
+		first := parts[0]
+
+		if strings.HasPrefix(first, ":") {
+			resolved, err := resolveDirPlaceholder(p, first)
+			if err != nil {
+				return c, err
+			}
+			resolved = filepath.ToSlash(resolved)
+
+			v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...))
+
+		} else if isOsFs && !path.IsAbs(dir) {
+			return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir)
+		}
+
+		if disabled {
+			v.MaxAge = 0
+		}
+
+		c[k] = v
+	}
+
+	return c, nil
+}
+
+// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
+func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
+	switch strings.ToLower(placeholder) {
+	case ":resourcedir":
+		return p.AbsResourcesDir, nil
+	case ":cachedir":
+		return helpers.GetCacheDir(p.Fs.Source, p.Cfg)
+	}
+
+	return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
+}

cache/filecache/filecache_config_test.go

@@ -0,0 +1,131 @@
+// Copyright 2018 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package filecache
+
+import (
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/gohugoio/hugo/config"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/gohugoio/hugo/hugolib/paths"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDecodeConfig(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+[caches]
+[caches.getJSON]
+maxAge = 1234
+dir = "/path/to/c1"
+[caches.getCSV]
+maxAge = 3456
+dir = "/path/to/c2"
+[caches.images]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	fs := hugofs.NewMem(cfg)
+	p, err := paths.New(fs, cfg)
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(p)
+	assert.NoError(err)
+
+	assert.Equal(4, len(decoded))
+
+	c2 := decoded["getcsv"]
+	assert.Equal(3456, c2.MaxAge)
+	assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
+
+	c3 := decoded["images"]
+	assert.Equal(-1, c3.MaxAge)
+	assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
+
+}
+
+func TestDecodeConfigIgnoreCache(t *testing.T) {
+	t.Parallel()
+
+	assert := require.New(t)
+
+	configStr := `
+ignoreCache = true
+[caches]
+[caches.getJSON]
+maxAge = 1234
+dir = "/path/to/c1"
+[caches.getCSV]
+maxAge = 3456
+dir = "/path/to/c2"
+[caches.images]
+dir = "/path/to/c3"
+
+`
+
+	cfg, err := config.FromConfigString(configStr, "toml")
+	assert.NoError(err)
+	fs := hugofs.NewMem(cfg)
+	p, err := paths.New(fs, cfg)
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(p)
+	assert.NoError(err)
+
+	assert.Equal(4, len(decoded))
+
+	for _, v := range decoded {
+		assert.Equal(0, v.MaxAge)
+	}
+
+}
+
+func TestDecodeConfigDefault(t *testing.T) {
+	assert := require.New(t)
+	cfg := viper.New()
+	if runtime.GOOS == "windows" {
+		cfg.Set("resourceDir", "c:\\cache\\resources")
+		cfg.Set("cacheDir", "c:\\cache\\thecache")
+
+	} else {
+		cfg.Set("resourceDir", "/cache/resources")
+		cfg.Set("cacheDir", "/cache/thecache")
+	}
+
+	fs := hugofs.NewMem(cfg)
+	p, err := paths.New(fs, cfg)
+	assert.NoError(err)
+
+	decoded, err := decodeConfig(p)
+
+	assert.NoError(err)
+
+	assert.Equal(4, len(decoded))
+
+	if runtime.GOOS == "windows" {
+		assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir)
+	} else {
+		assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir)
+	}
+}

cache/filecache/filecache_test.go

@@ -19,7 +19,6 @@ import (
 	"io/ioutil"
 	"path/filepath"
 	"regexp"
-	"runtime"
 	"strings"
 	"sync"
 	"testing"
@@ -31,7 +30,6 @@ import (
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/hugolib/paths"
 	"github.com/spf13/afero"
-	"github.com/spf13/viper"
 
 	"github.com/stretchr/testify/require"
 )
@@ -200,107 +198,3 @@ dir = "/cache/c"
 	}
 	wg.Wait()
 }
-
-func TestDecodeConfig(t *testing.T) {
-	t.Parallel()
-
-	assert := require.New(t)
-
-	configStr := `
-[caches]
-[caches.getJSON]
-maxAge = 1234
-dir = "/path/to/c1"
-[caches.getCSV]
-maxAge = 3456
-dir = "/path/to/c2"
-[caches.images]
-dir = "/path/to/c3"
-
-`
-
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
-	assert.NoError(err)
-
-	decoded, err := decodeConfig(p)
-	assert.NoError(err)
-
-	assert.Equal(4, len(decoded))
-
-	c2 := decoded["getcsv"]
-	assert.Equal(3456, c2.MaxAge)
-	assert.Equal(filepath.FromSlash("/path/to/c2"), c2.Dir)
-
-	c3 := decoded["images"]
-	assert.Equal(-1, c3.MaxAge)
-	assert.Equal(filepath.FromSlash("/path/to/c3"), c3.Dir)
-
-}
-
-func TestDecodeConfigIgnoreCache(t *testing.T) {
-	t.Parallel()
-
-	assert := require.New(t)
-
-	configStr := `
-ignoreCache = true
-[caches]
-[caches.getJSON]
-maxAge = 1234
-dir = "/path/to/c1"
-[caches.getCSV]
-maxAge = 3456
-dir = "/path/to/c2"
-[caches.images]
-dir = "/path/to/c3"
-
-`
-
-	cfg, err := config.FromConfigString(configStr, "toml")
-	assert.NoError(err)
-	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
-	assert.NoError(err)
-
-	decoded, err := decodeConfig(p)
-	assert.NoError(err)
-
-	assert.Equal(4, len(decoded))
-
-	for _, v := range decoded {
-		assert.Equal(0, v.MaxAge)
-	}
-
-}
-
-func TestDecodeConfigDefault(t *testing.T) {
-	assert := require.New(t)
-	cfg := viper.New()
-	if runtime.GOOS == "windows" {
-		cfg.Set("resourceDir", "c:\\cache\\resources")
-		cfg.Set("cacheDir", "c:\\cache\\thecache")
-
-	} else {
-		cfg.Set("resourceDir", "/cache/resources")
-		cfg.Set("cacheDir", "/cache/thecache")
-	}
-
-	fs := hugofs.NewMem(cfg)
-	p, err := paths.New(fs, cfg)
-	assert.NoError(err)
-
-	decoded, err := decodeConfig(p)
-
-	assert.NoError(err)
-
-	assert.Equal(4, len(decoded))
-
-	if runtime.GOOS == "windows" {
-		assert.Equal("c:\\cache\\resources\\_gen", decoded[cacheKeyImages].Dir)
-	} else {
-		assert.Equal("/cache/resources/_gen", decoded[cacheKeyImages].Dir)
-	}
-}