hugo/cache/filecache/filecache.go

// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package filecache

import (
	"bytes"
	"io"
	"io/ioutil"
	"path"
	"path/filepath"
	"strings"
	"time"

	"github.com/gohugoio/hugo/common/hugio"

	"github.com/gohugoio/hugo/helpers"

	"github.com/gohugoio/hugo/hugolib/paths"

	"github.com/pkg/errors"

	"github.com/BurntSushi/locker"
	"github.com/bep/mapstructure"
	"github.com/spf13/afero"
)

const (
	cachesConfigKey = "caches"

	resourcesGenDir = ":resourceDir/_gen"
)

var defaultCacheConfig = cacheConfig{
	MaxAge: -1, // Never expire
	Dir:    ":cacheDir",
}

const (
	cacheKeyGetJSON = "getjson"
	cacheKeyGetCSV  = "getcsv"
	cacheKeyImages  = "images"
	cacheKeyAssets  = "assets"
)

var defaultCacheConfigs = map[string]cacheConfig{
	cacheKeyGetJSON: defaultCacheConfig,
	cacheKeyGetCSV:  defaultCacheConfig,
	cacheKeyImages: cacheConfig{
		MaxAge: -1,
		Dir:    resourcesGenDir,
	},
	cacheKeyAssets: cacheConfig{
		MaxAge: -1,
		Dir:    resourcesGenDir,
	},
}

type cachesConfig map[string]cacheConfig

type cacheConfig struct {
	// Max age of cache entries in this cache. Any items older than this will
	// be removed and not returned from the cache.
	// -1 means forever, 0 means cache is disabled.
	MaxAge int

	// The directory where files are stored.
	Dir string
}

// Cache caches a set of files in a directory. This is usually a file on
// disk, but since this is backed by an Afero file system, it can be anything.
type Cache struct {
	Fs afero.Fs

	// Max age in seconds.
	maxAge int

	nlocker *locker.Locker
}

// ItemInfo contains info about a cached file.
type ItemInfo struct {
	// This is the file's name relative to the cache's filesystem.
	Name string
}

// NewCache creates a new file cache with the given filesystem and max age.
func NewCache(fs afero.Fs, maxAge int) *Cache {
	return &Cache{
		Fs:      fs,
		nlocker: locker.NewLocker(),
		maxAge:  maxAge,
	}
}

// lockedFile is a file with a lock that is released on Close.
type lockedFile struct {
	afero.File
	unlock func()
}

func (l *lockedFile) Close() error {
	defer l.unlock()
	return l.File.Close()
}

// WriteCloser returns a transactional writer into the cache.
// It's important that it's closed when done.
func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
	id = cleanID(id)
	c.nlocker.Lock(id)

	info := ItemInfo{Name: id}

	f, err := helpers.OpenFileForWriting(c.Fs, id)
	if err != nil {
		c.nlocker.Unlock(id)
		return info, nil, err
	}

	return info, &lockedFile{
		File:   f,
		unlock: func() { c.nlocker.Unlock(id) },
	}, nil
}

// ReadOrCreate tries to lookup the file in cache.
// If found, it is passed to read and then closed.
// If not found a new file is created and passed to create, which should close
// it when done.
func (c *Cache) ReadOrCreate(id string,
	read func(info ItemInfo, r io.Reader) error,
	create func(info ItemInfo, w io.WriteCloser) error) (info ItemInfo, err error) {
	id = cleanID(id)

	c.nlocker.Lock(id)
	defer c.nlocker.Unlock(id)

	info = ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		err = read(info, r)
		defer r.Close()
		return
	}

	f, err := helpers.OpenFileForWriting(c.Fs, id)
	if err != nil {
		return
	}

	err = create(info, f)

	return

}

// GetOrCreate tries to get the file with the given id from cache. If not found or expired, create will
// be invoked and the result cached.
// This method is protected by a named lock using the given id as identifier.
func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (ItemInfo, io.ReadCloser, error) {
	id = cleanID(id)

	c.nlocker.Lock(id)
	defer c.nlocker.Unlock(id)

	info := ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		return info, r, nil
	}

	r, err := create()
	if err != nil {
		return info, nil, err
	}

	if c.maxAge == 0 {
		// No caching.
		return info, hugio.ToReadCloser(r), nil
	}

	var buff bytes.Buffer
	return info,
		hugio.ToReadCloser(&buff),
		afero.WriteReader(c.Fs, id, io.TeeReader(r, &buff))
}

// GetOrCreateBytes is the same as GetOrCreate, but produces a byte slice.
func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (ItemInfo, []byte, error) {
	id = cleanID(id)

	c.nlocker.Lock(id)
	defer c.nlocker.Unlock(id)

	info := ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		defer r.Close()
		b, err := ioutil.ReadAll(r)
		return info, b, err
	}

	b, err := create()
	if err != nil {
		return info, nil, err
	}

	if c.maxAge == 0 {
		return info, b, nil
	}

	if err := afero.WriteReader(c.Fs, id, bytes.NewReader(b)); err != nil {
		return info, nil, err
	}
	return info, b, nil

}

// GetBytes gets the file content with the given id from the cahce, nil if none found.
func (c *Cache) GetBytes(id string) (ItemInfo, []byte, error) {
	id = cleanID(id)

	c.nlocker.Lock(id)
	defer c.nlocker.Unlock(id)

	info := ItemInfo{Name: id}

	if r := c.getOrRemove(id); r != nil {
		defer r.Close()
		b, err := ioutil.ReadAll(r)
		return info, b, err
	}

	return info, nil, nil
}

// Get gets the file with the given id from the cahce, nil if none found.
func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
	id = cleanID(id)

	c.nlocker.Lock(id)
	defer c.nlocker.Unlock(id)

	info := ItemInfo{Name: id}

	r := c.getOrRemove(id)

	return info, r, nil
}

// getOrRemove gets the file with the given id. If it's expired, it will
// be removed.
func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
	if c.maxAge == 0 {
		// No caching.
		return nil
	}

	if c.maxAge > 0 {
		fi, err := c.Fs.Stat(id)
		if err != nil {
			return nil
		}

		expiry := time.Now().Add(-time.Duration(c.maxAge) * time.Second)
		expired := fi.ModTime().Before(expiry)
		if expired {
			c.Fs.Remove(id)
			return nil
		}
	}

	f, err := c.Fs.Open(id)

	if err != nil {
		return nil
	}

	return f
}

// For testing
func (c *Cache) getString(id string) string {
	id = cleanID(id)

	c.nlocker.Lock(id)
	defer c.nlocker.Unlock(id)

	if r := c.getOrRemove(id); r != nil {
		defer r.Close()
		b, _ := ioutil.ReadAll(r)
		return string(b)
	}

	return ""

}

// Caches is a named set of caches.
type Caches map[string]*Cache

// Get gets a named cache, nil if none found.
func (f Caches) Get(name string) *Cache {
	return f[strings.ToLower(name)]
}

// GetJSONCache gets the file cache for getJSON.
func (f Caches) GetJSONCache() *Cache {
	return f[cacheKeyGetJSON]
}

// GetCSVCache gets the file cache for getCSV.
func (f Caches) GetCSVCache() *Cache {
	return f[cacheKeyGetCSV]
}

// ImageCache gets the file cache for processed images.
func (f Caches) ImageCache() *Cache {
	return f[cacheKeyImages]
}

// AssetsCache gets the file cache for assets (processed resources, SCSS etc.).
func (f Caches) AssetsCache() *Cache {
	return f[cacheKeyAssets]
}

// NewCachesFromPaths creates a new set of file caches from the given
// configuration.
func NewCachesFromPaths(p *paths.Paths) (Caches, error) {
	dcfg, err := decodeConfig(p)
	if err != nil {
		return nil, err
	}

	fs := p.Fs.Source

	m := make(Caches)
	for k, v := range dcfg {
		baseDir := filepath.Join(v.Dir, k)
		if err = fs.MkdirAll(baseDir, 0777); err != nil {
			return nil, err
		}
		bfs := afero.NewBasePathFs(fs, baseDir)
		m[k] = NewCache(bfs, v.MaxAge)
	}

	return m, nil
}

func decodeConfig(p *paths.Paths) (cachesConfig, error) {
	c := make(cachesConfig)
	valid := make(map[string]bool)
	// Add defaults
	for k, v := range defaultCacheConfigs {
		c[k] = v
		valid[k] = true
	}

	cfg := p.Cfg

	m := cfg.GetStringMap(cachesConfigKey)

	_, isOsFs := p.Fs.Source.(*afero.OsFs)

	for k, v := range m {
		cc := defaultCacheConfig

		if err := mapstructure.WeakDecode(v, &cc); err != nil {
			return nil, err
		}

		if cc.Dir == "" {
			return c, errors.New("must provide cache Dir")
		}

		name := strings.ToLower(k)
		if !valid[name] {
			return nil, errors.Errorf("%q is not a valid cache name", name)
		}

		c[name] = cc
	}

	// This is a very old flag in Hugo, but we need to respect it.
	disabled := cfg.GetBool("ignoreCache")

	for k, v := range c {
		v.Dir = filepath.Clean(v.Dir)
		dir := filepath.ToSlash(v.Dir)
		parts := strings.Split(dir, "/")
		first := parts[0]

		if strings.HasPrefix(first, ":") {
			resolved, err := resolveDirPlaceholder(p, first)
			if err != nil {
				return c, err
			}
			resolved = filepath.ToSlash(resolved)

			v.Dir = filepath.FromSlash(path.Join((append([]string{resolved}, parts[1:]...))...))

		} else if isOsFs && !path.IsAbs(dir) {
			return c, errors.Errorf("%q must either start with a placeholder (e.g. :cacheDir, :resourceDir) or be absolute", v.Dir)
		}

		if disabled {
			v.MaxAge = 0
		}

		c[k] = v
	}

	return c, nil
}

// Resolves :resourceDir => /myproject/resources etc., :cacheDir => ...
func resolveDirPlaceholder(p *paths.Paths, placeholder string) (string, error) {
	switch strings.ToLower(placeholder) {
	case ":resourcedir":
		return p.AbsResourcesDir, nil
	case ":cachedir":
		return helpers.GetCacheDir(p.Fs.Source, p.Cfg)
	}

	return "", errors.Errorf("%q is not a valid placeholder (valid values are :cacheDir or :resourceDir)", placeholder)
}

func cleanID(name string) string {
	return filepath.Clean(name)
}