mirror of
https://github.com/gohugoio/hugo.git
synced 2024-11-21 20:46:30 -05:00
f7aeaa6129
This commits reworks how file caching is performed in Hugo. Now there is only one way, and it can be configured.
This is the default configuration:
```toml
[caches]
[caches.getjson]
dir = ":cacheDir"
maxAge = -1
[caches.getcsv]
dir = ":cacheDir"
maxAge = -1
[caches.images]
dir = ":resourceDir/_gen"
maxAge = -1
[caches.assets]
dir = ":resourceDir/_gen"
maxAge = -1
```
You can override any of these cache setting in your own `config.toml`.
The placeholders explained:
`:cacheDir`: This is the value of the `cacheDir` config option if set (can also be set via OS env variable `HUGO_CACHEDIR`). It will fall back to `/opt/build/cache/hugo_cache/` on Netlify, or a `hugo_cache` directory below the OS temp dir for the others.
`:resourceDir`: This is the value of the `resourceDir` config option.
`maxAge` is the time in seconds before a cache entry will be evicted, -1 means forever and 0 effectively turns that particular cache off.
This means that if you run your builds on Netlify, all caches configured with `:cacheDir` will be saved and restored on the next build. For other CI vendors, please read their documentation. For an CircleCI example, see 6c3960a8f4/.circleci/config.yml
Fixes #5404
123 lines
3.1 KiB
Go
123 lines
3.1 KiB
Go
// Copyright 2016 The Hugo Authors. All rights reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package data
|
|
|
|
import (
|
|
"io/ioutil"
|
|
"net/http"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/gohugoio/hugo/cache/filecache"
|
|
|
|
"github.com/gohugoio/hugo/config"
|
|
"github.com/gohugoio/hugo/helpers"
|
|
"github.com/spf13/afero"
|
|
)
|
|
|
|
var (
|
|
resSleep = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
|
|
resRetries = 1 // number of retries to load the JSON from URL
|
|
)
|
|
|
|
// getRemote loads the content of a remote file. This method is thread safe.
|
|
func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (error, bool), req *http.Request) error {
|
|
url := req.URL.String()
|
|
id := helpers.MD5String(url)
|
|
var handled bool
|
|
var retry bool
|
|
|
|
_, b, err := cache.GetOrCreateBytes(id, func() ([]byte, error) {
|
|
var err error
|
|
handled = true
|
|
for i := 0; i <= resRetries; i++ {
|
|
ns.deps.Log.INFO.Printf("Downloading: %s ...", url)
|
|
var res *http.Response
|
|
res, err = ns.client.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if isHTTPError(res) {
|
|
return nil, errors.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode))
|
|
}
|
|
|
|
var b []byte
|
|
b, err = ioutil.ReadAll(res.Body)
|
|
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
res.Body.Close()
|
|
|
|
err, retry = unmarshal(b)
|
|
|
|
if err == nil {
|
|
// Return it so it can be cached.
|
|
return b, nil
|
|
}
|
|
|
|
if !retry {
|
|
return nil, err
|
|
}
|
|
|
|
ns.deps.Log.INFO.Printf("Cannot read remote resource %s: %s", url, err)
|
|
ns.deps.Log.INFO.Printf("Retry #%d for %s and sleeping for %s", i+1, url, resSleep)
|
|
time.Sleep(resSleep)
|
|
}
|
|
|
|
return nil, err
|
|
|
|
})
|
|
|
|
if !handled {
|
|
// This is cached content and should be correct.
|
|
err, _ = unmarshal(b)
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
// getLocal loads the content of a local file
|
|
func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
|
|
filename := filepath.Join(cfg.GetString("workingDir"), url)
|
|
if e, err := helpers.Exists(filename, fs); !e {
|
|
return nil, err
|
|
}
|
|
|
|
return afero.ReadFile(fs, filename)
|
|
|
|
}
|
|
|
|
// getResource loads the content of a local or remote file and returns its content and the
|
|
// cache ID used, if relevant.
|
|
func (ns *Namespace) getResource(cache *filecache.Cache, unmarshal func(b []byte) (error, bool), req *http.Request) error {
|
|
switch req.URL.Scheme {
|
|
case "":
|
|
b, err := getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err, _ = unmarshal(b)
|
|
return err
|
|
default:
|
|
return ns.getRemote(cache, unmarshal, req)
|
|
}
|
|
}
|
|
|
|
func isHTTPError(res *http.Response) bool {
|
|
return res.StatusCode < 200 || res.StatusCode > 299
|
|
}
|