hugo/tpl/data/data.go

209 lines
5.3 KiB
Go
Raw Permalink Normal View History

// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package data provides template functions for working with external data
// sources.
package data
import (
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/config/security"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/common/constants"
"github.com/spf13/cast"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/deps"
)
// New returns a new instance of the data-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
return &Namespace{
deps: deps,
cacheGetCSV: deps.ResourceSpec.FileCaches.GetCSVCache(),
cacheGetJSON: deps.ResourceSpec.FileCaches.GetJSONCache(),
client: http.DefaultClient,
}
}
// Namespace provides template functions for the "data" namespace.
type Namespace struct {
deps *deps.Deps
cacheGetJSON *filecache.Cache
cacheGetCSV *filecache.Cache
client *http.Client
}
2022-12-21 07:11:08 -05:00
// GetCSV expects the separator sep and one or n-parts of a URL to a resource which
// can either be a local or a remote one.
// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
// If you provide multiple parts for the URL they will be joined together to the final URL.
// GetCSV returns nil or a slice slice to use in a short code.
func (ns *Namespace) GetCSV(sep string, args ...any) (d [][]string, err error) {
url, headers := toURLAndHeaders(args)
cache := ns.cacheGetCSV
2019-03-24 05:11:16 -04:00
unmarshal := func(b []byte) (bool, error) {
if d, err = parseCSV(b, sep); err != nil {
err = fmt.Errorf("failed to parse CSV file %s: %w", url, err)
2019-03-24 05:11:16 -04:00
return true, err
}
2019-03-24 05:11:16 -04:00
return false, nil
}
var req *http.Request
req, err = http.NewRequest("GET", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request for getCSV for resource %s: %w", url, err)
}
// Add custom user headers.
addUserProvidedHeaders(headers, req)
addDefaultHeaders(req, "text/csv", "text/plain")
err = ns.getResource(cache, unmarshal, req)
if err != nil {
if security.IsAccessDenied(err) {
return nil, err
}
ns.deps.Log.Errorsf(constants.ErrRemoteGetCSV, "Failed to get CSV resource %q: %s", url, err)
return nil, nil
}
return
}
2022-12-21 07:11:08 -05:00
// GetJSON expects one or n-parts of a URL in args to a resource which can either be a local or a remote one.
// If you provide multiple parts they will be joined together to the final URL.
// GetJSON returns nil or parsed JSON to use in a short code.
func (ns *Namespace) GetJSON(args ...any) (any, error) {
var v any
url, headers := toURLAndHeaders(args)
cache := ns.cacheGetJSON
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, fmt.Errorf("Failed to create request for getJSON resource %s: %w", url, err)
}
2019-03-24 05:11:16 -04:00
unmarshal := func(b []byte) (bool, error) {
err := json.Unmarshal(b, &v)
if err != nil {
2019-03-24 05:11:16 -04:00
return true, err
}
2019-03-24 05:11:16 -04:00
return false, nil
}
addUserProvidedHeaders(headers, req)
addDefaultHeaders(req, "application/json")
err = ns.getResource(cache, unmarshal, req)
if err != nil {
if security.IsAccessDenied(err) {
return nil, err
}
ns.deps.Log.Errorsf(constants.ErrRemoteGetJSON, "Failed to get JSON resource %q: %s", url, err)
return nil, nil
}
return v, nil
}
func addDefaultHeaders(req *http.Request, accepts ...string) {
for _, accept := range accepts {
if !hasHeaderValue(req.Header, "Accept", accept) {
req.Header.Add("Accept", accept)
}
}
if !hasHeaderKey(req.Header, "User-Agent") {
req.Header.Add("User-Agent", "Hugo Static Site Generator")
}
}
func addUserProvidedHeaders(headers map[string]any, req *http.Request) {
if headers == nil {
return
}
for key, val := range headers {
vals := types.ToStringSlicePreserveString(val)
for _, s := range vals {
req.Header.Add(key, s)
}
}
}
func hasHeaderValue(m http.Header, key, value string) bool {
var s []string
var ok bool
if s, ok = m[key]; !ok {
return false
}
for _, v := range s {
if v == value {
return true
}
}
return false
}
func hasHeaderKey(m http.Header, key string) bool {
_, ok := m[key]
return ok
}
func toURLAndHeaders(urlParts []any) (string, map[string]any) {
if len(urlParts) == 0 {
return "", nil
}
// The last argument may be a map.
headers, err := maps.ToStringMapE(urlParts[len(urlParts)-1])
if err == nil {
urlParts = urlParts[:len(urlParts)-1]
} else {
headers = nil
}
return strings.Join(cast.ToStringSlice(urlParts), ""), headers
}
// parseCSV parses bytes of CSV data into a slice slice string or an error
func parseCSV(c []byte, sep string) ([][]string, error) {
if len(sep) != 1 {
return nil, errors.New("Incorrect length of CSV separator: " + sep)
}
b := bytes.NewReader(c)
r := csv.NewReader(b)
rSep := []rune(sep)
r.Comma = rSep[0]
r.FieldsPerRecord = 0
return r.ReadAll()
}