mirror of
https://github.com/gohugoio/hugo.git
synced 2025-03-23 18:35:11 +00:00
Add resources.GetRemote
In Hugo 0.89 we added remote support to `resources.Get`. In hindsight that was not a great idea, as a poll from many Hugo users showed. See Issue #9285 for more details. After this commit `resources.Get` only supports local resource lookups. If you want to support both, you need to use a construct similar to: Also improve some option case handling. ``` {{ resource := "" }} {{ if (urls.Parse $url).IsAbs }} {{ $resource = resources.GetRemote $url }} {{ else }} {{ $resource = resources.Get $url }} {{ end }} ``` Fixes #9285 Fixes #9296
This commit is contained in:
parent
5758c370ea
commit
22ef5da20d
7 changed files with 375 additions and 253 deletions
|
@ -408,10 +408,10 @@ FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }}
|
|||
CSS integrity Data first: {{ $cssFingerprinted1.Data.Integrity }} {{ $cssFingerprinted1.RelPermalink }}
|
||||
CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerprinted2.Data.Integrity }}
|
||||
|
||||
{{ $rimg := resources.Get "%[1]s/sunset.jpg" }}
|
||||
{{ $remotenotfound := resources.Get "%[1]s/notfound.jpg" }}
|
||||
{{ $rimg := resources.GetRemote "%[1]s/sunset.jpg" }}
|
||||
{{ $remotenotfound := resources.GetRemote "%[1]s/notfound.jpg" }}
|
||||
{{ $localnotfound := resources.Get "images/notfound.jpg" }}
|
||||
{{ $gopherprotocol := resources.Get "gopher://example.org" }}
|
||||
{{ $gopherprotocol := resources.GetRemote "gopher://example.org" }}
|
||||
{{ $rfit := $rimg.Fit "200x200" }}
|
||||
{{ $rfit2 := $rfit.Fit "100x200" }}
|
||||
{{ $rimg = $rimg | fingerprint }}
|
||||
|
@ -453,8 +453,8 @@ SUNSET REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s.a9bf1d944e19c0f382e0d8f51de690f7d0
|
|||
FIT REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s_hu59e56ffff1bc1d8d122b1403d34e039f_0_200x200_fit_q75_box.jpg|200
|
||||
REMOTE NOT FOUND: OK
|
||||
LOCAL NOT FOUND: OK
|
||||
PRINT PROTOCOL ERROR1: error calling resources.Get: Get "gopher://example.org": unsupported protocol scheme "gopher"
|
||||
PRINT PROTOCOL ERROR2: error calling resources.Get: Get "gopher://example.org": unsupported protocol scheme "gopher"
|
||||
PRINT PROTOCOL ERROR1: error calling resources.GetRemote: Get "gopher://example.org": unsupported protocol scheme "gopher"
|
||||
PRINT PROTOCOL ERROR2: error calling resources.GetRemote: Get "gopher://example.org": unsupported protocol scheme "gopher"
|
||||
|
||||
|
||||
`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]interface{}{})))
|
||||
|
@ -691,18 +691,18 @@ T6: {{ $bundle1.Permalink }}
|
|||
`)
|
||||
b.WithTemplates("home.html", fmt.Sprintf(`
|
||||
Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }}
|
||||
Min CSS Remote: {{ ( resources.Get "%[1]s/css/styles1.css" | minify ).Content }}
|
||||
Min CSS Remote: {{ ( resources.GetRemote "%[1]s/css/styles1.css" | minify ).Content }}
|
||||
Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }}
|
||||
Min JS Remote: {{ ( resources.Get "%[1]s/js/script1.js" | minify ).Content }}
|
||||
Min JS Remote: {{ ( resources.GetRemote "%[1]s/js/script1.js" | minify ).Content }}
|
||||
Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }}
|
||||
Min JSON Remote: {{ ( resources.Get "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
|
||||
Min JSON Remote: {{ ( resources.GetRemote "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
|
||||
Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
|
||||
Min XML Remote: {{ ( resources.Get "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
|
||||
Min XML Remote: {{ ( resources.GetRemote "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
|
||||
Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
|
||||
Min SVG Remote: {{ ( resources.Get "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
|
||||
Min SVG Remote: {{ ( resources.GetRemote "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
|
||||
Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
|
||||
Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }}
|
||||
Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
|
||||
Min HTML Remote: {{ ( resources.GetRemote "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
|
||||
`, ts.URL))
|
||||
}, func(b *sitesBuilder) {
|
||||
b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`)
|
||||
|
@ -722,13 +722,13 @@ Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify
|
|||
|
||||
{"remote", func() bool { return true }, func(b *sitesBuilder) {
|
||||
b.WithTemplates("home.html", fmt.Sprintf(`
|
||||
{{$js := resources.Get "%[1]s/js/script1.js" }}
|
||||
{{$js := resources.GetRemote "%[1]s/js/script1.js" }}
|
||||
Remote Filename: {{ $js.RelPermalink }}
|
||||
{{$svg := resources.Get "%[1]s/mydata/svg1.svg" }}
|
||||
{{$svg := resources.GetRemote "%[1]s/mydata/svg1.svg" }}
|
||||
Remote Content-Disposition: {{ $svg.RelPermalink }}
|
||||
{{$auth := resources.Get "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
|
||||
{{$auth := resources.GetRemote "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
|
||||
Remote Authorization: {{ $auth.Content }}
|
||||
{{$post := resources.Get "%[1]s/post" (dict "method" "post" "body" "Request body") }}
|
||||
{{$post := resources.GetRemote "%[1]s/post" (dict "method" "post" "body" "Request body") }}
|
||||
Remote POST: {{ $post.Content }}
|
||||
`, ts.URL))
|
||||
}, func(b *sitesBuilder) {
|
||||
|
|
|
@ -148,19 +148,19 @@ func TestSecurityPolicies(t *testing.T) {
|
|||
testVariant(c, cb, `(?s).*"dart-sass-embedded" is not whitelisted in policy "security\.exec\.allow".*`)
|
||||
})
|
||||
|
||||
c.Run("resources.Get, OK", func(c *qt.C) {
|
||||
c.Run("resources.GetRemote, OK", func(c *qt.C) {
|
||||
c.Parallel()
|
||||
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil)
|
||||
httpTestVariant(c, `{{ $json := resources.GetRemote "%[1]s/fruits.json" }}{{ $json.Content }}`, "", nil)
|
||||
})
|
||||
|
||||
c.Run("resources.Get, denied method", func(c *qt.C) {
|
||||
c.Run("resources.GetRemote, denied method", func(c *qt.C) {
|
||||
c.Parallel()
|
||||
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil)
|
||||
httpTestVariant(c, `{{ $json := resources.GetRemote "%[1]s/fruits.json" (dict "method" "DELETE" ) }}{{ $json.Content }}`, `(?s).*"DELETE" is not whitelisted in policy "security\.http\.method".*`, nil)
|
||||
})
|
||||
|
||||
c.Run("resources.Get, denied URL", func(c *qt.C) {
|
||||
c.Run("resources.GetRemote, denied URL", func(c *qt.C) {
|
||||
c.Parallel()
|
||||
httpTestVariant(c, `{{ $json := resources.Get "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
|
||||
httpTestVariant(c, `{{ $json := resources.GetRemote "%[1]s/fruits.json" }}{{ $json.Content }}`, `(?s).*is not whitelisted in policy "security\.http\.urls".*`,
|
||||
func(b *sitesBuilder) {
|
||||
b.WithConfigFile("toml", `
|
||||
[security]
|
||||
|
|
|
@ -16,15 +16,7 @@
|
|||
package create
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
@ -36,13 +28,8 @@ import (
|
|||
|
||||
"github.com/gohugoio/hugo/cache/filecache"
|
||||
"github.com/gohugoio/hugo/common/hugio"
|
||||
"github.com/gohugoio/hugo/common/maps"
|
||||
"github.com/gohugoio/hugo/common/types"
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/resources"
|
||||
"github.com/gohugoio/hugo/resources/resource"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Client contains methods to create Resource objects.
|
||||
|
@ -150,203 +137,3 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
|
|||
})
|
||||
})
|
||||
}
|
||||
|
||||
// FromRemote expects one or n-parts of a URL to a resource
|
||||
// If you provide multiple parts they will be joined together to the final URL.
|
||||
func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) {
|
||||
if err := c.validateFromRemoteArgs(uri, options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rURL, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
|
||||
}
|
||||
|
||||
resourceID := helpers.HashString(uri, options)
|
||||
|
||||
_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
|
||||
method, reqBody, err := getMethodAndBody(options)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(method, uri, reqBody)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
|
||||
}
|
||||
addDefaultHeaders(req)
|
||||
|
||||
if _, ok := options["headers"]; ok {
|
||||
headers, err := maps.ToStringMapE(options["headers"])
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri)
|
||||
}
|
||||
addUserProvidedHeaders(headers, req)
|
||||
}
|
||||
|
||||
res, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusNotFound {
|
||||
if res.StatusCode < 200 || res.StatusCode > 299 {
|
||||
return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
|
||||
}
|
||||
}
|
||||
|
||||
httpResponse, err := httputil.DumpResponse(res, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResponse.Close()
|
||||
|
||||
res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
// Not found. This matches how looksup for local resources work.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
|
||||
}
|
||||
|
||||
filename := path.Base(rURL.Path)
|
||||
if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
|
||||
if _, ok := params["filename"]; ok {
|
||||
filename = params["filename"]
|
||||
}
|
||||
}
|
||||
|
||||
var extension string
|
||||
if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
|
||||
extension = arr[0]
|
||||
}
|
||||
|
||||
// If extension was not determined by header, look for a file extention
|
||||
if extension == "" {
|
||||
if ext := path.Ext(filename); ext != "" {
|
||||
extension = ext
|
||||
}
|
||||
}
|
||||
|
||||
// If extension was not determined by header or file extention, try using content itself
|
||||
if extension == "" {
|
||||
if ct := http.DetectContentType(body); ct != "application/octet-stream" {
|
||||
if ct == "image/jpeg" {
|
||||
extension = ".jpg"
|
||||
} else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
|
||||
extension = arr[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension
|
||||
|
||||
return c.rs.New(
|
||||
resources.ResourceSourceDescriptor{
|
||||
LazyPublish: true,
|
||||
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
|
||||
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
|
||||
},
|
||||
RelTargetFilename: filepath.Clean(resourceID),
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func (c *Client) validateFromRemoteArgs(uri string, options map[string]interface{}) error {
|
||||
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if method, ok := options["method"].(string); ok {
|
||||
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(method); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func addDefaultHeaders(req *http.Request, accepts ...string) {
|
||||
for _, accept := range accepts {
|
||||
if !hasHeaderValue(req.Header, "Accept", accept) {
|
||||
req.Header.Add("Accept", accept)
|
||||
}
|
||||
}
|
||||
if !hasHeaderKey(req.Header, "User-Agent") {
|
||||
req.Header.Add("User-Agent", "Hugo Static Site Generator")
|
||||
}
|
||||
}
|
||||
|
||||
func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
|
||||
if headers == nil {
|
||||
return
|
||||
}
|
||||
for key, val := range headers {
|
||||
vals := types.ToStringSlicePreserveString(val)
|
||||
for _, s := range vals {
|
||||
req.Header.Add(key, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func hasHeaderValue(m http.Header, key, value string) bool {
|
||||
var s []string
|
||||
var ok bool
|
||||
|
||||
if s, ok = m[key]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, v := range s {
|
||||
if v == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasHeaderKey(m http.Header, key string) bool {
|
||||
_, ok := m[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) {
|
||||
if options == nil {
|
||||
return "GET", nil, nil
|
||||
}
|
||||
|
||||
if method, ok := options["method"].(string); ok {
|
||||
method = strings.ToUpper(method)
|
||||
switch method {
|
||||
case "GET", "DELETE", "HEAD", "OPTIONS":
|
||||
return method, nil, nil
|
||||
case "POST", "PUT", "PATCH":
|
||||
var body []byte
|
||||
if _, ok := options["body"]; ok {
|
||||
switch b := options["body"].(type) {
|
||||
case string:
|
||||
body = []byte(b)
|
||||
case []byte:
|
||||
body = b
|
||||
}
|
||||
}
|
||||
return method, bytes.NewBuffer(body), nil
|
||||
}
|
||||
|
||||
return "", nil, fmt.Errorf("invalid HTTP method %q", method)
|
||||
}
|
||||
|
||||
return "GET", nil, nil
|
||||
}
|
||||
|
|
230
resources/resource_factories/create/remote.go
Normal file
230
resources/resource_factories/create/remote.go
Normal file
|
@ -0,0 +1,230 @@
|
|||
// Copyright 2021 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package create
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gohugoio/hugo/common/hugio"
|
||||
"github.com/gohugoio/hugo/common/types"
|
||||
"github.com/gohugoio/hugo/helpers"
|
||||
"github.com/gohugoio/hugo/resources"
|
||||
"github.com/gohugoio/hugo/resources/resource"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// FromRemote expects one or n-parts of a URL to a resource
|
||||
// If you provide multiple parts they will be joined together to the final URL.
|
||||
func (c *Client) FromRemote(uri string, optionsm map[string]interface{}) (resource.Resource, error) {
|
||||
rURL, err := url.Parse(uri)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
|
||||
}
|
||||
|
||||
resourceID := helpers.HashString(uri, optionsm)
|
||||
|
||||
_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
|
||||
options, err := decodeRemoteOptions(optionsm)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to decode options for resource %s", uri)
|
||||
}
|
||||
if err := c.validateFromRemoteArgs(uri, options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(options.Method, uri, options.BodyReader())
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
|
||||
}
|
||||
addDefaultHeaders(req)
|
||||
|
||||
if options.Headers != nil {
|
||||
addUserProvidedHeaders(options.Headers, req)
|
||||
}
|
||||
|
||||
res, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusNotFound {
|
||||
if res.StatusCode < 200 || res.StatusCode > 299 {
|
||||
return nil, errors.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode))
|
||||
}
|
||||
}
|
||||
|
||||
httpResponse, err := httputil.DumpResponse(res, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return hugio.ToReadCloser(bytes.NewReader(httpResponse)), nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResponse.Close()
|
||||
|
||||
res, err := http.ReadResponse(bufio.NewReader(httpResponse), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if res.StatusCode == http.StatusNotFound {
|
||||
// Not found. This matches how looksup for local resources work.
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
|
||||
}
|
||||
|
||||
filename := path.Base(rURL.Path)
|
||||
if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
|
||||
if _, ok := params["filename"]; ok {
|
||||
filename = params["filename"]
|
||||
}
|
||||
}
|
||||
|
||||
var extension string
|
||||
if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
|
||||
extension = arr[0]
|
||||
}
|
||||
|
||||
// If extension was not determined by header, look for a file extention
|
||||
if extension == "" {
|
||||
if ext := path.Ext(filename); ext != "" {
|
||||
extension = ext
|
||||
}
|
||||
}
|
||||
|
||||
// If extension was not determined by header or file extention, try using content itself
|
||||
if extension == "" {
|
||||
if ct := http.DetectContentType(body); ct != "application/octet-stream" {
|
||||
if ct == "image/jpeg" {
|
||||
extension = ".jpg"
|
||||
} else if arr, _ := mime.ExtensionsByType(ct); arr != nil {
|
||||
extension = arr[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + extension
|
||||
|
||||
return c.rs.New(
|
||||
resources.ResourceSourceDescriptor{
|
||||
LazyPublish: true,
|
||||
OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
|
||||
return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
|
||||
},
|
||||
RelTargetFilename: filepath.Clean(resourceID),
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) error {
|
||||
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPURL(uri); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := c.rs.ExecHelper.Sec().CheckAllowedHTTPMethod(options.Method); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func addDefaultHeaders(req *http.Request, accepts ...string) {
|
||||
for _, accept := range accepts {
|
||||
if !hasHeaderValue(req.Header, "Accept", accept) {
|
||||
req.Header.Add("Accept", accept)
|
||||
}
|
||||
}
|
||||
if !hasHeaderKey(req.Header, "User-Agent") {
|
||||
req.Header.Add("User-Agent", "Hugo Static Site Generator")
|
||||
}
|
||||
}
|
||||
|
||||
func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
|
||||
if headers == nil {
|
||||
return
|
||||
}
|
||||
for key, val := range headers {
|
||||
vals := types.ToStringSlicePreserveString(val)
|
||||
for _, s := range vals {
|
||||
req.Header.Add(key, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func hasHeaderValue(m http.Header, key, value string) bool {
|
||||
var s []string
|
||||
var ok bool
|
||||
|
||||
if s, ok = m[key]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, v := range s {
|
||||
if v == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func hasHeaderKey(m http.Header, key string) bool {
|
||||
_, ok := m[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
type fromRemoteOptions struct {
|
||||
Method string
|
||||
Headers map[string]interface{}
|
||||
Body []byte
|
||||
}
|
||||
|
||||
func (o fromRemoteOptions) BodyReader() io.Reader {
|
||||
if o.Body == nil {
|
||||
return nil
|
||||
}
|
||||
return bytes.NewBuffer(o.Body)
|
||||
}
|
||||
|
||||
func decodeRemoteOptions(optionsm map[string]interface{}) (fromRemoteOptions, error) {
|
||||
var options = fromRemoteOptions{
|
||||
Method: "GET",
|
||||
}
|
||||
|
||||
err := mapstructure.WeakDecode(optionsm, &options)
|
||||
if err != nil {
|
||||
return options, err
|
||||
}
|
||||
options.Method = strings.ToUpper(options.Method)
|
||||
|
||||
return options, nil
|
||||
|
||||
}
|
85
resources/resource_factories/create/remote_test.go
Normal file
85
resources/resource_factories/create/remote_test.go
Normal file
|
@ -0,0 +1,85 @@
|
|||
// Copyright 2021 The Hugo Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package create
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
qt "github.com/frankban/quicktest"
|
||||
)
|
||||
|
||||
func TestDecodeRemoteOptions(t *testing.T) {
|
||||
c := qt.New(t)
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args map[string]interface{}
|
||||
want fromRemoteOptions
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
"POST",
|
||||
map[string]interface{}{
|
||||
"meThod": "PoST",
|
||||
"headers": map[string]interface{}{
|
||||
"foo": "bar",
|
||||
},
|
||||
},
|
||||
fromRemoteOptions{
|
||||
Method: "POST",
|
||||
Headers: map[string]interface{}{
|
||||
"foo": "bar",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Body",
|
||||
map[string]interface{}{
|
||||
"meThod": "POST",
|
||||
"body": []byte("foo"),
|
||||
},
|
||||
fromRemoteOptions{
|
||||
Method: "POST",
|
||||
Body: []byte("foo"),
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"Body, string",
|
||||
map[string]interface{}{
|
||||
"meThod": "POST",
|
||||
"body": "foo",
|
||||
},
|
||||
fromRemoteOptions{
|
||||
Method: "POST",
|
||||
Body: []byte("foo"),
|
||||
},
|
||||
false,
|
||||
},
|
||||
} {
|
||||
c.Run(test.name, func(c *qt.C) {
|
||||
got, err := decodeRemoteOptions(test.args)
|
||||
isErr := qt.IsNil
|
||||
if test.wantErr {
|
||||
isErr = qt.IsNotNil
|
||||
}
|
||||
|
||||
c.Assert(err, isErr)
|
||||
c.Assert(got, qt.DeepEquals, test.want)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
}
|
|
@ -783,6 +783,8 @@ func (s *state) validateType(value reflect.Value, typ reflect.Type) reflect.Valu
|
|||
}
|
||||
case reflect.PtrTo(value.Type()).AssignableTo(typ) && value.CanAddr():
|
||||
value = value.Addr()
|
||||
case value.IsZero():
|
||||
s.errorf("got <nil>, expected %s", typ)
|
||||
default:
|
||||
s.errorf("wrong type for value; expected %s; got %s", typ, value.Type())
|
||||
}
|
||||
|
|
|
@ -16,7 +16,6 @@ package resources
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
|
@ -108,42 +107,61 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) {
|
|||
return ns.scssClientDartSass, err
|
||||
}
|
||||
|
||||
// Get locates the filename given in Hugo's assets filesystem or downloads
|
||||
// a file from an URL and creates a Resource object that can be used for
|
||||
// Get locates the filename given in Hugo's assets filesystem and
|
||||
// creates a Resource object that can be used for
|
||||
// further transformations.
|
||||
func (ns *Namespace) Get(filename interface{}) resource.Resource {
|
||||
get := func(args ...interface{}) (resource.Resource, error) {
|
||||
filenamestr, err := cast.ToStringE(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ns.createClient.Get(filepath.Clean(filenamestr))
|
||||
}
|
||||
|
||||
r, err := get(filename)
|
||||
if err != nil {
|
||||
// This allows the client to reason about the .Err in the template.
|
||||
// This is not as relevant for local resources as remotes, but
|
||||
// it makes this method work the same way as resources.GetRemote.
|
||||
return resources.NewErrorResource(errors.Wrap(err, "error calling resources.Get"))
|
||||
}
|
||||
return r
|
||||
|
||||
}
|
||||
|
||||
// GetRemote gets the URL (via HTTP(s)) in the first argument in args and creates Resource object that can be used for
|
||||
// further transformations.
|
||||
//
|
||||
// For URLs an additional argument with options can be provided.
|
||||
func (ns *Namespace) Get(args ...interface{}) resource.Resource {
|
||||
// A second argument may be provided with an option map.
|
||||
func (ns *Namespace) GetRemote(args ...interface{}) resource.Resource {
|
||||
get := func(args ...interface{}) (resource.Resource, error) {
|
||||
if len(args) != 1 && len(args) != 2 {
|
||||
return nil, errors.New("must provide a filename or URL")
|
||||
if len(args) < 1 {
|
||||
return nil, errors.New("must provide an URL")
|
||||
}
|
||||
|
||||
filenamestr, err := cast.ToStringE(args[0])
|
||||
urlstr, err := cast.ToStringE(args[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if u, err := url.Parse(filenamestr); err == nil && u.Scheme != "" {
|
||||
if len(args) == 2 {
|
||||
options, err := maps.ToStringMapE(args[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ns.createClient.FromRemote(filenamestr, options)
|
||||
var options map[string]interface{}
|
||||
|
||||
if len(args) > 1 {
|
||||
options, err = maps.ToStringMapE(args[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ns.createClient.FromRemote(filenamestr, nil)
|
||||
}
|
||||
|
||||
filenamestr = filepath.Clean(filenamestr)
|
||||
return ns.createClient.FromRemote(urlstr, options)
|
||||
|
||||
return ns.createClient.Get(filenamestr)
|
||||
}
|
||||
|
||||
r, err := get(args...)
|
||||
if err != nil {
|
||||
// This allows the client to reason about the .Err in the template.
|
||||
return resources.NewErrorResource(errors.Wrap(err, "error calling resources.Get"))
|
||||
return resources.NewErrorResource(errors.Wrap(err, "error calling resources.GetRemote"))
|
||||
}
|
||||
return r
|
||||
|
||||
|
|
Loading…
Reference in a new issue