diff --git a/media/mediaType.go b/media/mediaType.go index cdfb1c654..5732b9030 100644 --- a/media/mediaType.go +++ b/media/mediaType.go @@ -117,7 +117,7 @@ func FromContent(types Types, extensionHints []string, content []byte) Type { // FromStringAndExt creates a Type from a MIME string and a given extension. func FromStringAndExt(t, ext string) (Type, error) { - tp, err := fromString(t) + tp, err := FromString(t) if err != nil { return tp, err } @@ -129,7 +129,7 @@ func FromStringAndExt(t, ext string) (Type, error) { // FromString creates a new Type given a type string on the form MainType/SubType and // an optional suffix, e.g. "text/html" or "text/html+html". -func fromString(t string) (Type, error) { +func FromString(t string) (Type, error) { t = strings.ToLower(t) parts := strings.Split(t, "/") if len(parts) != 2 { @@ -470,7 +470,7 @@ func DecodeTypes(mms ...map[string]any) (Types, error) { mediaType, found := mmm[k] if !found { var err error - mediaType, err = fromString(k) + mediaType, err = FromString(k) if err != nil { return m, err } diff --git a/media/mediaType_test.go b/media/mediaType_test.go index 2a1b48849..3d12c31bb 100644 --- a/media/mediaType_test.go +++ b/media/mediaType_test.go @@ -132,22 +132,22 @@ func TestGetFirstBySuffix(t *testing.T) { func TestFromTypeString(t *testing.T) { c := qt.New(t) - f, err := fromString("text/html") + f, err := FromString("text/html") c.Assert(err, qt.IsNil) c.Assert(f.Type(), qt.Equals, HTMLType.Type()) - f, err = fromString("application/custom") + f, err = FromString("application/custom") c.Assert(err, qt.IsNil) c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: ""}) - f, err = fromString("application/custom+sfx") + f, err = FromString("application/custom+sfx") c.Assert(err, qt.IsNil) c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: "sfx"}) - _, err = fromString("noslash") + _, err = FromString("noslash") c.Assert(err, qt.Not(qt.IsNil)) - f, err = fromString("text/xml; charset=utf-8") + f, err = FromString("text/xml; charset=utf-8") c.Assert(err, qt.IsNil) c.Assert(f, qt.Equals, Type{MainType: "text", SubType: "xml", mimeSuffix: ""}) diff --git a/resources/resource_factories/create/integration_test.go b/resources/resource_factories/create/integration_test.go new file mode 100644 index 000000000..e3a41d335 --- /dev/null +++ b/resources/resource_factories/create/integration_test.go @@ -0,0 +1,56 @@ +// Copyright 2023 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package create_test + +import ( + "testing" + + "github.com/gohugoio/hugo/hugolib" +) + +func TestGetResourceHead(t *testing.T) { + + files := ` +-- config.toml -- +[security] + [security.http] + methods = ['(?i)GET|POST|HEAD'] + urls = ['.*gohugo\.io.*'] + +-- layouts/index.html -- +{{ $url := "https://gohugo.io/img/hugo.png" }} +{{ $opts := dict "method" "head" }} +{{ with resources.GetRemote $url $opts }} + {{ with .Err }} + {{ errorf "Unable to get remote resource: %s" . }} + {{ else }} + Head Content: {{ .Content }}. + {{ end }} +{{ else }} + {{ errorf "Unable to get remote resource: %s" $url }} +{{ end }} +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + }, + ) + + b.Build() + + b.AssertFileContent("public/index.html", "Head Content: .") + +} diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go index fa5533d7b..5216fff79 100644 --- a/resources/resource_factories/create/remote.go +++ b/resources/resource_factories/create/remote.go @@ -45,7 +45,29 @@ type HTTPError struct { Body string } -func toHTTPError(err error, res *http.Response) *HTTPError { +func responseToData(res *http.Response, readBody bool) map[string]any { + var body []byte + if readBody { + body, _ = ioutil.ReadAll(res.Body) + } + + m := map[string]any{ + "StatusCode": res.StatusCode, + "Status": res.Status, + "TransferEncoding": res.TransferEncoding, + "ContentLength": res.ContentLength, + "ContentType": res.Header.Get("Content-Type"), + } + + if readBody { + m["Body"] = string(body) + } + + return m + +} + +func toHTTPError(err error, res *http.Response, readBody bool) *HTTPError { if err == nil { panic("err is nil") } @@ -56,19 +78,9 @@ func toHTTPError(err error, res *http.Response) *HTTPError { } } - var body []byte - body, _ = ioutil.ReadAll(res.Body) - return &HTTPError{ error: err, - Data: map[string]any{ - "StatusCode": res.StatusCode, - "Status": res.Status, - "Body": string(body), - "TransferEncoding": res.TransferEncoding, - "ContentLength": res.ContentLength, - "ContentType": res.Header.Get("Content-Type"), - }, + Data: responseToData(res, readBody), } } @@ -80,6 +92,12 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou return nil, fmt.Errorf("failed to parse URL for resource %s: %w", uri, err) } + method := "GET" + if s, ok := maps.LookupEqualFold(optionsm, "method"); ok { + method = strings.ToUpper(s.(string)) + } + isHeadMethod := method == "HEAD" + resourceID := calculateResourceID(uri, optionsm) _, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) { @@ -100,15 +118,16 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou if err != nil { return nil, err } + defer res.Body.Close() httpResponse, err := httputil.DumpResponse(res, true) if err != nil { - return nil, toHTTPError(err, res) + return nil, toHTTPError(err, res, !isHeadMethod) } if res.StatusCode != http.StatusNotFound { if res.StatusCode < 200 || res.StatusCode > 299 { - return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res) + return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod) } } @@ -124,15 +143,24 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou if err != nil { return nil, err } + defer res.Body.Close() if res.StatusCode == http.StatusNotFound { // Not found. This matches how looksup for local resources work. return nil, nil } - body, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err) + var ( + body []byte + mediaType media.Type + ) + // A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored. + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD + if !isHeadMethod && res.Body != nil { + body, err = ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err) + } } filename := path.Base(rURL.Path) @@ -142,30 +170,38 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou } } - var extensionHints []string - contentType := res.Header.Get("Content-Type") - // mime.ExtensionsByType gives a long list of extensions for text/plain, - // just use ".txt". - if strings.HasPrefix(contentType, "text/plain") { - extensionHints = []string{".txt"} + if isHeadMethod { + // We have no body to work with, so we need to use the Content-Type header. + mediaType, _ = media.FromString(contentType) } else { - exts, _ := mime.ExtensionsByType(contentType) - if exts != nil { - extensionHints = exts + + var extensionHints []string + + // mime.ExtensionsByType gives a long list of extensions for text/plain, + // just use ".txt". + if strings.HasPrefix(contentType, "text/plain") { + extensionHints = []string{".txt"} + } else { + exts, _ := mime.ExtensionsByType(contentType) + if exts != nil { + extensionHints = exts + } } + + // Look for a file extension. If it's .txt, look for a more specific. + if extensionHints == nil || extensionHints[0] == ".txt" { + if ext := path.Ext(filename); ext != "" { + extensionHints = []string{ext} + } + } + + // Now resolve the media type primarily using the content. + mediaType = media.FromContent(c.rs.MediaTypes, extensionHints, body) + } - // Look for a file extension. If it's .txt, look for a more specific. - if extensionHints == nil || extensionHints[0] == ".txt" { - if ext := path.Ext(filename); ext != "" { - extensionHints = []string{ext} - } - } - - // Now resolve the media type primarily using the content. - mediaType := media.FromContent(c.rs.MediaTypes, extensionHints, body) if mediaType.IsZero() { return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri) }