Add remote support to resources.Get

Closes #5255
Supports #9044

docs/content/en/hugo-pipes/introduction.md

@@ -20,12 +20,39 @@ aliases: [/assets/]
 
 Asset files must be stored in the asset directory. This is `/assets` by default, but can be configured via the configuration file's `assetDir` key.
 
-### From file to resource
+### From file or URL to resource
 
-In order to process an asset with Hugo Pipes, it must be retrieved as a resource using `resources.Get`, which takes one argument: the filepath of the file relative to the asset directory.
+In order to process an asset with Hugo Pipes, it must be retrieved as a resource using `resources.Get`. The first argument can be the filepath of the file relative to the asset directory or the URL of the file.
 
 ```go-html-template
 {{ $style := resources.Get "sass/main.scss" }}
+{{ $remoteStyle := resources.Get "https://www.example.com/styles.scss" }}
+```
+
+#### Request options
+
+When using an URL, the `resources.Get` function takes an optional options map as the last argument, e.g.:
+
+```
+{{ $resource := resources.Get "https://example.org/api" (dict "headers" (dict "Authorization" "Bearer abcd"))  }}
+```
+
+If you need multiple values for the same header key, use a slice:
+
+```
+{{ $resource := resources.Get "https://example.org/api"  (dict "headers" (dict "X-List" (slice "a" "b" "c")))  }}
+```
+
+You can also change the request method and set the request body:
+
+```
+{{ $postResponse := resources.Get "https://example.org/api"  (dict 
+    "method" "post"
+    "body" `{"complete": true}` 
+    "headers" (dict 
+        "Content-Type" "application/json"
+    )
+)}}
 ```
 
 ### Asset publishing

hugolib/resource_chain_test.go

@@ -17,7 +17,10 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
+	"net/http"
+	"net/http/httptest"
 	"os"
 
 	"github.com/gohugoio/hugo/config"
@@ -35,6 +38,7 @@ import (
 
 	"github.com/gohugoio/hugo/common/herrors"
 
+	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/htesting"
 
 	qt "github.com/frankban/quicktest"
@@ -385,8 +389,13 @@ T1: {{ $r.Content }}
 func TestResourceChainBasic(t *testing.T) {
 	t.Parallel()
 
+	ts := httptest.NewServer(http.FileServer(http.Dir("testdata/")))
+	t.Cleanup(func() {
+		ts.Close()
+	})
+
 	b := newTestSitesBuilder(t)
-	b.WithTemplatesAdded("index.html", `
+	b.WithTemplatesAdded("index.html", fmt.Sprintf(`
 {{ $hello := "<h1>     Hello World!   </h1>" | resources.FromString "hello.html" | fingerprint "sha512" | minify  | fingerprint }}
 {{ $cssFingerprinted1 := "body {  background-color: lightblue; }" | resources.FromString "styles.css" |  minify  | fingerprint }}
 {{ $cssFingerprinted2 := "body {  background-color: orange; }" | resources.FromString "styles2.css" |  minify  | fingerprint }}
@@ -403,7 +412,14 @@ FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }}
 CSS integrity Data first: {{ $cssFingerprinted1.Data.Integrity }} {{ $cssFingerprinted1.RelPermalink }}
 CSS integrity Data last:  {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerprinted2.Data.Integrity }}
 
-`)
+{{ $rimg := resources.Get "%[1]s/sunset.jpg" }}
+{{ $rfit := $rimg.Fit "200x200" }}
+{{ $rfit2 := $rfit.Fit "100x200" }}
+{{ $rimg = $rimg | fingerprint }}
+SUNSET REMOTE: {{ $rimg.Name }}|{{ $rimg.RelPermalink }}|{{ $rimg.Width }}|{{ len $rimg.Content }}
+FIT REMOTE: {{ $rfit.Name }}|{{ $rfit.RelPermalink }}|{{ $rfit.Width }}
+
+`, ts.URL))
 
 	fs := b.Fs.Source
 
@@ -424,12 +440,16 @@ CSS integrity Data last:  {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerpri
 		b.Build(BuildCfg{})
 
 		b.AssertFileContent("public/index.html",
-			`
+			fmt.Sprintf(`
 SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
 FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200
 CSS integrity Data first: sha256-od9YaHw8nMOL8mUy97Sy8sKwMV3N4hI3aVmZXATxH&#43;8= /styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css
 CSS integrity Data last:  /styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css sha256-HPxSmGg2QF03&#43;ZmKY/1t2GCOjEEOXj2x2qow94vCc7o=
-`)
+
+SUNSET REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587
+FIT REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s_hu59e56ffff1bc1d8d122b1403d34e039f_0_200x200_fit_q75_box.jpg|200
+
+`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]interface{}{})))
 
 		b.AssertFileContent("public/styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css", "body{background-color:#add8e6}")
 		b.AssertFileContent("public//styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css", "body{background-color:orange}")
@@ -537,6 +557,89 @@ func TestResourceChains(t *testing.T) {
 
 	c := qt.New(t)
 
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/css/styles1.css":
+			w.Header().Set("Content-Type", "text/css")
+			w.Write([]byte(`h1 { 
+				font-style: bold;
+			}`))
+			return
+
+		case "/js/script1.js":
+			w.Write([]byte(`var x; x = 5, document.getElementById("demo").innerHTML = x * 10`))
+			return
+
+		case "/mydata/json1.json":
+			w.Write([]byte(`{
+				"employees": [
+					{
+						"firstName": "John",
+						"lastName": "Doe"
+					},
+					{
+						"firstName": "Anna",
+						"lastName": "Smith"
+					},
+					{
+						"firstName": "Peter",
+						"lastName": "Jones"
+					}
+				]
+			}`))
+			return
+
+		case "/mydata/xml1.xml":
+			w.Write([]byte(`
+				<hello>
+					<world>Hugo Rocks!</<world>
+				</hello>`))
+			return
+
+		case "/mydata/svg1.svg":
+			w.Header().Set("Content-Disposition", `attachment; filename="image.svg"`)
+			w.Write([]byte(`
+				<svg height="100" width="100">
+					<path d="M1e2 1e2H3e2 2e2z"/>
+				</svg>`))
+			return
+
+		case "/mydata/html1.html":
+			w.Write([]byte(`
+				<html>
+					<a href=#>Cool</a>
+				</html>`))
+			return
+
+		case "/authenticated/":
+			if r.Header.Get("Authorization") == "Bearer abcd" {
+				w.Write([]byte(`Welcome`))
+				return
+			}
+			http.Error(w, "Forbidden", http.StatusForbidden)
+			return
+
+		case "/post":
+			if r.Method == http.MethodPost {
+				body, err := ioutil.ReadAll(r.Body)
+				if err != nil {
+					http.Error(w, "Internal server error", http.StatusInternalServerError)
+					return
+				}
+				w.Write(body)
+				return
+			}
+			http.Error(w, "Bad request", http.StatusBadRequest)
+			return
+		}
+
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}))
+	t.Cleanup(func() {
+		ts.Close()
+	})
+
 	tests := []struct {
 		name      string
 		shouldRun func() bool
@@ -578,25 +681,53 @@ T6: {{ $bundle1.Permalink }}
     [minify.tdewolff.html]
       keepWhitespace = false
 `)
-			b.WithTemplates("home.html", `
+			b.WithTemplates("home.html", fmt.Sprintf(`
 Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }}
+Min CSS Remote: {{ ( resources.Get "%[1]s/css/styles1.css" | minify ).Content }}
 Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }}
+Min JS Remote: {{ ( resources.Get "%[1]s/js/script1.js" | minify ).Content }}
 Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }}
+Min JSON Remote: {{ ( resources.Get "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }}
 Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
+Min XML Remote: {{ ( resources.Get "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }}
 Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
+Min SVG Remote: {{ ( resources.Get "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
 Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }}
 Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }}
-
+Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }}
-
+`, ts.URL))
-`)
 		}, func(b *sitesBuilder) {
 			b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`)
+			b.AssertFileContent("public/index.html", `Min CSS Remote: h1{font-style:bold}`)
 			b.AssertFileContent("public/index.html", `Min JS: var x;x=5,document.getElementById(&#34;demo&#34;).innerHTML=x*10`)
+			b.AssertFileContent("public/index.html", `Min JS Remote: var x;x=5,document.getElementById(&#34;demo&#34;).innerHTML=x*10`)
 			b.AssertFileContent("public/index.html", `Min JSON: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`)
+			b.AssertFileContent("public/index.html", `Min JSON Remote: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`)
 			b.AssertFileContent("public/index.html", `Min XML: <hello><world>Hugo Rocks!</<world></hello>`)
+			b.AssertFileContent("public/index.html", `Min XML Remote: <hello><world>Hugo Rocks!</<world></hello>`)
 			b.AssertFileContent("public/index.html", `Min SVG: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
+			b.AssertFileContent("public/index.html", `Min SVG Remote: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
 			b.AssertFileContent("public/index.html", `Min SVG again: <svg height="100" width="100"><path d="M1e2 1e2H3e2 2e2z"/></svg>`)
 			b.AssertFileContent("public/index.html", `Min HTML: <html><a href=#>Cool</a></html>`)
+			b.AssertFileContent("public/index.html", `Min HTML Remote: <html><a href=#>Cool</a></html>`)
+		}},
+
+		{"remote", func() bool { return true }, func(b *sitesBuilder) {
+			b.WithTemplates("home.html", fmt.Sprintf(`
+{{$js := resources.Get "%[1]s/js/script1.js" }}
+Remote Filename: {{ $js.RelPermalink }}
+{{$svg := resources.Get "%[1]s/mydata/svg1.svg" }}
+Remote Content-Disposition: {{ $svg.RelPermalink }}
+{{$auth := resources.Get "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }}
+Remote Authorization: {{ $auth.Content }}
+{{$post := resources.Get "%[1]s/post" (dict "method" "post" "body" "Request body") }}
+Remote POST: {{ $post.Content }}
+`, ts.URL))
+		}, func(b *sitesBuilder) {
+			b.AssertFileContent("public/index.html", `Remote Filename: /script1_`)
+			b.AssertFileContent("public/index.html", `Remote Content-Disposition: /image_`)
+			b.AssertFileContent("public/index.html", `Remote Authorization: Welcome`)
+			b.AssertFileContent("public/index.html", `Remote POST: Request body`)
 		}},
 
 		{"concat", func() bool { return true }, func(b *sitesBuilder) {

resources/resource_factories/create/create.go

@@ -16,28 +16,47 @@
 package create
 
 import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/url"
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/gohugoio/hugo/hugofs/glob"
 
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/gohugoio/hugo/common/hugio"
+	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/common/types"
+	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
+
+	"github.com/pkg/errors"
 )
 
 // Client contains methods to create Resource objects.
 // tasks to Resource objects.
 type Client struct {
-	rs *resources.Spec
+	rs         *resources.Spec
+	httpClient *http.Client
 }
 
 // New creates a new Client with the given specification.
 func New(rs *resources.Spec) *Client {
-	return &Client{rs: rs}
+	return &Client{
+		rs: rs,
+		httpClient: &http.Client{
+			Timeout: 10 * time.Second,
+		},
+	}
 }
 
 // Get creates a new Resource by opening the given filename in the assets filesystem.
@@ -126,3 +145,161 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro
 			})
 	})
 }
+
+// FromRemote expects one or n-parts of a URL to a resource
+// If you provide multiple parts they will be joined together to the final URL.
+func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) {
+	rURL, err := url.Parse(uri)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri)
+	}
+
+	resourceID := helpers.HashString(uri, options)
+
+	return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(resourceID), func() (resource.Resource, error) {
+		method, reqBody, err := getMethodAndBody(options)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri)
+		}
+
+		req, err := http.NewRequest(method, uri, reqBody)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create request for resource %s", uri)
+		}
+		addDefaultHeaders(req)
+
+		if _, ok := options["headers"]; ok {
+			headers, err := maps.ToStringMapE(options["headers"])
+			if err != nil {
+				return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri)
+			}
+			addUserProvidedHeaders(headers, req)
+		}
+		res, err := c.httpClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+
+		if res.StatusCode < 200 || res.StatusCode > 299 {
+			return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode))
+		}
+
+		body, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to read remote resource %s", uri)
+		}
+
+		filename := path.Base(rURL.Path)
+		if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil {
+			if _, ok := params["filename"]; ok {
+				filename = params["filename"]
+			}
+		}
+
+		var contentType string
+		if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 {
+			contentType = arr[0]
+		}
+
+		// If content type was not determined by header, look for a file extention
+		if contentType == "" {
+			if ext := path.Ext(filename); ext != "" {
+				contentType = ext
+			}
+		}
+
+		// If content type was not determined by header or file extention, try using content itself
+		if contentType == "" {
+			if ct := http.DetectContentType(body); ct != "application/octet-stream" {
+				if arr, _ := mime.ExtensionsByType(ct); arr != nil {
+					contentType = arr[0]
+				}
+			}
+		}
+
+		resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType
+
+		return c.rs.New(
+			resources.ResourceSourceDescriptor{
+				Fs:          c.rs.FileCaches.AssetsCache().Fs,
+				LazyPublish: true,
+				OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
+					return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
+				},
+				RelTargetFilename: filepath.Clean(resourceID),
+			})
+	})
+}
+
+func addDefaultHeaders(req *http.Request, accepts ...string) {
+	for _, accept := range accepts {
+		if !hasHeaderValue(req.Header, "Accept", accept) {
+			req.Header.Add("Accept", accept)
+		}
+	}
+	if !hasHeaderKey(req.Header, "User-Agent") {
+		req.Header.Add("User-Agent", "Hugo Static Site Generator")
+	}
+}
+
+func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) {
+	if headers == nil {
+		return
+	}
+	for key, val := range headers {
+		vals := types.ToStringSlicePreserveString(val)
+		for _, s := range vals {
+			req.Header.Add(key, s)
+		}
+	}
+}
+
+func hasHeaderValue(m http.Header, key, value string) bool {
+	var s []string
+	var ok bool
+
+	if s, ok = m[key]; !ok {
+		return false
+	}
+
+	for _, v := range s {
+		if v == value {
+			return true
+		}
+	}
+	return false
+}
+
+func hasHeaderKey(m http.Header, key string) bool {
+	_, ok := m[key]
+	return ok
+}
+
+func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) {
+	if options == nil {
+		return "GET", nil, nil
+	}
+
+	if method, ok := options["method"].(string); ok {
+		method = strings.ToUpper(method)
+		switch method {
+		case "GET", "DELETE", "HEAD", "OPTIONS":
+			return method, nil, nil
+		case "POST", "PUT", "PATCH":
+			var body []byte
+			if _, ok := options["body"]; ok {
+				switch b := options["body"].(type) {
+				case string:
+					body = []byte(b)
+				case []byte:
+					body = b
+				}
+			}
+			return method, bytes.NewBuffer(body), nil
+		}
+
+		return "", nil, fmt.Errorf("invalid HTTP method %q", method)
+	}
+
+	return "GET", nil, nil
+}

tpl/resources/resources.go

@@ -16,6 +16,7 @@ package resources
 
 import (
 	"fmt"
+	"net/url"
 	"path/filepath"
 	"sync"
 
@@ -107,14 +108,32 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) {
 	return ns.scssClientDartSass, err
 }
 
-// Get locates the filename given in Hugo's assets filesystem
+// Get locates the filename given in Hugo's assets filesystem or downloads
-// and creates a Resource object that can be used for further transformations.
+// a file from an URL and creates a Resource object that can be used for
-func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {
+// further transformations.
-	filenamestr, err := cast.ToStringE(filename)
+//
+// For URLs an additional argument with options can be provided.
+func (ns *Namespace) Get(args ...interface{}) (resource.Resource, error) {
+	if len(args) != 1 && len(args) != 2 {
+		return nil, errors.New("must provide a filename or URL")
+	}
+
+	filenamestr, err := cast.ToStringE(args[0])
 	if err != nil {
 		return nil, err
 	}
 
+	if u, err := url.Parse(filenamestr); err == nil && u.Scheme != "" {
+		if len(args) == 2 {
+			options, err := maps.ToStringMapE(args[1])
+			if err != nil {
+				return nil, err
+			}
+			return ns.createClient.FromRemote(filenamestr, options)
+		}
+		return ns.createClient.FromRemote(filenamestr, nil)
+	}
+
 	filenamestr = filepath.Clean(filenamestr)
 
 	return ns.createClient.Get(filenamestr)