diff --git a/docs/content/en/hugo-pipes/introduction.md b/docs/content/en/hugo-pipes/introduction.md index bd5bfdc4c..7a0ce5fc5 100755 --- a/docs/content/en/hugo-pipes/introduction.md +++ b/docs/content/en/hugo-pipes/introduction.md @@ -20,12 +20,39 @@ aliases: [/assets/] Asset files must be stored in the asset directory. This is `/assets` by default, but can be configured via the configuration file's `assetDir` key. -### From file to resource +### From file or URL to resource -In order to process an asset with Hugo Pipes, it must be retrieved as a resource using `resources.Get`, which takes one argument: the filepath of the file relative to the asset directory. +In order to process an asset with Hugo Pipes, it must be retrieved as a resource using `resources.Get`. The first argument can be the filepath of the file relative to the asset directory or the URL of the file. ```go-html-template {{ $style := resources.Get "sass/main.scss" }} +{{ $remoteStyle := resources.Get "https://www.example.com/styles.scss" }} +``` + +#### Request options + +When using an URL, the `resources.Get` function takes an optional options map as the last argument, e.g.: + +``` +{{ $resource := resources.Get "https://example.org/api" (dict "headers" (dict "Authorization" "Bearer abcd")) }} +``` + +If you need multiple values for the same header key, use a slice: + +``` +{{ $resource := resources.Get "https://example.org/api" (dict "headers" (dict "X-List" (slice "a" "b" "c"))) }} +``` + +You can also change the request method and set the request body: + +``` +{{ $postResponse := resources.Get "https://example.org/api" (dict + "method" "post" + "body" `{"complete": true}` + "headers" (dict + "Content-Type" "application/json" + ) +)}} ``` ### Asset publishing diff --git a/hugolib/resource_chain_test.go b/hugolib/resource_chain_test.go index 85b1b3abd..0a997cda3 100644 --- a/hugolib/resource_chain_test.go +++ b/hugolib/resource_chain_test.go @@ -17,7 +17,10 @@ import ( "bytes" "fmt" "io" + "io/ioutil" "math/rand" + "net/http" + "net/http/httptest" "os" "github.com/gohugoio/hugo/config" @@ -35,6 +38,7 @@ import ( "github.com/gohugoio/hugo/common/herrors" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/htesting" qt "github.com/frankban/quicktest" @@ -385,8 +389,13 @@ T1: {{ $r.Content }} func TestResourceChainBasic(t *testing.T) { t.Parallel() + ts := httptest.NewServer(http.FileServer(http.Dir("testdata/"))) + t.Cleanup(func() { + ts.Close() + }) + b := newTestSitesBuilder(t) - b.WithTemplatesAdded("index.html", ` + b.WithTemplatesAdded("index.html", fmt.Sprintf(` {{ $hello := "

Hello World!

" | resources.FromString "hello.html" | fingerprint "sha512" | minify | fingerprint }} {{ $cssFingerprinted1 := "body { background-color: lightblue; }" | resources.FromString "styles.css" | minify | fingerprint }} {{ $cssFingerprinted2 := "body { background-color: orange; }" | resources.FromString "styles2.css" | minify | fingerprint }} @@ -403,7 +412,14 @@ FIT: {{ $fit.Name }}|{{ $fit.RelPermalink }}|{{ $fit.Width }} CSS integrity Data first: {{ $cssFingerprinted1.Data.Integrity }} {{ $cssFingerprinted1.RelPermalink }} CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerprinted2.Data.Integrity }} -`) +{{ $rimg := resources.Get "%[1]s/sunset.jpg" }} +{{ $rfit := $rimg.Fit "200x200" }} +{{ $rfit2 := $rfit.Fit "100x200" }} +{{ $rimg = $rimg | fingerprint }} +SUNSET REMOTE: {{ $rimg.Name }}|{{ $rimg.RelPermalink }}|{{ $rimg.Width }}|{{ len $rimg.Content }} +FIT REMOTE: {{ $rfit.Name }}|{{ $rfit.RelPermalink }}|{{ $rfit.Width }} + +`, ts.URL)) fs := b.Fs.Source @@ -424,12 +440,16 @@ CSS integrity Data last: {{ $cssFingerprinted2.RelPermalink }} {{ $cssFingerpri b.Build(BuildCfg{}) b.AssertFileContent("public/index.html", - ` + fmt.Sprintf(` SUNSET: images/sunset.jpg|/images/sunset.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587 FIT: images/sunset.jpg|/images/sunset_hu59e56ffff1bc1d8d122b1403d34e039f_90587_200x200_fit_q75_box.jpg|200 CSS integrity Data first: sha256-od9YaHw8nMOL8mUy97Sy8sKwMV3N4hI3aVmZXATxH+8= /styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css CSS integrity Data last: /styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css sha256-HPxSmGg2QF03+ZmKY/1t2GCOjEEOXj2x2qow94vCc7o= -`) + +SUNSET REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s.a9bf1d944e19c0f382e0d8f51de690f7d0bc8fa97390c4242a86c3e5c0737e71.jpg|900|90587 +FIT REMOTE: sunset_%[1]s.jpg|/sunset_%[1]s_hu59e56ffff1bc1d8d122b1403d34e039f_0_200x200_fit_q75_box.jpg|200 + +`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]interface{}{}))) b.AssertFileContent("public/styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css", "body{background-color:#add8e6}") b.AssertFileContent("public//styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css", "body{background-color:orange}") @@ -537,6 +557,89 @@ func TestResourceChains(t *testing.T) { c := qt.New(t) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/css/styles1.css": + w.Header().Set("Content-Type", "text/css") + w.Write([]byte(`h1 { + font-style: bold; + }`)) + return + + case "/js/script1.js": + w.Write([]byte(`var x; x = 5, document.getElementById("demo").innerHTML = x * 10`)) + return + + case "/mydata/json1.json": + w.Write([]byte(`{ + "employees": [ + { + "firstName": "John", + "lastName": "Doe" + }, + { + "firstName": "Anna", + "lastName": "Smith" + }, + { + "firstName": "Peter", + "lastName": "Jones" + } + ] + }`)) + return + + case "/mydata/xml1.xml": + w.Write([]byte(` + + Hugo Rocks! + `)) + return + + case "/mydata/svg1.svg": + w.Header().Set("Content-Disposition", `attachment; filename="image.svg"`) + w.Write([]byte(` + + + `)) + return + + case "/mydata/html1.html": + w.Write([]byte(` + + Cool + `)) + return + + case "/authenticated/": + if r.Header.Get("Authorization") == "Bearer abcd" { + w.Write([]byte(`Welcome`)) + return + } + http.Error(w, "Forbidden", http.StatusForbidden) + return + + case "/post": + if r.Method == http.MethodPost { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + w.Write(body) + return + } + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + + http.Error(w, "Not found", http.StatusNotFound) + return + })) + t.Cleanup(func() { + ts.Close() + }) + tests := []struct { name string shouldRun func() bool @@ -578,25 +681,53 @@ T6: {{ $bundle1.Permalink }} [minify.tdewolff.html] keepWhitespace = false `) - b.WithTemplates("home.html", ` + b.WithTemplates("home.html", fmt.Sprintf(` Min CSS: {{ ( resources.Get "css/styles1.css" | minify ).Content }} +Min CSS Remote: {{ ( resources.Get "%[1]s/css/styles1.css" | minify ).Content }} Min JS: {{ ( resources.Get "js/script1.js" | resources.Minify ).Content | safeJS }} +Min JS Remote: {{ ( resources.Get "%[1]s/js/script1.js" | minify ).Content }} Min JSON: {{ ( resources.Get "mydata/json1.json" | resources.Minify ).Content | safeHTML }} +Min JSON Remote: {{ ( resources.Get "%[1]s/mydata/json1.json" | resources.Minify ).Content | safeHTML }} Min XML: {{ ( resources.Get "mydata/xml1.xml" | resources.Minify ).Content | safeHTML }} +Min XML Remote: {{ ( resources.Get "%[1]s/mydata/xml1.xml" | resources.Minify ).Content | safeHTML }} Min SVG: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} +Min SVG Remote: {{ ( resources.Get "%[1]s/mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} Min SVG again: {{ ( resources.Get "mydata/svg1.svg" | resources.Minify ).Content | safeHTML }} Min HTML: {{ ( resources.Get "mydata/html1.html" | resources.Minify ).Content | safeHTML }} - - -`) +Min HTML Remote: {{ ( resources.Get "%[1]s/mydata/html1.html" | resources.Minify ).Content | safeHTML }} +`, ts.URL)) }, func(b *sitesBuilder) { b.AssertFileContent("public/index.html", `Min CSS: h1{font-style:bold}`) + b.AssertFileContent("public/index.html", `Min CSS Remote: h1{font-style:bold}`) b.AssertFileContent("public/index.html", `Min JS: var x;x=5,document.getElementById("demo").innerHTML=x*10`) + b.AssertFileContent("public/index.html", `Min JS Remote: var x;x=5,document.getElementById("demo").innerHTML=x*10`) b.AssertFileContent("public/index.html", `Min JSON: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`) + b.AssertFileContent("public/index.html", `Min JSON Remote: {"employees":[{"firstName":"John","lastName":"Doe"},{"firstName":"Anna","lastName":"Smith"},{"firstName":"Peter","lastName":"Jones"}]}`) b.AssertFileContent("public/index.html", `Min XML: Hugo Rocks!`) + b.AssertFileContent("public/index.html", `Min XML Remote: Hugo Rocks!`) b.AssertFileContent("public/index.html", `Min SVG: `) + b.AssertFileContent("public/index.html", `Min SVG Remote: `) b.AssertFileContent("public/index.html", `Min SVG again: `) b.AssertFileContent("public/index.html", `Min HTML: Cool`) + b.AssertFileContent("public/index.html", `Min HTML Remote: Cool`) + }}, + + {"remote", func() bool { return true }, func(b *sitesBuilder) { + b.WithTemplates("home.html", fmt.Sprintf(` +{{$js := resources.Get "%[1]s/js/script1.js" }} +Remote Filename: {{ $js.RelPermalink }} +{{$svg := resources.Get "%[1]s/mydata/svg1.svg" }} +Remote Content-Disposition: {{ $svg.RelPermalink }} +{{$auth := resources.Get "%[1]s/authenticated/" (dict "headers" (dict "Authorization" "Bearer abcd")) }} +Remote Authorization: {{ $auth.Content }} +{{$post := resources.Get "%[1]s/post" (dict "method" "post" "body" "Request body") }} +Remote POST: {{ $post.Content }} +`, ts.URL)) + }, func(b *sitesBuilder) { + b.AssertFileContent("public/index.html", `Remote Filename: /script1_`) + b.AssertFileContent("public/index.html", `Remote Content-Disposition: /image_`) + b.AssertFileContent("public/index.html", `Remote Authorization: Welcome`) + b.AssertFileContent("public/index.html", `Remote POST: Request body`) }}, {"concat", func() bool { return true }, func(b *sitesBuilder) { diff --git a/resources/resource_factories/create/create.go b/resources/resource_factories/create/create.go index 97a3fc0c4..4832b3cbe 100644 --- a/resources/resource_factories/create/create.go +++ b/resources/resource_factories/create/create.go @@ -16,28 +16,47 @@ package create import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "mime" + "net/http" + "net/url" "path" "path/filepath" "strings" + "time" "github.com/gohugoio/hugo/hugofs/glob" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/common/hugio" + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/types" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" + + "github.com/pkg/errors" ) // Client contains methods to create Resource objects. // tasks to Resource objects. type Client struct { - rs *resources.Spec + rs *resources.Spec + httpClient *http.Client } // New creates a new Client with the given specification. func New(rs *resources.Spec) *Client { - return &Client{rs: rs} + return &Client{ + rs: rs, + httpClient: &http.Client{ + Timeout: 10 * time.Second, + }, + } } // Get creates a new Resource by opening the given filename in the assets filesystem. @@ -126,3 +145,161 @@ func (c *Client) FromString(targetPath, content string) (resource.Resource, erro }) }) } + +// FromRemote expects one or n-parts of a URL to a resource +// If you provide multiple parts they will be joined together to the final URL. +func (c *Client) FromRemote(uri string, options map[string]interface{}) (resource.Resource, error) { + rURL, err := url.Parse(uri) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse URL for resource %s", uri) + } + + resourceID := helpers.HashString(uri, options) + + return c.rs.ResourceCache.GetOrCreate(resources.ResourceCacheKey(resourceID), func() (resource.Resource, error) { + method, reqBody, err := getMethodAndBody(options) + if err != nil { + return nil, errors.Wrapf(err, "failed to get method or body for resource %s", uri) + } + + req, err := http.NewRequest(method, uri, reqBody) + if err != nil { + return nil, errors.Wrapf(err, "failed to create request for resource %s", uri) + } + addDefaultHeaders(req) + + if _, ok := options["headers"]; ok { + headers, err := maps.ToStringMapE(options["headers"]) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse request headers for resource %s", uri) + } + addUserProvidedHeaders(headers, req) + } + res, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, errors.Errorf("failed to retrieve remote resource: %s", http.StatusText(res.StatusCode)) + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, errors.Wrapf(err, "failed to read remote resource %s", uri) + } + + filename := path.Base(rURL.Path) + if _, params, _ := mime.ParseMediaType(res.Header.Get("Content-Disposition")); params != nil { + if _, ok := params["filename"]; ok { + filename = params["filename"] + } + } + + var contentType string + if arr, _ := mime.ExtensionsByType(res.Header.Get("Content-Type")); len(arr) == 1 { + contentType = arr[0] + } + + // If content type was not determined by header, look for a file extention + if contentType == "" { + if ext := path.Ext(filename); ext != "" { + contentType = ext + } + } + + // If content type was not determined by header or file extention, try using content itself + if contentType == "" { + if ct := http.DetectContentType(body); ct != "application/octet-stream" { + if arr, _ := mime.ExtensionsByType(ct); arr != nil { + contentType = arr[0] + } + } + } + + resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + contentType + + return c.rs.New( + resources.ResourceSourceDescriptor{ + Fs: c.rs.FileCaches.AssetsCache().Fs, + LazyPublish: true, + OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) { + return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil + }, + RelTargetFilename: filepath.Clean(resourceID), + }) + }) +} + +func addDefaultHeaders(req *http.Request, accepts ...string) { + for _, accept := range accepts { + if !hasHeaderValue(req.Header, "Accept", accept) { + req.Header.Add("Accept", accept) + } + } + if !hasHeaderKey(req.Header, "User-Agent") { + req.Header.Add("User-Agent", "Hugo Static Site Generator") + } +} + +func addUserProvidedHeaders(headers map[string]interface{}, req *http.Request) { + if headers == nil { + return + } + for key, val := range headers { + vals := types.ToStringSlicePreserveString(val) + for _, s := range vals { + req.Header.Add(key, s) + } + } +} + +func hasHeaderValue(m http.Header, key, value string) bool { + var s []string + var ok bool + + if s, ok = m[key]; !ok { + return false + } + + for _, v := range s { + if v == value { + return true + } + } + return false +} + +func hasHeaderKey(m http.Header, key string) bool { + _, ok := m[key] + return ok +} + +func getMethodAndBody(options map[string]interface{}) (string, io.Reader, error) { + if options == nil { + return "GET", nil, nil + } + + if method, ok := options["method"].(string); ok { + method = strings.ToUpper(method) + switch method { + case "GET", "DELETE", "HEAD", "OPTIONS": + return method, nil, nil + case "POST", "PUT", "PATCH": + var body []byte + if _, ok := options["body"]; ok { + switch b := options["body"].(type) { + case string: + body = []byte(b) + case []byte: + body = b + } + } + return method, bytes.NewBuffer(body), nil + } + + return "", nil, fmt.Errorf("invalid HTTP method %q", method) + } + + return "GET", nil, nil +} diff --git a/tpl/resources/resources.go b/tpl/resources/resources.go index 850def00e..a3f3aaa3e 100644 --- a/tpl/resources/resources.go +++ b/tpl/resources/resources.go @@ -16,6 +16,7 @@ package resources import ( "fmt" + "net/url" "path/filepath" "sync" @@ -107,14 +108,32 @@ func (ns *Namespace) getscssClientDartSass() (*dartsass.Client, error) { return ns.scssClientDartSass, err } -// Get locates the filename given in Hugo's assets filesystem -// and creates a Resource object that can be used for further transformations. -func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) { - filenamestr, err := cast.ToStringE(filename) +// Get locates the filename given in Hugo's assets filesystem or downloads +// a file from an URL and creates a Resource object that can be used for +// further transformations. +// +// For URLs an additional argument with options can be provided. +func (ns *Namespace) Get(args ...interface{}) (resource.Resource, error) { + if len(args) != 1 && len(args) != 2 { + return nil, errors.New("must provide a filename or URL") + } + + filenamestr, err := cast.ToStringE(args[0]) if err != nil { return nil, err } + if u, err := url.Parse(filenamestr); err == nil && u.Scheme != "" { + if len(args) == 2 { + options, err := maps.ToStringMapE(args[1]) + if err != nil { + return nil, err + } + return ns.createClient.FromRemote(filenamestr, options) + } + return ns.createClient.FromRemote(filenamestr, nil) + } + filenamestr = filepath.Clean(filenamestr) return ns.createClient.Get(filenamestr)