github/hugo
1
0
Fork 0
mirror of https://github.com/gohugoio/hugo.git synced 2024-11-21 20:46:30 -05:00
Code Issues Projects Releases Packages Wiki Activity

tpl/data: Clean up data namespace

Browse source 
- Move the main GetCSV and GetJSON into data.go.
- Add error returns to GetCSV and GetJSON.
- Add http client to Namespace for test mocking.
- Send accept headers on remote requests. Fixes #3395
- Return an error on non-2XX HTTP response codes and don't retry.
- Move cache tests to cache_test.go.
This commit is contained in:
Cameron Moore 2017-05-01 22:41:08 -05:00 committed by Bjørn Erik Pedersen
parent 1cf2f3dc4f
commit 08c0de5cc3
5 changed files with 476 additions and 310 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

63
tpl/data/cache_test.go Normal file
View file

@ -0,0 +1,63 @@
// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data
import (
"fmt"
"testing"
"github.com/spf13/afero"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
)
func TestCache(t *testing.T) {
t.Parallel()
fs := new(afero.MemMapFs)
for i, test := range []struct {
path string
content []byte
ignore bool
}{
{"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false},
{"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false},
{"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false},
{"трям/трям", []byte(`T€st трям/трям Content 123`), false},
{"은행", []byte(`T€st C은행ontent 123`), false},
{"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false},
{"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true},
} {
msg := fmt.Sprintf("Test #%d: %v", i, test)
cfg := viper.New()
c, err := getCache(test.path, fs, cfg, test.ignore)
assert.NoError(t, err, msg)
assert.Nil(t, c, msg)
err = writeCache(test.path, test.content, fs, cfg, test.ignore)
assert.NoError(t, err, msg)
c, err = getCache(test.path, fs, cfg, test.ignore)
assert.NoError(t, err, msg)
if test.ignore {
assert.Nil(t, c, msg)
} else {
assert.Equal(t, string(test.content), string(c))
}
}
}

114
tpl/data/data.go
View file

@ -13,16 +13,126 @@
package data
import "github.com/spf13/hugo/deps"
import (
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"net/http"
"strings"
"time"
"github.com/spf13/hugo/deps"
jww "github.com/spf13/jwalterweatherman"
)
// New returns a new instance of the data-namespaced template functions.
func New(deps *deps.Deps) *Namespace {
return &Namespace{
deps: deps,
deps: deps,
client: http.DefaultClient,
}
}
// Namespace provides template functions for the "data" namespace.
type Namespace struct {
deps *deps.Deps
client *http.Client
}
// GetCSV expects a data separator and one or n-parts of a URL to a resource which
// can either be a local or a remote one.
// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
// If you provide multiple parts for the URL they will be joined together to the final URL.
// GetCSV returns nil or a slice slice to use in a short code.
func (ns *Namespace) GetCSV(sep string, urlParts ...string) (d [][]string, err error) {
url := strings.Join(urlParts, "")
var clearCacheSleep = func(i int, u string) {
jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
time.Sleep(resSleep)
deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
}
for i := 0; i <= resRetries; i++ {
var req *http.Request
req, err = http.NewRequest("GET", url, nil)
if err != nil {
jww.ERROR.Printf("Failed to create request for getJSON: %s", err)
return nil, err
}
req.Header.Add("Accept", "text/csv")
req.Header.Add("Accept", "text/plain")
var c []byte
c, err = ns.getResource(req)
if err != nil {
jww.ERROR.Printf("Failed to read csv resource %q with error message %s", url, err)
return nil, err
}
if !bytes.Contains(c, []byte(sep)) {
err = errors.New("Cannot find separator " + sep + " in CSV.")
return
}
if d, err = parseCSV(c, sep); err != nil {
jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
clearCacheSleep(i, url)
continue
}
break
}
return
}
// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
// If you provide multiple parts they will be joined together to the final URL.
// GetJSON returns nil or parsed JSON to use in a short code.
func (ns *Namespace) GetJSON(urlParts ...string) (v interface{}, err error) {
url := strings.Join(urlParts, "")
for i := 0; i <= resRetries; i++ {
var req *http.Request
req, err = http.NewRequest("GET", url, nil)
if err != nil {
jww.ERROR.Printf("Failed to create request for getJSON: %s", err)
return nil, err
}
req.Header.Add("Accept", "application/json")
var c []byte
c, err = ns.getResource(req)
if err != nil {
jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
return nil, err
}
err = json.Unmarshal(c, &v)
if err != nil {
jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
time.Sleep(resSleep)
deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
continue
}
break
}
return
}
// parseCSV parses bytes of CSV data into a slice slice string or an error
func parseCSV(c []byte, sep string) ([][]string, error) {
if len(sep) != 1 {
return nil, errors.New("Incorrect length of csv separator: " + sep)
}
b := bytes.NewReader(c)
r := csv.NewReader(b)
rSep := []rune(sep)
r.Comma = rSep[0]
r.FieldsPerRecord = 0
return r.ReadAll()
}

251
tpl/data/data_test.go Normal file
View file

@ -0,0 +1,251 @@
// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data
import (
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetCSV(t *testing.T) {
t.Parallel()
ns := New(newDeps(viper.New()))
for i, test := range []struct {
sep string
url string
content string
expect interface{}
}{
// Remotes
{
",",
`http://success/`,
"gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm\n",
[][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}},
},
{
",",
`http://error.extra.field/`,
"gomeetup,city\nyes,Sydney\nyes,San Francisco\nyes,Stockholm,EXTRA\n",
false,
},
{
",",
`http://error.no.sep/`,
"gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
false,
},
{
",",
`http://nofound/404`,
``,
false,
},
// Locals
{
";",
"pass/semi",
"gomeetup;city\nyes;Sydney\nyes;San Francisco\nyes;Stockholm\n",
[][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}},
},
{
";",
"fail/no-file",
"",
false,
},
} {
msg := fmt.Sprintf("Test %d", i)
// Setup HTTP test server
var srv *httptest.Server
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
if !haveHeader(r.Header, "Accept", "text/csv") && !haveHeader(r.Header, "Accept", "text/plain") {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
if r.URL.Path == "/404" {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
w.Header().Add("Content-type", "text/csv")
w.Write([]byte(test.content))
})
defer func() { srv.Close() }()
// Setup local test file for schema-less URLs
if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
require.NoError(t, err, msg)
f.WriteString(test.content)
f.Close()
}
// Get on with it
got, err := ns.GetCSV(test.sep, test.url)
if _, ok := test.expect.(bool); ok {
assert.Error(t, err, msg)
continue
}
require.NoError(t, err, msg)
require.NotNil(t, got, msg)
assert.EqualValues(t, test.expect, got, msg)
}
}
func TestGetJSON(t *testing.T) {
t.Parallel()
ns := New(newDeps(viper.New()))
for i, test := range []struct {
url string
content string
expect interface{}
}{
{
`http://success/`,
`{"gomeetup":["Sydney","San Francisco","Stockholm"]}`,
map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}},
},
{
`http://malformed/`,
`{gomeetup:["Sydney","San Francisco","Stockholm"]}`,
false,
},
{
`http://nofound/404`,
``,
false,
},
// Locals
{
"pass/semi",
`{"gomeetup":["Sydney","San Francisco","Stockholm"]}`,
map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}},
},
{
"fail/no-file",
"",
false,
},
} {
msg := fmt.Sprintf("Test %d", i)
// Setup HTTP test server
var srv *httptest.Server
srv, ns.client = getTestServer(func(w http.ResponseWriter, r *http.Request) {
if !haveHeader(r.Header, "Accept", "application/json") {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
if r.URL.Path == "/404" {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
w.Header().Add("Content-type", "application/json")
w.Write([]byte(test.content))
})
defer func() { srv.Close() }()
// Setup local test file for schema-less URLs
if !strings.Contains(test.url, ":") && !strings.HasPrefix(test.url, "fail/") {
f, err := ns.deps.Fs.Source.Create(filepath.Join(ns.deps.Cfg.GetString("workingDir"), test.url))
require.NoError(t, err, msg)
f.WriteString(test.content)
f.Close()
}
// Get on with it
got, err := ns.GetJSON(test.url)
if _, ok := test.expect.(bool); ok {
assert.Error(t, err, msg)
continue
}
require.NoError(t, err, msg)
require.NotNil(t, got, msg)
assert.EqualValues(t, test.expect, got, msg)
}
}
func TestParseCSV(t *testing.T) {
t.Parallel()
for i, test := range []struct {
csv []byte
sep string
exp string
err bool
}{
{[]byte("a,b,c\nd,e,f\n"), "", "", true},
{[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
{[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
{[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
{[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
{[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
} {
msg := fmt.Sprintf("Test %d: %v", i, test)
csv, err := parseCSV(test.csv, test.sep)
if test.err {
assert.Error(t, err, msg)
continue
}
require.NoError(t, err, msg)
act := ""
for _, v := range csv {
act = act + strings.Join(v, "")
}
assert.Equal(t, test.exp, act, msg)
}
}
func haveHeader(m http.Header, key, needle string) bool {
var s []string
var ok bool
if s, ok = m[key]; !ok {
return false
}
for _, v := range s {
if v == needle {
return true
}
}
return false
}

124
tpl/data/resources.go
View file

@ -14,14 +14,10 @@
package data
import (
"bytes"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"path/filepath"
"strings"
"sync"
"time"
@ -67,14 +63,16 @@ func (l *remoteLock) URLUnlock(url string) {
}
// getRemote loads the content of a remote file. This method is thread safe.
func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
func getRemote(req *http.Request, fs afero.Fs, cfg config.Provider, hc *http.Client) ([]byte, error) {
url := req.URL.String()
c, err := getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
if c != nil && err == nil {
return c, nil
}
if err != nil {
return nil, err
}
if c != nil {
return c, nil
}
// avoid race condition with locks, block other goroutines if the current url is processing
remoteURLLock.URLLock(url)
@ -82,27 +80,34 @@ func getRemote(url string, fs afero.Fs, cfg config.Provider, hc *http.Client) ([
// avoid multiple locks due to calling getCache twice
c, err = getCache(url, fs, cfg, cfg.GetBool("ignoreCache"))
if c != nil && err == nil {
if err != nil {
return nil, err
}
if c != nil {
return c, nil
}
jww.INFO.Printf("Downloading: %s ...", url)
res, err := hc.Do(req)
if err != nil {
return nil, err
}
jww.INFO.Printf("Downloading: %s ...", url)
res, err := hc.Get(url)
if err != nil {
return nil, err
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, fmt.Errorf("Failed to retrieve remote file: %s", http.StatusText(res.StatusCode))
}
c, err = ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
return nil, err
}
err = writeCache(url, c, fs, cfg, cfg.GetBool("ignoreCache"))
if err != nil {
return nil, err
}
jww.INFO.Printf("... and cached to: %s", getCacheFileID(cfg, url))
return c, nil
}
@ -119,90 +124,11 @@ func getLocal(url string, fs afero.Fs, cfg config.Provider) ([]byte, error) {
}
// getResource loads the content of a local or remote file
func (ns *Namespace) getResource(url string) ([]byte, error) {
if url == "" {
return nil, nil
func (ns *Namespace) getResource(req *http.Request) ([]byte, error) {
switch req.URL.Scheme {
case "":
return getLocal(req.URL.String(), ns.deps.Fs.Source, ns.deps.Cfg)
default:
return getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, ns.client)
}
if strings.Contains(url, "://") {
return getRemote(url, ns.deps.Fs.Source, ns.deps.Cfg, http.DefaultClient)
}
return getLocal(url, ns.deps.Fs.Source, ns.deps.Cfg)
}
// GetJSON expects one or n-parts of a URL to a resource which can either be a local or a remote one.
// If you provide multiple parts they will be joined together to the final URL.
// GetJSON returns nil or parsed JSON to use in a short code.
func (ns *Namespace) GetJSON(urlParts ...string) interface{} {
var v interface{}
url := strings.Join(urlParts, "")
for i := 0; i <= resRetries; i++ {
c, err := ns.getResource(url)
if err != nil {
jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err)
return nil
}
err = json.Unmarshal(c, &v)
if err != nil {
jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err)
jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
time.Sleep(resSleep)
deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
continue
}
break
}
return v
}
// parseCSV parses bytes of CSV data into a slice slice string or an error
func parseCSV(c []byte, sep string) ([][]string, error) {
if len(sep) != 1 {
return nil, errors.New("Incorrect length of csv separator: " + sep)
}
b := bytes.NewReader(c)
r := csv.NewReader(b)
rSep := []rune(sep)
r.Comma = rSep[0]
r.FieldsPerRecord = 0
return r.ReadAll()
}
// GetCSV expects a data separator and one or n-parts of a URL to a resource which
// can either be a local or a remote one.
// The data separator can be a comma, semi-colon, pipe, etc, but only one character.
// If you provide multiple parts for the URL they will be joined together to the final URL.
// GetCSV returns nil or a slice slice to use in a short code.
func (ns *Namespace) GetCSV(sep string, urlParts ...string) [][]string {
var d [][]string
url := strings.Join(urlParts, "")
var clearCacheSleep = func(i int, u string) {
jww.ERROR.Printf("Retry #%d for %s and sleeping for %s", i, url, resSleep)
time.Sleep(resSleep)
deleteCache(url, ns.deps.Fs.Source, ns.deps.Cfg)
}
for i := 0; i <= resRetries; i++ {
c, err := ns.getResource(url)
if err == nil && !bytes.Contains(c, []byte(sep)) {
err = errors.New("Cannot find separator " + sep + " in CSV.")
}
if err != nil {
jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err)
clearCacheSleep(i, url)
continue
}
if d, err = parseCSV(c, sep); err != nil {
jww.ERROR.Printf("Failed to parse csv file %s with error message %s", url, err)
clearCacheSleep(i, url)
continue
}
break
}
return d
}

234
tpl/data/resources_test.go
View file

@ -19,7 +19,6 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync"
"testing"
"time"
@ -31,58 +30,9 @@ import (
"github.com/spf13/hugo/hugofs"
"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestScpCache(t *testing.T) {
t.Parallel()
tests := []struct {
path string
content []byte
ignore bool
}{
{"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false},
{"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false},
{"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false},
{"трям/трям", []byte(`T€st трям/трям Content 123`), false},
{"은행", []byte(`T€st C은행ontent 123`), false},
{"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false},
{"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true},
}
fs := new(afero.MemMapFs)
for _, test := range tests {
cfg := viper.New()
c, err := getCache(test.path, fs, cfg, test.ignore)
if err != nil {
t.Errorf("Error getting cache: %s", err)
}
if c != nil {
t.Errorf("There is content where there should not be anything: %s", string(c))
}
err = writeCache(test.path, test.content, fs, cfg, test.ignore)
if err != nil {
t.Errorf("Error writing cache: %s", err)
}
c, err = getCache(test.path, fs, cfg, test.ignore)
if err != nil {
t.Errorf("Error getting cache after writing: %s", err)
}
if test.ignore {
if c != nil {
t.Errorf("Cache ignored but content is not nil: %s", string(c))
}
} else {
if !bytes.Equal(c, test.content) {
t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c))
}
}
}
}
func TestScpGetLocal(t *testing.T) {
t.Parallel()
v := viper.New()
@ -146,6 +96,10 @@ func TestScpGetRemote(t *testing.T) {
}
for _, test := range tests {
msg := fmt.Sprintf("%v", test)
req, err := http.NewRequest("GET", test.path, nil)
require.NoError(t, err, msg)
srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
w.Write(test.content)
@ -154,41 +108,38 @@ func TestScpGetRemote(t *testing.T) {
cfg := viper.New()
c, err := getRemote(test.path, fs, cfg, cl)
if err != nil {
t.Errorf("Error getting resource content: %s", err)
}
if !bytes.Equal(c, test.content) {
t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c))
}
cc, cErr := getCache(test.path, fs, cfg, test.ignore)
if cErr != nil {
t.Error(cErr)
}
c, err := getRemote(req, fs, cfg, cl)
require.NoError(t, err, msg)
assert.Equal(t, string(test.content), string(c))
c, err = getCache(req.URL.String(), fs, cfg, test.ignore)
require.NoError(t, err, msg)
if test.ignore {
if cc != nil {
t.Errorf("Cache ignored but content is not nil: %s", string(cc))
}
assert.Empty(t, c, msg)
} else {
if !bytes.Equal(cc, test.content) {
t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(cc))
}
assert.Equal(t, string(test.content), string(c))
}
}
}
func TestScpGetRemoteParallel(t *testing.T) {
t.Parallel()
fs := new(afero.MemMapFs)
ns := New(newDeps(viper.New()))
content := []byte(`T€st Content 123`)
url := "http://Foo.Bar/foo_Bar-Foo"
srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) {
w.Write(content)
})
defer func() { srv.Close() }()
for _, ignoreCache := range []bool{false, true} {
url := "http://Foo.Bar/foo_Bar-Foo"
req, err := http.NewRequest("GET", url, nil)
require.NoError(t, err)
for _, ignoreCache := range []bool{false, true} {
cfg := viper.New()
cfg.Set("ignoreCache", ignoreCache)
@ -199,13 +150,9 @@ func TestScpGetRemoteParallel(t *testing.T) {
go func(gor int) {
defer wg.Done()
for j := 0; j < 10; j++ {
c, err := getRemote(url, fs, cfg, cl)
if err != nil {
t.Errorf("Error getting resource content: %s", err)
}
if !bytes.Equal(c, content) {
t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(content), string(c))
}
c, err := getRemote(req, ns.deps.Fs.Source, ns.deps.Cfg, cl)
assert.NoError(t, err)
assert.Equal(t, string(content), string(c))
time.Sleep(23 * time.Millisecond)
}
@ -214,137 +161,6 @@ func TestScpGetRemoteParallel(t *testing.T) {
wg.Wait()
}
t.Log("Done!")
}
func TestParseCSV(t *testing.T) {
t.Parallel()
tests := []struct {
csv []byte
sep string
exp string
err bool
}{
{[]byte("a,b,c\nd,e,f\n"), "", "", true},
{[]byte("a,b,c\nd,e,f\n"), "~/", "", true},
{[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false},
{[]byte("q,w,e\nd,e,f"), ",", "qwedef", false},
{[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true},
{[]byte("z|y|c\nd|e|f"), "|", "zycdef", false},
}
for _, test := range tests {
csv, err := parseCSV(test.csv, test.sep)
if test.err && err == nil {
t.Error("Expecting an error")
}
if test.err {
continue
}
if !test.err && err != nil {
t.Error(err)
}
act := ""
for _, v := range csv {
act = act + strings.Join(v, "")
}
if act != test.exp {
t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv)
}
}
}
func TestGetJSONFailParse(t *testing.T) {
t.Parallel()
ns := New(newDeps(viper.New()))
reqCount := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if reqCount > 0 {
w.Header().Add("Content-type", "application/json")
fmt.Fprintln(w, `{"gomeetup":["Sydney", "San Francisco", "Stockholm"]}`)
} else {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, `ERROR 500`)
}
reqCount++
}))
defer ts.Close()
url := ts.URL + "/test.json"
want := map[string]interface{}{"gomeetup": []interface{}{"Sydney", "San Francisco", "Stockholm"}}
have := ns.GetJSON(url)
assert.NotNil(t, have)
if have != nil {
assert.EqualValues(t, want, have)
}
}
func TestGetCSVFailParseSep(t *testing.T) {
t.Parallel()
ns := New(newDeps(viper.New()))
reqCount := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if reqCount > 0 {
w.Header().Add("Content-type", "application/json")
fmt.Fprintln(w, `gomeetup,city`)
fmt.Fprintln(w, `yes,Sydney`)
fmt.Fprintln(w, `yes,San Francisco`)
fmt.Fprintln(w, `yes,Stockholm`)
} else {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, `ERROR 500`)
}
reqCount++
}))
defer ts.Close()
url := ts.URL + "/test.csv"
want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}
have := ns.GetCSV(",", url)
assert.NotNil(t, have)
if have != nil {
assert.EqualValues(t, want, have)
}
}
func TestGetCSVFailParse(t *testing.T) {
t.Parallel()
ns := New(newDeps(viper.New()))
reqCount := 0
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-type", "application/json")
if reqCount > 0 {
fmt.Fprintln(w, `gomeetup,city`)
fmt.Fprintln(w, `yes,Sydney`)
fmt.Fprintln(w, `yes,San Francisco`)
fmt.Fprintln(w, `yes,Stockholm`)
} else {
fmt.Fprintln(w, `gomeetup,city`)
fmt.Fprintln(w, `yes,Sydney,Bondi,`) // wrong number of fields in line
fmt.Fprintln(w, `yes,San Francisco`)
fmt.Fprintln(w, `yes,Stockholm`)
}
reqCount++
}))
defer ts.Close()
url := ts.URL + "/test.csv"
want := [][]string{{"gomeetup", "city"}, {"yes", "Sydney"}, {"yes", "San Francisco"}, {"yes", "Stockholm"}}
have := ns.GetCSV(",", url)
assert.NotNil(t, have)
if have != nil {
assert.EqualValues(t, want, have)
}
}
func newDeps(cfg config.Provider) *deps.Deps {