hugo/transform/chain_test.go

package transform

import (
	"bytes"
	"github.com/spf13/hugo/helpers"
	"strings"
	"testing"
)

const H5_JS_CONTENT_DOUBLE_QUOTE = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"/foobar\">Follow up</a></article></body></html>"
const H5_JS_CONTENT_SINGLE_QUOTE = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='/foobar'>Follow up</a></article></body></html>"
const H5_JS_CONTENT_ABS_URL = "<!DOCTYPE html><html><head><script src=\"http://user@host:10234/foobar.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"https://host/foobar\">foobar</a>. Follow up</article></body></html>"
const H5_JS_CONTENT_ABS_URL_SCHEMALESS = "<!DOCTYPE html><html><head><script src=\"//host/foobar.js\"></script><script src='//host2/barfoo.js'></head><body><nav><h1>title</h1></nav><article>content <a href=\"//host/foobar\">foobar</a>. <a href='//host2/foobar'>Follow up</a></article></body></html>"
const CORRECT_OUTPUT_SRC_HREF_DQ = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"http://base/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"http://base/foobar\">Follow up</a></article></body></html>"
const CORRECT_OUTPUT_SRC_HREF_SQ = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='http://base/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='http://base/foobar'>Follow up</a></article></body></html>"

const H5_XML_CONTENT_ABS_URL = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\">&lt;p&gt;&lt;a href=&#34;/foobar&#34;&gt;foobar&lt;/a&gt;&lt;/p&gt; &lt;p&gt;A video: &lt;iframe src=&#39;/foo&#39;&gt;&lt;/iframe&gt;&lt;/p&gt;</content></entry></feed>"
const CORRECT_OUTPUT_SRC_HREF_IN_XML = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\">&lt;p&gt;&lt;a href=&#34;http://base/foobar&#34;&gt;foobar&lt;/a&gt;&lt;/p&gt; &lt;p&gt;A video: &lt;iframe src=&#39;http://base/foo&#39;&gt;&lt;/iframe&gt;&lt;/p&gt;</content></entry></feed>"
const H5_XML_CONTENT_GUARDED = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\">&lt;p&gt;&lt;a href=&#34;//foobar&#34;&gt;foobar&lt;/a&gt;&lt;/p&gt; &lt;p&gt;A video: &lt;iframe src=&#39;//foo&#39;&gt;&lt;/iframe&gt;&lt;/p&gt;</content></entry></feed>"

// additional sanity tests for replacements testing
const REPLACE_1 = "No replacements."
const REPLACE_2 = "ᚠᛇᚻ ᛒᛦᚦ ᚠᚱᚩᚠᚢᚱ\nᚠᛁᚱᚪ ᚷᛖᚻᚹᛦᛚᚳᚢᛗ"

// Issue: 816, schemaless links combined with others
const REPLACE_SCHEMALESS_HTML = `Pre. src='//schemaless' src='/normal'  <a href="//schemaless">Schemaless</a>. <a href="/normal">normal</a>. Post.`
const REPLACE_SCHEMALESS_HTML_CORRECT = `Pre. src='//schemaless' src='http://base/normal'  <a href="//schemaless">Schemaless</a>. <a href="http://base/normal">normal</a>. Post.`
const REPLACE_SCHEMALESS_XML = `Pre. src=&#34;//schemaless&#34; src=&#34;/normal&#34;  <a href=&#39;//schemaless&#39;>Schemaless</a>. <a href=&#39;/normal&#39;>normal</a>. Post.`
const REPLACE_SCHEMALESS_XML_CORRECT = `Pre. src=&#34;//schemaless&#34; src=&#34;http://base/normal&#34;  <a href=&#39;//schemaless&#39;>Schemaless</a>. <a href=&#39;http://base/normal&#39;>normal</a>. Post.`

var abs_url_bench_tests = []test{
	{H5_JS_CONTENT_DOUBLE_QUOTE, CORRECT_OUTPUT_SRC_HREF_DQ},
	{H5_JS_CONTENT_SINGLE_QUOTE, CORRECT_OUTPUT_SRC_HREF_SQ},
	{H5_JS_CONTENT_ABS_URL, H5_JS_CONTENT_ABS_URL},
	{H5_JS_CONTENT_ABS_URL_SCHEMALESS, H5_JS_CONTENT_ABS_URL_SCHEMALESS},
}

var xml_abs_url_bench_tests = []test{
	{H5_XML_CONTENT_ABS_URL, CORRECT_OUTPUT_SRC_HREF_IN_XML},
	{H5_XML_CONTENT_GUARDED, H5_XML_CONTENT_GUARDED},
}

var sanity_tests = []test{{REPLACE_1, REPLACE_1}, {REPLACE_2, REPLACE_2}}
var extra_tests_html = []test{{REPLACE_SCHEMALESS_HTML, REPLACE_SCHEMALESS_HTML_CORRECT}}
var abs_url_tests = append(abs_url_bench_tests, append(sanity_tests, extra_tests_html...)...)
var extra_tests_xml = []test{{REPLACE_SCHEMALESS_XML, REPLACE_SCHEMALESS_XML_CORRECT}}
var xml_abs_url_tests = append(xml_abs_url_bench_tests, append(sanity_tests, extra_tests_xml...)...)

func TestChainZeroTransformers(t *testing.T) {
	tr := NewChain()
	in := new(bytes.Buffer)
	out := new(bytes.Buffer)
	if err := tr.Apply(in, out); err != nil {
		t.Errorf("A zero transformer chain returned an error.")
	}
}

func TestChaingMultipleTransformers(t *testing.T) {
	f1 := func(ct contentTransformer) {
		ct.Write(bytes.Replace(ct.Content(), []byte("f1"), []byte("f1r"), -1))
	}
	f2 := func(ct contentTransformer) {
		ct.Write(bytes.Replace(ct.Content(), []byte("f2"), []byte("f2r"), -1))
	}
	f3 := func(ct contentTransformer) {
		ct.Write(bytes.Replace(ct.Content(), []byte("f3"), []byte("f3r"), -1))
	}

	f4 := func(ct contentTransformer) {
		ct.Write(bytes.Replace(ct.Content(), []byte("f4"), []byte("f4r"), -1))
	}

	tr := NewChain(f1, f2, f3, f4)

	out := new(bytes.Buffer)
	if err := tr.Apply(out, helpers.StringToReader("Test: f4 f3 f1 f2 f1 The End.")); err != nil {
		t.Errorf("Multi transformer chain returned an error: %s", err)
	}

	expected := "Test: f4r f3r f1r f2r f1r The End."

	if string(out.Bytes()) != expected {
		t.Errorf("Expected %s got %s", expected, string(out.Bytes()))
	}
}

func BenchmarkAbsURL(b *testing.B) {
	absURL, _ := absURLFromURL("http://base")
	tr := NewChain(absURL...)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		apply(b.Errorf, tr, abs_url_bench_tests)
	}
}

func TestAbsURL(t *testing.T) {
	absURL, _ := absURLFromURL("http://base")
	tr := NewChain(absURL...)

	apply(t.Errorf, tr, abs_url_tests)

}

func BenchmarkXMLAbsURL(b *testing.B) {
	absURLInXML, _ := absURLInXMLFromURL("http://base")
	tr := NewChain(absURLInXML...)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		apply(b.Errorf, tr, xml_abs_url_bench_tests)
	}
}

func TestXMLAbsURL(t *testing.T) {
	absURLInXML, _ := absURLInXMLFromURL("http://base")
	tr := NewChain(absURLInXML...)
	apply(t.Errorf, tr, xml_abs_url_tests)
}

type errorf func(string, ...interface{})

func apply(ef errorf, tr chain, tests []test) {
	for _, test := range tests {
		out := new(bytes.Buffer)
		err := tr.Apply(out, strings.NewReader(test.content))
		if err != nil {
			ef("Unexpected error: %s", err)
		}
		if test.expected != string(out.Bytes()) {
			ef("Expected:\n%s\nGot:\n%s", test.expected, string(out.Bytes()))
		}
	}
}

type test struct {
	content  string
	expected string
}
-												Chain transformers and test cases

Transformers can now be chained together, working on the output of the
previous run.

											
										
										
											2013-10-01 17:26:21 -04:00
+								package transform
 								import (
 									"bytes"
-												Write to rotating ContentReWriter in transformer chain

This commit adds the interface ContentReWriter in the tranformer chain.

This is backed by two pooled byte buffers, alternating between being the reader or the writer.

This keeps the performance characteristic of the old implementation, but in a thread safe way.

Fixes #911

Benchmark old vs new:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsURL        17614         17384         -1.31%
BenchmarkXMLAbsURL     9431          9248          -1.94%

benchmark              old allocs     new allocs     delta
BenchmarkAbsURL        24             28             +16.67%
BenchmarkXMLAbsURL     12             14             +16.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsURL        3295          3424          +3.92%
BenchmarkXMLAbsURL     1954          1987          +1.69%

											
										
										
											2015-03-17 19:36:48 -04:00
+									"github.com/spf13/hugo/helpers"
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+									"strings"
-												Chain transformers and test cases

Transformers can now be chained together, working on the output of the
previous run.

											
										
										
											2013-10-01 17:26:21 -04:00
+									"testing"
 								)
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+								const H5_JS_CONTENT_DOUBLE_QUOTE = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"/foobar\">Follow up</a></article></body></html>"
 								const H5_JS_CONTENT_SINGLE_QUOTE = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='/foobar'>Follow up</a></article></body></html>"
 								const H5_JS_CONTENT_ABS_URL = "<!DOCTYPE html><html><head><script src=\"http://user@host:10234/foobar.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"https://host/foobar\">foobar</a>. Follow up</article></body></html>"
 								const H5_JS_CONTENT_ABS_URL_SCHEMALESS = "<!DOCTYPE html><html><head><script src=\"//host/foobar.js\"></script><script src='//host2/barfoo.js'></head><body><nav><h1>title</h1></nav><article>content <a href=\"//host/foobar\">foobar</a>. <a href='//host2/foobar'>Follow up</a></article></body></html>"
 								const CORRECT_OUTPUT_SRC_HREF_DQ = "<!DOCTYPE html><html><head><script src=\"foobar.js\"></script><script src=\"http://base/barfoo.js\"></script></head><body><nav><h1>title</h1></nav><article>content <a href=\"foobar\">foobar</a>. <a href=\"http://base/foobar\">Follow up</a></article></body></html>"
 								const CORRECT_OUTPUT_SRC_HREF_SQ = "<!DOCTYPE html><html><head><script src='foobar.js'></script><script src='http://base/barfoo.js'></script></head><body><nav><h1>title</h1></nav><article>content <a href='foobar'>foobar</a>. <a href='http://base/foobar'>Follow up</a></article></body></html>"
-												Chain transformers and test cases

Transformers can now be chained together, working on the output of the
previous run.

											
										
										
											2013-10-01 17:26:21 -04:00
-												Add ability to canonify URLs in rendered XML output.

											
										
										
											2014-12-18 14:59:39 -05:00
+								const H5_XML_CONTENT_ABS_URL = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\">&lt;p&gt;&lt;a href=&#34;/foobar&#34;&gt;foobar&lt;/a&gt;&lt;/p&gt; &lt;p&gt;A video: &lt;iframe src=&#39;/foo&#39;&gt;&lt;/iframe&gt;&lt;/p&gt;</content></entry></feed>"
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+								const CORRECT_OUTPUT_SRC_HREF_IN_XML = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\">&lt;p&gt;&lt;a href=&#34;http://base/foobar&#34;&gt;foobar&lt;/a&gt;&lt;/p&gt; &lt;p&gt;A video: &lt;iframe src=&#39;http://base/foo&#39;&gt;&lt;/iframe&gt;&lt;/p&gt;</content></entry></feed>"
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+								const H5_XML_CONTENT_GUARDED = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?><feed xmlns=\"http://www.w3.org/2005/Atom\"><entry><content type=\"html\">&lt;p&gt;&lt;a href=&#34;//foobar&#34;&gt;foobar&lt;/a&gt;&lt;/p&gt; &lt;p&gt;A video: &lt;iframe src=&#39;//foo&#39;&gt;&lt;/iframe&gt;&lt;/p&gt;</content></entry></feed>"
-												Add ability to canonify URLs in rendered XML output.

											
										
										
											2014-12-18 14:59:39 -05:00
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+								// additional sanity tests for replacements testing
 								const REPLACE_1 = "No replacements."
 								const REPLACE_2 = "ᚠᛇᚻ ᛒᛦᚦ ᚠᚱᚩᚠᚢᚱ\nᚠᛁᚱᚪ ᚷᛖᚻᚹᛦᛚᚳᚢᛗ"
-												Fix canonifyurl vs schemaless links

And looks even faster:

Compared to previous attempt:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        30902         27206         -11.96%
BenchmarkXmlAbsUrl     15389         14216         -7.62%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        33             28             -15.15%
BenchmarkXmlAbsUrl     16             14             -12.50%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        4167          3504          -15.91%
BenchmarkXmlAbsUrl     2057          2048          -0.44%

Compared to before I started all of this:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        36219         27206         -24.88%
BenchmarkXmlAbsUrl     23903         14216         -40.53%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             28             -53.33%
BenchmarkXmlAbsUrl     30             14             -53.33%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5842          3504          -40.02%
BenchmarkXmlAbsUrl     3754          2048          -45.44%

Fixes #816

											
										
										
											2015-02-16 22:33:44 -05:00
+								// Issue: 816, schemaless links combined with others
 								const REPLACE_SCHEMALESS_HTML = `Pre. src='//schemaless' src='/normal'  <a href="//schemaless">Schemaless</a>. <a href="/normal">normal</a>. Post.`
 								const REPLACE_SCHEMALESS_HTML_CORRECT = `Pre. src='//schemaless' src='http://base/normal'  <a href="//schemaless">Schemaless</a>. <a href="http://base/normal">normal</a>. Post.`
 								const REPLACE_SCHEMALESS_XML = `Pre. src=&#34;//schemaless&#34; src=&#34;/normal&#34;  <a href=&#39;//schemaless&#39;>Schemaless</a>. <a href=&#39;/normal&#39;>normal</a>. Post.`
 								const REPLACE_SCHEMALESS_XML_CORRECT = `Pre. src=&#34;//schemaless&#34; src=&#34;http://base/normal&#34;  <a href=&#39;//schemaless&#39;>Schemaless</a>. <a href=&#39;http://base/normal&#39;>normal</a>. Post.`
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+								var abs_url_bench_tests = []test{
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+									{H5_JS_CONTENT_DOUBLE_QUOTE, CORRECT_OUTPUT_SRC_HREF_DQ},
 									{H5_JS_CONTENT_SINGLE_QUOTE, CORRECT_OUTPUT_SRC_HREF_SQ},
 									{H5_JS_CONTENT_ABS_URL, H5_JS_CONTENT_ABS_URL},
 									{H5_JS_CONTENT_ABS_URL_SCHEMALESS, H5_JS_CONTENT_ABS_URL_SCHEMALESS},
-												Chain transformers and test cases

Transformers can now be chained together, working on the output of the
previous run.

											
										
										
											2013-10-01 17:26:21 -04:00
+								}
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+								var xml_abs_url_bench_tests = []test{
-												Add ability to canonify URLs in rendered XML output.

											
										
										
											2014-12-18 14:59:39 -05:00
+									{H5_XML_CONTENT_ABS_URL, CORRECT_OUTPUT_SRC_HREF_IN_XML},
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+									{H5_XML_CONTENT_GUARDED, H5_XML_CONTENT_GUARDED},
-												Add ability to canonify URLs in rendered XML output.

											
										
										
											2014-12-18 14:59:39 -05:00
+								}
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+								var sanity_tests = []test{{REPLACE_1, REPLACE_1}, {REPLACE_2, REPLACE_2}}
-												Fix canonifyurl vs schemaless links

And looks even faster:

Compared to previous attempt:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        30902         27206         -11.96%
BenchmarkXmlAbsUrl     15389         14216         -7.62%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        33             28             -15.15%
BenchmarkXmlAbsUrl     16             14             -12.50%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        4167          3504          -15.91%
BenchmarkXmlAbsUrl     2057          2048          -0.44%

Compared to before I started all of this:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        36219         27206         -24.88%
BenchmarkXmlAbsUrl     23903         14216         -40.53%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             28             -53.33%
BenchmarkXmlAbsUrl     30             14             -53.33%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5842          3504          -40.02%
BenchmarkXmlAbsUrl     3754          2048          -45.44%

Fixes #816

											
										
										
											2015-02-16 22:33:44 -05:00
+								var extra_tests_html = []test{{REPLACE_SCHEMALESS_HTML, REPLACE_SCHEMALESS_HTML_CORRECT}}
 								var abs_url_tests = append(abs_url_bench_tests, append(sanity_tests, extra_tests_html...)...)
 								var extra_tests_xml = []test{{REPLACE_SCHEMALESS_XML, REPLACE_SCHEMALESS_XML_CORRECT}}
 								var xml_abs_url_tests = append(xml_abs_url_bench_tests, append(sanity_tests, extra_tests_xml...)...)
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
-												Remove the hugo-nav function

Remove the hugo-nav since it relied on a slow library.  The current
build reimplements the absurl functionality based on string replace.
Discovered that my prior implementation missed the requirement for
making absolute paths (/path) absolute with the host, whereas a relative
path is left untouched.  Updated the test cases to support this if this
is reimplemented.

											
										
										
											2013-11-05 17:28:06 -05:00
+								func TestChainZeroTransformers(t *testing.T) {
 									tr := NewChain()
 									in := new(bytes.Buffer)
 									out := new(bytes.Buffer)
 									if err := tr.Apply(in, out); err != nil {
 										t.Errorf("A zero transformer chain returned an error.")
 									}
-												Adding benchmark for transformation module.

											
										
										
											2013-10-29 23:24:29 -04:00
+								}
-												Write to rotating ContentReWriter in transformer chain

This commit adds the interface ContentReWriter in the tranformer chain.

This is backed by two pooled byte buffers, alternating between being the reader or the writer.

This keeps the performance characteristic of the old implementation, but in a thread safe way.

Fixes #911

Benchmark old vs new:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsURL        17614         17384         -1.31%
BenchmarkXMLAbsURL     9431          9248          -1.94%

benchmark              old allocs     new allocs     delta
BenchmarkAbsURL        24             28             +16.67%
BenchmarkXMLAbsURL     12             14             +16.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsURL        3295          3424          +3.92%
BenchmarkXMLAbsURL     1954          1987          +1.69%

											
										
										
											2015-03-17 19:36:48 -04:00
+								func TestChaingMultipleTransformers(t *testing.T) {
-												Rename interface contentRewriter to contentTransformer

Is is a much better name.

											
										
										
											2015-03-18 20:55:49 -04:00
+									f1 := func(ct contentTransformer) {
 										ct.Write(bytes.Replace(ct.Content(), []byte("f1"), []byte("f1r"), -1))
-												Write to rotating ContentReWriter in transformer chain

This commit adds the interface ContentReWriter in the tranformer chain.

This is backed by two pooled byte buffers, alternating between being the reader or the writer.

This keeps the performance characteristic of the old implementation, but in a thread safe way.

Fixes #911

Benchmark old vs new:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsURL        17614         17384         -1.31%
BenchmarkXMLAbsURL     9431          9248          -1.94%

benchmark              old allocs     new allocs     delta
BenchmarkAbsURL        24             28             +16.67%
BenchmarkXMLAbsURL     12             14             +16.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsURL        3295          3424          +3.92%
BenchmarkXMLAbsURL     1954          1987          +1.69%

											
										
										
											2015-03-17 19:36:48 -04:00
+									}
-												Rename interface contentRewriter to contentTransformer

Is is a much better name.

											
										
										
											2015-03-18 20:55:49 -04:00
+									f2 := func(ct contentTransformer) {
 										ct.Write(bytes.Replace(ct.Content(), []byte("f2"), []byte("f2r"), -1))
-												Write to rotating ContentReWriter in transformer chain

This commit adds the interface ContentReWriter in the tranformer chain.

This is backed by two pooled byte buffers, alternating between being the reader or the writer.

This keeps the performance characteristic of the old implementation, but in a thread safe way.

Fixes #911

Benchmark old vs new:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsURL        17614         17384         -1.31%
BenchmarkXMLAbsURL     9431          9248          -1.94%

benchmark              old allocs     new allocs     delta
BenchmarkAbsURL        24             28             +16.67%
BenchmarkXMLAbsURL     12             14             +16.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsURL        3295          3424          +3.92%
BenchmarkXMLAbsURL     1954          1987          +1.69%

											
										
										
											2015-03-17 19:36:48 -04:00
+									}
-												Rename interface contentRewriter to contentTransformer

Is is a much better name.

											
										
										
											2015-03-18 20:55:49 -04:00
+									f3 := func(ct contentTransformer) {
 										ct.Write(bytes.Replace(ct.Content(), []byte("f3"), []byte("f3r"), -1))
-												Write to rotating ContentReWriter in transformer chain

This commit adds the interface ContentReWriter in the tranformer chain.

This is backed by two pooled byte buffers, alternating between being the reader or the writer.

This keeps the performance characteristic of the old implementation, but in a thread safe way.

Fixes #911

Benchmark old vs new:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsURL        17614         17384         -1.31%
BenchmarkXMLAbsURL     9431          9248          -1.94%

benchmark              old allocs     new allocs     delta
BenchmarkAbsURL        24             28             +16.67%
BenchmarkXMLAbsURL     12             14             +16.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsURL        3295          3424          +3.92%
BenchmarkXMLAbsURL     1954          1987          +1.69%

											
										
										
											2015-03-17 19:36:48 -04:00
+									}
-												Rename interface contentRewriter to contentTransformer

Is is a much better name.

											
										
										
											2015-03-18 20:55:49 -04:00
+									f4 := func(ct contentTransformer) {
 										ct.Write(bytes.Replace(ct.Content(), []byte("f4"), []byte("f4r"), -1))
-												Write to rotating ContentReWriter in transformer chain

This commit adds the interface ContentReWriter in the tranformer chain.

This is backed by two pooled byte buffers, alternating between being the reader or the writer.

This keeps the performance characteristic of the old implementation, but in a thread safe way.

Fixes #911

Benchmark old vs new:

benchmark              old ns/op     new ns/op     delta
BenchmarkAbsURL        17614         17384         -1.31%
BenchmarkXMLAbsURL     9431          9248          -1.94%

benchmark              old allocs     new allocs     delta
BenchmarkAbsURL        24             28             +16.67%
BenchmarkXMLAbsURL     12             14             +16.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsURL        3295          3424          +3.92%
BenchmarkXMLAbsURL     1954          1987          +1.69%

											
										
										
											2015-03-17 19:36:48 -04:00
+									}
 									tr := NewChain(f1, f2, f3, f4)
 									out := new(bytes.Buffer)
 									if err := tr.Apply(out, helpers.StringToReader("Test: f4 f3 f1 f2 f1 The End.")); err != nil {
 										t.Errorf("Multi transformer chain returned an error: %s", err)
 									}
 									expected := "Test: f4r f3r f1r f2r f1r The End."
 									if string(out.Bytes()) != expected {
 										t.Errorf("Expected %s got %s", expected, string(out.Bytes()))
 									}
 								}
-												Correct initialisms as suggested by golint

First step to use initialisms that golint suggests,
for example:

    Line 116: func GetHtmlRenderer should be GetHTMLRenderer

as see on http://goreportcard.com/report/spf13/hugo

Thanks to @bep for the idea!

Note that command-line flags (cobra and pflag)
as well as struct fields like .BaseUrl and .Url
that are used in Go HTML templates need more work
to maintain backward-compatibility, and thus
are NOT yet dealt with in this commit.

First step in fixing #959.

											
										
										
											2015-03-11 13:34:57 -04:00
+								func BenchmarkAbsURL(b *testing.B) {
-												Remove URL param from exported AbsURL methods

This is to make it clear that there is only one URL in play,
the BaseURL.

											
										
										
											2015-03-18 14:42:46 -04:00
+									absURL, _ := absURLFromURL("http://base")
-												Remove the hugo-nav function

Remove the hugo-nav since it relied on a slow library.  The current
build reimplements the absurl functionality based on string replace.
Discovered that my prior implementation missed the requirement for
making absolute paths (/path) absolute with the host, whereas a relative
path is left untouched.  Updated the test cases to support this if this
is reimplemented.

											
										
										
											2013-11-05 17:28:06 -05:00
+									tr := NewChain(absURL...)
-												Improve rendering time

50% speedup.  Fix #91

to run the benchmark:

		go test -test.run=NONE -bench=".*" -test.benchmem=true ./transform/ > new.txt

to compare the results:

		/usr/local/go/misc/benchcmp baseline.txt new.txt

Speedup and memory improvements

		benchmark             old ns/op    new ns/op    delta
		BenchmarkChain           101219        50453  -50.15%
		BenchmarkTransform        51625        45531  -11.80%

		benchmark            old allocs   new allocs    delta
		BenchmarkChain              222          103  -53.60%
		BenchmarkTransform          135          106  -21.48%

		benchmark             old bytes    new bytes    delta
		BenchmarkChain            23919        10998  -54.02%
		BenchmarkTransform        11858        10665  -10.06%

											
										
										
											2013-11-01 01:14:11 -04:00
 									b.ResetTimer()
-												Adding benchmark for transformation module.

											
										
										
											2013-10-29 23:24:29 -04:00
+									for i := 0; i < b.N; i++ {
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+										apply(b.Errorf, tr, abs_url_bench_tests)
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+									}
 								}
-												Correct initialisms as suggested by golint

First step to use initialisms that golint suggests,
for example:

    Line 116: func GetHtmlRenderer should be GetHTMLRenderer

as see on http://goreportcard.com/report/spf13/hugo

Thanks to @bep for the idea!

Note that command-line flags (cobra and pflag)
as well as struct fields like .BaseUrl and .Url
that are used in Go HTML templates need more work
to maintain backward-compatibility, and thus
are NOT yet dealt with in this commit.

First step in fixing #959.

											
										
										
											2015-03-11 13:34:57 -04:00
+								func TestAbsURL(t *testing.T) {
-												Remove URL param from exported AbsURL methods

This is to make it clear that there is only one URL in play,
the BaseURL.

											
										
										
											2015-03-18 14:42:46 -04:00
+									absURL, _ := absURLFromURL("http://base")
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+									tr := NewChain(absURL...)
 									apply(t.Errorf, tr, abs_url_tests)
 								}
-												Correct initialisms as suggested by golint

First step to use initialisms that golint suggests,
for example:

    Line 116: func GetHtmlRenderer should be GetHTMLRenderer

as see on http://goreportcard.com/report/spf13/hugo

Thanks to @bep for the idea!

Note that command-line flags (cobra and pflag)
as well as struct fields like .BaseUrl and .Url
that are used in Go HTML templates need more work
to maintain backward-compatibility, and thus
are NOT yet dealt with in this commit.

First step in fixing #959.

											
										
										
											2015-03-11 13:34:57 -04:00
+								func BenchmarkXMLAbsURL(b *testing.B) {
-												Remove URL param from exported AbsURL methods

This is to make it clear that there is only one URL in play,
the BaseURL.

											
										
										
											2015-03-18 14:42:46 -04:00
+									absURLInXML, _ := absURLInXMLFromURL("http://base")
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
+									tr := NewChain(absURLInXML...)
 									b.ResetTimer()
 									for i := 0; i < b.N; i++ {
-												Improve abs url replacement speed

This commit replaces the multuple `bytes.Containts` and `bytes.Replace` with a custom replacer that does one pass through the document and exploits the fact that there are two common prefixes we search for, `src=` and `href=`.

This is both faster and consumes less memory. There may be even better algos to use here, but we must leave some room for improvements for future versions.

This should also make it possible to solve #816.

```
benchmark              old ns/op     new ns/op     delta
BenchmarkAbsUrl        25795         22597         -12.40%
BenchmarkXmlAbsUrl     17187         11166         -35.03%

benchmark              old allocs     new allocs     delta
BenchmarkAbsUrl        60             33             -45.00%
BenchmarkXmlAbsUrl     30             16             -46.67%

benchmark              old bytes     new bytes     delta
BenchmarkAbsUrl        5844          4167          -28.70%
BenchmarkXmlAbsUrl     3754          2069          -44.89%
```

Fixes #894

											
										
										
											2015-02-16 04:48:15 -05:00
+										apply(b.Errorf, tr, xml_abs_url_bench_tests)
-												Adding benchmark for transformation module.

											
										
										
											2013-10-29 23:24:29 -04:00
+									}
-												Chain transformers and test cases

Transformers can now be chained together, working on the output of the
previous run.

											
										
										
											2013-10-01 17:26:21 -04:00
+								}
-												Add ability to canonify URLs in rendered XML output.

											
										
										
											2014-12-18 14:59:39 -05:00
-												Correct initialisms as suggested by golint

First step to use initialisms that golint suggests,
for example:

    Line 116: func GetHtmlRenderer should be GetHTMLRenderer

as see on http://goreportcard.com/report/spf13/hugo

Thanks to @bep for the idea!

Note that command-line flags (cobra and pflag)
as well as struct fields like .BaseUrl and .Url
that are used in Go HTML templates need more work
to maintain backward-compatibility, and thus
are NOT yet dealt with in this commit.

First step in fixing #959.

											
										
										
											2015-03-11 13:34:57 -04:00
+								func TestXMLAbsURL(t *testing.T) {
-												Remove URL param from exported AbsURL methods

This is to make it clear that there is only one URL in play,
the BaseURL.

											
										
										
											2015-03-18 14:42:46 -04:00
+									absURLInXML, _ := absURLInXMLFromURL("http://base")
-												Add ability to canonify URLs in rendered XML output.

											
										
										
											2014-12-18 14:59:39 -05:00
+									tr := NewChain(absURLInXML...)
 									apply(t.Errorf, tr, xml_abs_url_tests)
 								}
-												Add benchmark for AbsUrlInXml

And a general test cleanup in /transform.

See #894

											
										
										
											2015-02-12 06:17:59 -05:00
 								type errorf func(string, ...interface{})
 								func apply(ef errorf, tr chain, tests []test) {
 									for _, test := range tests {
 										out := new(bytes.Buffer)
 										err := tr.Apply(out, strings.NewReader(test.content))
 										if err != nil {
 											ef("Unexpected error: %s", err)
 										}
 										if test.expected != string(out.Bytes()) {
 											ef("Expected:\n%s\nGot:\n%s", test.expected, string(out.Bytes()))
 										}
 									}
 								}
 								type test struct {
 									content  string
 									expected string
 								}