Move the emoji parsing to pageparser
This avoids double parsing the page content when `enableEmoji=true`.
This commit also adds some general improvements to the parser, making it in general much faster:
```bash
benchmark old ns/op new ns/op delta
BenchmarkShortcodeLexer-4 90258 101730 +12.71%
BenchmarkParse-4 148940 15037 -89.90%
benchmark old allocs new allocs delta
BenchmarkShortcodeLexer-4 456 700 +53.51%
BenchmarkParse-4 28 33 +17.86%
benchmark old bytes new bytes delta
BenchmarkShortcodeLexer-4 69875 81014 +15.94%
BenchmarkParse-4 8128 8304 +2.17%
```
Running some site benchmarks with Emoji support turned on:
```bash
benchmark old ns/op new ns/op delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4 924556797 818115620 -11.51%
benchmark old allocs new allocs delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4 4112613 4133787 +0.51%
benchmark old bytes new bytes delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4 426982864 424363832 -0.61%
```
Fixes #5534
2018-12-17 15:03:23 -05:00
// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pageparser
import (
"fmt"
"testing"
)
func TestMain ( t * testing . T ) {
t . Parallel ( )
2020-12-02 07:23:25 -05:00
mainTests := [ ] lexerTest {
Move the emoji parsing to pageparser
This avoids double parsing the page content when `enableEmoji=true`.
This commit also adds some general improvements to the parser, making it in general much faster:
```bash
benchmark old ns/op new ns/op delta
BenchmarkShortcodeLexer-4 90258 101730 +12.71%
BenchmarkParse-4 148940 15037 -89.90%
benchmark old allocs new allocs delta
BenchmarkShortcodeLexer-4 456 700 +53.51%
BenchmarkParse-4 28 33 +17.86%
benchmark old bytes new bytes delta
BenchmarkShortcodeLexer-4 69875 81014 +15.94%
BenchmarkParse-4 8128 8304 +2.17%
```
Running some site benchmarks with Emoji support turned on:
```bash
benchmark old ns/op new ns/op delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4 924556797 818115620 -11.51%
benchmark old allocs new allocs delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4 4112613 4133787 +0.51%
benchmark old bytes new bytes delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4 426982864 424363832 -0.61%
```
Fixes #5534
2018-12-17 15:03:23 -05:00
{ "emoji #1" , "Some text with :emoji:" , [ ] Item { nti ( tText , "Some text with " ) , nti ( TypeEmoji , ":emoji:" ) , tstEOF } } ,
{ "emoji #2" , "Some text with :emoji: and some text." , [ ] Item { nti ( tText , "Some text with " ) , nti ( TypeEmoji , ":emoji:" ) , nti ( tText , " and some text." ) , tstEOF } } ,
{ "looks like an emoji #1" , "Some text and then :emoji" , [ ] Item { nti ( tText , "Some text and then " ) , nti ( tText , ":" ) , nti ( tText , "emoji" ) , tstEOF } } ,
{ "looks like an emoji #2" , "Some text and then ::" , [ ] Item { nti ( tText , "Some text and then " ) , nti ( tText , ":" ) , nti ( tText , ":" ) , tstEOF } } ,
{ "looks like an emoji #3" , ":Some :text" , [ ] Item { nti ( tText , ":" ) , nti ( tText , "Some " ) , nti ( tText , ":" ) , nti ( tText , "text" ) , tstEOF } } ,
}
for i , test := range mainTests {
items := collectWithConfig ( [ ] byte ( test . input ) , false , lexMainSection , Config { EnableEmoji : true } )
if ! equal ( items , test . items ) {
got := crLfReplacer . Replace ( fmt . Sprint ( items ) )
expected := crLfReplacer . Replace ( fmt . Sprint ( test . items ) )
t . Errorf ( "[%d] %s: got\n\t%v\nexpected\n\t%v" , i , test . name , got , expected )
}
}
}
